zcy
/
zhihu_spider
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

driver download lastest

master
ZouJiu 2023-08-29 16:11:05 +08:00
parent 2546508a01
commit 469efcc0ce
1 changed files with 21 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
crawler.py
View File

@ -934,10 +934,19 @@ def login_loadsavecookie():
username = url.split("/")[-1] username = url.split("/")[-1]
return driver, username return driver, username
def zhihu(): def downloaddriver():
# #crawl articles links url = "https://msedgedriver.azureedge.net/116.0.1938.62/edgedriver_win64.zip"
if not os.path.exists(driverpath): if not os.path.exists(driverpath):
response = requests.get("https://msedgedriver.azureedge.net/114.0.1823.67/edgedriver_win64.zip") ret = requests.get("https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/")
ret = BeautifulSoup(ret.content, 'html.parser')
ddl = ret.find_all('a', class_='driver-download__link')
for k in ddl:
key = k.attrs.keys()
href = k.attrs['href']
if 'href' in key and "win64" in href:
url = href
break
response = requests.get(url)
if response.status_code==200: if response.status_code==200:
with open(os.path.join(abspath, 'msedgedriver/edgedriver.zip'), 'wb') as obj: with open(os.path.join(abspath, 'msedgedriver/edgedriver.zip'), 'wb') as obj:
obj.write(response.content) obj.write(response.content)
@ -958,7 +967,15 @@ def zhihu():
if kk < 0: if kk < 0:
break break
driver, username = login_loadsavecookie() def zhihu():
# #crawl articles links
try:
downloaddriver()
driver, username = login_loadsavecookie()
except Exception as e:
os.remove(os.path.join(abspath, 'msedgedriver', "msedgedriver.exe"))
downloaddriver()
driver, username = login_loadsavecookie()
# #crawl think links # #crawl think links
if crawl_think: if crawl_think: