|
|
@@ -169,25 +169,10 @@ def detect_latest_month(driver, url):
|
|
|
log.error("三个月内未找到有效数据")
|
|
|
return None
|
|
|
|
|
|
-def check_internet_connection(url="http://www.baidu.com", timeout=5):
|
|
|
- """检查网络连接"""
|
|
|
- try:
|
|
|
- urllib.request.urlopen(url, timeout=timeout)
|
|
|
- return True
|
|
|
- except urllib.error.URLError:
|
|
|
- return False
|
|
|
-
|
|
|
def crawl_with_selenium(url, mark):
|
|
|
- # if not check_internet_connection():
|
|
|
- # log.error("无法连接到互联网,请检查网络设置")
|
|
|
- # raise Exception("网络连接失败")
|
|
|
-
|
|
|
driver = None
|
|
|
year_month = None
|
|
|
try:
|
|
|
- # 使用WebDriverManager自动管理geckodriver
|
|
|
- # service = Service(GeckoDriverManager().install())
|
|
|
- # driver = webdriver.Firefox(service=service, options=configure_stealth_options())
|
|
|
driver = webdriver.Firefox(options=configure_stealth_options(download_dir))
|
|
|
log.info("Firefox WebDriver初始化成功")
|
|
|
|