crossborder/shandong/selenium_shandong_download.py

import argparse
import random
import time
from datetime import datetime, timedelta

from selenium import webdriver
from selenium.common import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

from utils.db_helper import DBHelper
from shandong.shandong_parse_excel import parse_excel
from utils.constants import DOWNLOAD_DIR
from utils.download_utils import configure_stealth_options, generate_month_sequence, download_excel
from utils.log import log
from utils.parse_utils import traverse_and_process

# 基础配置

MAX_RETRY = 3
BASE_URL = "http://qingdao.customs.gov.cn/qingdao_customs/406535/fdzdgknr30/406514/406515/index.html"
download_dir = DOWNLOAD_DIR / "shandong"



def detect_latest_month(driver):
    """三级回溯智能检测最新有效月份"""
    driver.get(BASE_URL)
    current_date = datetime.now()
    for offset in range(0, 3):
        check_date = current_date - timedelta(days=offset * 30)
        check_year = check_date.year
        check_month = check_date.month

        target_title = f"{check_year}年{check_month}月山东省进出口主要国别（地区）总值"
        try:
            WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.XPATH, f'//a[contains(@title, "{target_title}")]'))
            )
            log.info(f"已找到最新月份数据 {check_year}-{check_month}")
            return check_year, check_month
        except:
            log.error(f"未找到 {target_title}")
            continue
    raise Exception("三个月内未找到有效数据")



def process_month_data(driver, year, month):
    required_titles = [
        f"{year}年{month}月山东省进口20位主要商品总值",
        f"{year}年{month}月山东省出口20位主要商品总值",
        f"{year}年{month}月山东省各地市进出口总值",
        f"{year}年{month}月山东省进出口主要国别（地区）总值"
    ]

    found_count = 0
    links = driver.find_elements(By.XPATH, '//a[contains(@title,"山东省")]')

    for link in links:
        title = link.get_attribute("title")
        if title in required_titles:
            retry = 0
            success = False
            while retry < MAX_RETRY and not success:
                try:
                    url = link.get_attribute("href")
                    download_excel(driver, url, year, month, title, download_dir)
                    found_count += 1
                    time.sleep(random.uniform(0.5, 1.5))  # 下载间隔
                    success = True  # 成功则跳出循环
                except Exception as e:
                    retry += 1
                    log.error(f"下载 {title} 失败（第{retry}次重试）: {e}")
                    if retry < MAX_RETRY:
                        time.sleep(random.uniform(2, 5))  # 随机等待后再试
                    else:
                        log.error(f"{title} 下载已达到最大重试次数，跳过该文件。")
    log.info(f"本页处理完成，找到{found_count}个有效表格")
    return found_count


def reverse_crawler(driver, target_months):
    """逆向分页抓取核心（优化分页逻辑）"""
    processed_months = set()
    # target_months = [(2023, 5), (2023, 4)]
    page = 1
    for year, month in target_months:
        log.info(f"开始处理 {year}年{month}月数据".center(55, "="))

        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.CLASS_NAME, "conList_ul"))
        )

        current_page = 1
        found_tables = 0

        while True:
            # 智能等待页面稳定
            random_sleep(base=2, variance=3)

            try:
                # 动态检测当前页面月份
                # page_year, page_month = extract_page_date(driver)
                log.info(f"当前页面：{driver.current_url}, 第{page}页")
                #
                # # 月份不匹配时中断循环
                # if (page_year, page_month) != (year, month):
                #     break

                # 处理当前页面的表格数据
                found = process_month_data(driver, year, month)
                found_tables += found

                # 完成四个表格采集
                if found_tables >= 4:
                    log.info(f"已完成{year}年{month}月全部表格采集")
                    processed_months.add((year, month))
                    break

                log.info(f"第{page}页已采集表格数：{found_tables}/4,前往下一页采集")
                # 分页操作（增强定位稳定性）
                WebDriverWait(driver, 15).until(
                    EC.element_to_be_clickable((By.XPATH, '//a[contains(text(),"下一页")]'))
                ).click()

                current_page += 1
                page += 1


            except TimeoutException:
                log.error(f"未找到更多分页，已采集表格数：{found_tables}/4")
                break
            except Exception as e:
                log.error(f"分页异常：{str(e)}")
                handle_retry(driver)  # 异常恢复函数
                break

    return processed_months



def random_sleep(base=2, variance=5):
    """智能随机等待"""
    sleep_time = base + random.random() * variance
    time.sleep(sleep_time)

def handle_retry(driver):
    """异常恢复处理"""
    try:
        driver.refresh()
        WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.CLASS_NAME, "conList_ul"))
        )
        log.warning("浏览器异常已恢复")
    except:
        log.error("需要人工干预的严重错误")
        raise


def main():
    """主入口（优化参数处理逻辑）"""
    parser = argparse.ArgumentParser(description='海关数据智能抓取系统')
    parser.add_argument('--year', type=int, default=None,
                        help='终止年份（如2023），未指定时抓取最新两个月')
    args = parser.parse_args()
    log.info("【山东海关】数据抓取开始".center(66, "*"))
    driver = webdriver.Firefox(options=configure_stealth_options(download_dir))
    try:
        # 智能检测最新有效月份
        valid_year, valid_month = detect_latest_month(driver)
        log.info(f"【山东海关】最新数据：{valid_year}年{valid_month:02d}月")

        # 生成目标序列
        if args.year:
            # 指定年份时：从最新月到目标年1月
            target_months = generate_month_sequence(
                start_year=valid_year,
                start_month=valid_month,
                end_year=args.year
            )
        else:
            # 未指定年份时：取最近两个月
            target_months = generate_month_sequence(valid_year, valid_month)

        log.info(f"【山东海关】目标采集月份序列：{target_months}")
        reverse_crawler(driver, target_months)
        log.info(f"{len(target_months)}个月份数据已采集完毕")

    finally:
        driver.quit()
        log.info("【山东海关】数据抓取结束".center(66, "*"))
        log.info("\n山东省数据清洗入库中...")
        traverse_and_process(download_dir, parse_excel, province_name="shandong")
        log.info("\n山东省地级市数据同比更新中...")
        db_helper = DBHelper()
        db_helper.update_prov_yoy("山东省")


if __name__ == "__main__":
    main()