打包

db_helper.py

@@ -63,7 +63,14 @@ class DBHelper:
 
         # 添加ON DUPLICATE KEY UPDATE（MySQL语法）
         if conflict_columns and update_columns:
-            update_set = ', '.join([f"{col}=VALUES({col})" for col in update_columns])
+            # 1. 处理用户指定的更新字段
+            update_clauses = [f"{col}=VALUES({col})" for col in update_columns]
+
+            # 2. 强制添加create_time=NOW()
+            update_clauses.append("create_time = NOW()")  # 新增
+
+            # 3. 合并所有更新条件
+            update_set = ', '.join(update_clauses)
             sql += f" ON DUPLICATE KEY UPDATE {update_set}"
 
         # 转换数据为字典列表格式

requirements.txt

@@ -0,0 +1,74 @@
+attrs==25.3.0
+Automat==25.4.16
+beautifulsoup4==4.13.4
+bs4==0.0.2
+certifi==2025.4.26
+cffi==1.17.1
+charset-normalizer==3.4.2
+click==8.1.8
+colorama==0.4.6
+colorlog==6.9.0
+constantly==23.10.4
+cryptography==44.0.3
+cssselect==1.3.0
+defusedxml==0.7.1
+et_xmlfile==2.0.0
+fake-useragent==2.2.0
+Faker==37.1.0
+filelock==3.18.0
+greenlet==3.2.1
+h11==0.16.0
+hyperlink==21.0.0
+idna==3.10
+incremental==24.7.2
+itemadapter==0.11.0
+itemloaders==1.3.2
+jmespath==1.0.1
+lxml==5.4.0
+numpy==2.2.5
+openpyxl==3.1.5
+outcome==1.3.0.post0
+packaging==25.0
+pandas==2.2.3
+parsel==1.10.0
+playwright==1.52.0
+Protego==0.4.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.22
+PyDispatcher==2.0.7
+pyee==13.0.0
+PyMySQL==1.1.1
+pyOpenSSL==25.0.0
+PySocks==1.7.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.0
+pytz==2025.2
+queuelib==1.8.0
+rarfile==4.2
+requests==2.32.3
+requests-file==2.1.0
+schedule==1.2.2
+Scrapy==2.13.0
+scrapy-playwright==0.0.43
+selenium==4.32.0
+service-identity==24.2.0
+setuptools==80.4.0
+six==1.17.0
+sniffio==1.3.1
+sortedcontainers==2.4.0
+soupsieve==2.7
+SQLAlchemy==2.0.41
+tldextract==5.3.0
+trio==0.30.0
+trio-websocket==0.12.2
+Twisted==24.11.0
+typing_extensions==4.13.2
+tzdata==2025.2
+urllib3==2.4.0
+w3lib==2.3.1
+webdriver-manager==4.0.2
+websocket-client==1.8.0
+wsproto==1.2.0
+xlrd==2.0.1
+zope.interface==7.2

setup.py

@@ -0,0 +1,39 @@
+import glob
+
+from setuptools import setup, find_packages
+import os
+
+# 读取 requirements.txt 文件内容
+def read_requirements():
+    req_path = "requirements.txt"
+    if not os.path.exists(req_path):
+        return []
+    with open(req_path, encoding="utf-8") as f:
+        return [line.strip() for line in f if line.strip() and not line.startswith("#")]
+
+
+# 找出所有根目录下的.py文件（不包括__init__.py）
+def find_py_modules():
+    return [os.path.splitext(os.path.basename(f))[0]
+            for f in glob.glob("*.py")
+            if os.path.isfile(f) and not f.startswith('__init__')]
+
+
+setup(
+    name="crossborder",
+    version="0.1.0",
+    packages=find_packages(where="."),
+    package_dir={"": "."},
+
+    # 添加根目录下的.py文件
+    py_modules=find_py_modules(),
+
+    include_package_data=True,
+    install_requires=read_requirements(),
+    python_requires='>=3.6',
+    entry_points={
+        'console_scripts': [
+            'run-shandong=shandong.selenium_shandong_download:main',
+        ],
+    },
+)