crawl update commit

.idea/Crossborder.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.13 (Crossborder)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -3,4 +3,5 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.13 (Crossborder)" />
   </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
 </project>

.idea/vcs.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

anhui/crawl_gov_anhui_full.py

@@ -15,7 +15,7 @@ from selenium.webdriver.support.ui import WebDriverWait
 import gov_commodity_anhui_city
 import gov_commodity_anhui_country
 import gov_commodity_anhui_import_export
-from com.zf.crawl import base_country_code, base_mysql
+from utils import base_country_code, base_mysql
 
 download_dir = base_country_code.download_dir
 Path(download_dir).mkdir(parents=True, exist_ok=True)

anhui/gov_commodity_anhui_city.py

@@ -3,8 +3,8 @@ from pathlib import Path
 import pandas
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 city_code_map = {
     "安徽省合肥市": "3401",
@@ -71,17 +71,16 @@ def process_folder(path):
             yoy_import_export, yoy_import, yoy_export = 0, 0, 0
             sql = (f"INSERT INTO t_yujin_crossborder_prov_region_trade "
                    f"(crossborder_year, crossborder_year_month, prov_code, prov_name, city_code, city_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-                   f"('{year}', '{year_month_2}', '340000', '安徽省', '{city_code}', '{city_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+                   f"('{year}', '{year_month_2}', '340000', '安徽省', '{city_code}', '{city_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
             sql_arr_copy.append(sql)
 
         # 组装 SQL 语句
         sql = (f"INSERT INTO t_yujin_crossborder_prov_region_trade "
                f"(crossborder_year, crossborder_year_month, prov_code, prov_name, city_code, city_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-               f"('{year}', '{year_month}', '340000', '安徽省', '{city_code}', '{city_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+               f"('{year}', '{year_month}', '340000', '安徽省', '{city_code}', '{city_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
         sql_arr.append(sql)
 
     print(f"√ {year_month} prov_region_trade 成功生成 SQL 文件 size {len(sql_arr)} ")
-    # 解析完后生成sql文件批量入库
     base_mysql.bulk_insert(sql_arr)
     if month == 2:
         print(f"√ {year_month} prov_region_trade 成功生成 SQL 文件 size {len(sql_arr_copy)} ")

anhui/gov_commodity_anhui_country.py

@@ -5,6 +5,8 @@ import pandas as pd
 from com.zf.crawl import base_country_code
 from com.zf.crawl import base_mysql
 
+from utils.base_country_code import format_sql_value
+
 # 排除地区名单
 EXCLUDE_REGIONS = ["亚洲", "非洲", "欧洲", "拉丁美洲", "北美洲", "大洋洲", "南极洲",
                    "东南亚国家联盟", "欧洲联盟", "亚太经济合作组织",
@@ -41,9 +43,6 @@ def process_folder(path):
         col_total_index, col_monthly_export_index, col_monthly_import_index = 3, 7, 11
 
     for index, row in df.iterrows():
-        # if index < 4:
-        #     continue
-
         # 提取国家名称并去除括号内容
         country_name = str(row.values[country_name_index]).strip()
         if country_name.endswith("）") or country_name.endswith(")"):
@@ -72,7 +71,7 @@ def process_folder(path):
             yoy_import_export, yoy_import, yoy_export = 0, 0, 0
             sql = (f"INSERT INTO t_yujin_crossborder_prov_country_trade "
                    f"(crossborder_year, crossborder_year_month, prov_code, prov_name, country_code, country_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-                   f"('{year}', '{year_month_2}', '340000', '安徽省', '{country_code}', '{country_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+                   f"('{year}', '{year_month_2}', '340000', '安徽省', '{country_code}', '{country_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
             sql_arr_copy.append(sql)
 
         # 构建 SQL
@@ -81,15 +80,12 @@ def process_folder(path):
             f"(crossborder_year, crossborder_year_month, prov_code, prov_name, country_code, country_name, "
             f"monthly_total, monthly_export, monthly_import, yoy_import_export, yoy_import, yoy_export, create_time) "
             f"VALUES ('{year}', '{year_month}', '340000', '安徽省', '{country_code}', '{country_name}', "
-            f"'{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', "
+            f"{format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', "
             f"'{yoy_export}', NOW());"
         )
         sql_arr.append(sql)
-    # except Exception as e:
-    #     print(f"{year_month} 处理时发生异常: {str(e)}")
 
     print(f"√ {year_month} 成功生成 SQL 条数: {len(sql_arr)}")
-
     # 批量插入数据库
     base_mysql.bulk_insert(sql_arr)
     if month == 2:

anhui/gov_commodity_anhui_import_export.py

@@ -3,8 +3,9 @@ from pathlib import Path
 
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+
+from utils.base_country_code import format_sql_value
 
 CUSTOM_COMMODITY_REPLACEMENTS = {
     '家具': '家具及其零件',
@@ -107,7 +108,11 @@ def save_to_database(import_df, export_df, year, month):
 
             monthly_import = round(row['import'] * 10000, 4)
             monthly_export = round(row['export'] * 10000, 4)
-            monthly_total = round(monthly_import + monthly_export, 4)
+            monthly_total = round(
+                (0 if pd.isna(monthly_import) else monthly_import) +
+                (0 if pd.isna(monthly_export) else monthly_export),
+                4
+            )
 
             if month == 2:
                 year_month_2 = f'{year}-01'
@@ -116,12 +121,12 @@ def save_to_database(import_df, export_df, year, month):
                 monthly_total = round(monthly_import + monthly_export, 4)
                 sql = (f"INSERT INTO t_yujin_crossborder_prov_commodity_trade "
                        f"(crossborder_year, crossborder_year_month, prov_code, prov_name, commodity_code, commodity_name, monthly_total, monthly_export, monthly_import, create_time) VALUES "
-                       f"('{year}', '{year_month_2}', '340000', '安徽省', '{commodity_code}', '{commodity_name_fix}', {monthly_total}, {monthly_export}, {monthly_import}, now());")
+                       f"('{year}', '{year_month_2}', '340000', '安徽省', '{commodity_code}', '{commodity_name_fix}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, now());")
                 sql_arr_copy.append(sql)
 
             sql = (f"INSERT INTO t_yujin_crossborder_prov_commodity_trade "
                    f"(crossborder_year, crossborder_year_month, prov_code, prov_name, commodity_code, commodity_name, monthly_total, monthly_export, monthly_import, create_time) VALUES "
-                   f"('{year}', '{year_month}', '340000', '安徽省', '{commodity_code}', '{commodity_name_fix}', {monthly_total}, {monthly_export}, {monthly_import}, now());")
+                   f"('{year}', '{year_month}', '340000', '安徽省', '{commodity_code}', '{commodity_name_fix}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, now());")
             sql_arr.append(sql)
 
             processed_commodities.add(commodity_name_fix)

hebei/crawl_gov_hebei_full.py

@@ -11,8 +11,7 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
 import gov_commodity_hebei_import_export
 import gov_commodity_hebei_country
 import gov_commodity_hebei_city
@@ -105,7 +104,11 @@ def find_target_links(driver):
             time.sleep(random.uniform(1, 3))
             elements.click()
 
-            downloaded_file = wait_for_download_complete(existing_files=existing_files)
+            try:
+                downloaded_file = wait_for_download_complete(existing_files=existing_files)
+            except Exception as e:
+                print(f"下载失败: {str(e)}")
+                continue
             year, start_month, month = extract_year_and_month(file_name)
             final_path = Path(download_dir) / year / month / f"{file_name}.xls"
             if os.path.exists(final_path):

hebei/gov_commodity_hebei_city.py

@@ -3,8 +3,8 @@ from pathlib import Path
 import pandas
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 city_code_map = {
     "石家庄市": "130100",
@@ -70,13 +70,13 @@ def process_folder(path):
             monthly_total = round(float(monthly_total) / 2, 4)
             sql_1 = (f"INSERT INTO t_yujin_crossborder_prov_region_trade "
                    f"(crossborder_year, crossborder_year_month, prov_code, prov_name, city_code, city_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-                   f"('2023', '2023-01', '130000', '河北省', '{city_code}', '{city_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+                   f"('2023', '2023-01', '130000', '河北省', '{city_code}', '{city_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
             sql_arr_copy.append(sql_1)
 
         # 组装 SQL 语句
         sql = (f"INSERT INTO t_yujin_crossborder_prov_region_trade "
                f"(crossborder_year, crossborder_year_month, prov_code, prov_name, city_code, city_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-               f"('{year}', '{year_month}', '130000', '河北省', '{city_code}', '{city_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+               f"('{year}', '{year_month}', '130000', '河北省', '{city_code}', '{city_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
         sql_arr.append(sql)
 
     print(f"√ {year_month} prov_region_trade 成功生成 SQL 文件 size {len(sql_arr)} ")

hebei/gov_commodity_hebei_country.py

@@ -3,8 +3,8 @@ from pathlib import Path
 import pandas
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 EXCLUDE_REGIONS = ["亚洲", "非洲", "欧洲", "拉丁美洲", "北美洲", "大洋洲", "南极洲",
                    "东南亚国家联盟", "欧洲联盟", "亚太经济合作组织",
@@ -67,12 +67,12 @@ def process_folder(path):
             monthly_total = round(float(monthly_total) / 2, 4)
             sql = (f"INSERT INTO t_yujin_crossborder_prov_country_trade "
                    f"(crossborder_year, crossborder_year_month, prov_code, prov_name, country_code, country_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-                   f"('2023', '2023-01', '130000', '河北省', '{country_code}', '{country_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+                   f"('2023', '2023-01', '130000', '河北省', '{country_code}', '{country_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
             sql_arr_copy.append(sql)
         # 组装 SQL 语句
         sql = (f"INSERT INTO t_yujin_crossborder_prov_country_trade "
                f"(crossborder_year, crossborder_year_month, prov_code, prov_name, country_code, country_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-               f"('{year}', '{year_month}', '130000', '河北省', '{country_code}', '{country_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+               f"('{year}', '{year_month}', '130000', '河北省', '{country_code}', '{country_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
         sql_arr.append(sql)
 
     print(f"√ {year_month} prov_country_trade 成功生成 SQL 文件 size {len(sql_arr)} ")

hebei/gov_commodity_hebei_import_export.py

@@ -3,8 +3,8 @@ from pathlib import Path
 import pandas as pd
 import re
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 CUSTOM_COMMODITY_REPLACEMENTS = {
     '稻谷及大米': '稻谷、大米及大米粉',
@@ -85,12 +85,12 @@ def save_to_database(merged_df, year, month):
                 monthly_total = round(monthly_import + monthly_export, 4)
                 sql = (f"INSERT INTO t_yujin_crossborder_prov_commodity_trade "
                        f"(crossborder_year, crossborder_year_month, prov_code, prov_name, commodity_code, commodity_name, monthly_total, monthly_export, monthly_import, create_time) VALUES "
-                       f"('2023', '2023-01', '130000', '河北省', '{commodity_code}', '{commodity_name_fix}', {monthly_total}, {monthly_export}, {monthly_import}, now());")
+                       f"('2023', '2023-01', '130000', '河北省', '{commodity_code}', '{commodity_name_fix}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, now());")
                 sql_arr_copy.append(sql)
 
             sql = (f"INSERT INTO t_yujin_crossborder_prov_commodity_trade "
                    f"(crossborder_year, crossborder_year_month, prov_code, prov_name, commodity_code, commodity_name, monthly_total, monthly_export, monthly_import, create_time) VALUES "
-                   f"('{year}', '{year_month}', '130000', '河北省', '{commodity_code}', '{commodity_name_fix}', {monthly_total}, {monthly_export}, {monthly_import}, now());")
+                   f"('{year}', '{year_month}', '130000', '河北省', '{commodity_code}', '{commodity_name_fix}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, now());")
             sql_arr.append(sql)
 
             processed_commodities.add(commodity_name_fix)

jiangsu/crawl_gov_jiangsu_full.py

@@ -17,8 +17,7 @@ import gov_commodity_jiangsu_country
 import gov_commodity_jiangsu_city
 import gov_commodity_jiangsu_import_export
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
 
 # 显式指定 unrar 路径（根据实际情况修改）
 rarfile.UNRAR_EXECUTABLE = r"C:\Program Files\WinRAR\UnRAR.exe"

jiangsu/gov_commodity_jiangsu_city.py

@@ -1,11 +1,10 @@
 import time
 from pathlib import Path
 
-import pandas
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 city_code_map = {
     "南京市": "3201",
@@ -93,7 +92,7 @@ def process_folder(path):
         # 组装 SQL 语句
         sql = (f"INSERT INTO t_yujin_crossborder_prov_region_trade "
                f"(crossborder_year, crossborder_year_month, prov_code, prov_name, city_code, city_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-               f"('{year}', '{year_month}', '320000', '江苏省', '{city_code}', '{city_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+               f"('{year}', '{year_month}', '320000', '江苏省', '{city_code}', '{city_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
         sql_arr.append(sql)
 
     print(f"√ {year_month} prov_region_trade 成功生成 SQL 文件 size {len(sql_arr)} ")

jiangsu/gov_commodity_jiangsu_country.py

@@ -2,8 +2,8 @@ from pathlib import Path
 
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 # 排除地区名单
 EXCLUDE_REGIONS = ["亚洲", "非洲", "欧洲", "拉丁美洲", "北美洲", "大洋洲", "南极洲",
@@ -83,7 +83,7 @@ def process_folder(path):
                 f"(crossborder_year, crossborder_year_month, prov_code, prov_name, country_code, country_name, "
                 f"monthly_total, monthly_export, monthly_import, yoy_import_export, yoy_import, yoy_export, create_time) "
                 f"VALUES ('{year}', '{year_month}', '320000', '江苏省', '{country_code}', '{country_name}', "
-                f"'{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', "
+                f"{format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', "
                 f"'{yoy_export}', NOW());"
             )
             sql_arr.append(sql)

jiangsu/gov_commodity_jiangsu_import_export.py

@@ -3,8 +3,8 @@ from pathlib import Path
 
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 YEAR_PATTERN = re.compile(r"^\d{4}$")
 MONTH_PATTERN = re.compile(r"^(0[1-9]|1[0-2])$")
@@ -111,7 +111,7 @@ def save_to_database(import_df, export_df, total_df, year, month, all_records):
 
         sql = (f"INSERT INTO t_yujin_crossborder_prov_commodity_trade "
                f"(crossborder_year, crossborder_year_month, prov_code, prov_name, commodity_code, commodity_name, monthly_total, monthly_export, monthly_import, create_time, commodity_source) VALUES "
-               f"('{year}', '{year_month}', '320000', '江苏省', '{commodity_code}', '{category_name}', {monthly_total}, {monthly_export}, {monthly_import}, now(), 1);")
+               f"('{year}', '{year_month}', '320000', '江苏省', '{commodity_code}', '{category_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, now(), 1);")
         sql_arr.append(sql)
 
         processed_commodities.add(commodity_code)

utils/base_country_code.py

@@ -2,11 +2,26 @@ import os
 import re
 from pathlib import Path
 
+import pandas as pd
 from openpyxl import load_workbook
 
 YEAR_PATTERN = re.compile(r"^\d{4}$")
 MONTH_PATTERN = re.compile(r"^(0[1-9]|1[0-2])$")
 
+def format_sql_value(value):
+    """
+    将 Python 值转换为 SQL 可识别的格式：
+    - None -> NULL
+    - 数值 -> 原样输出
+    - 字符串 -> 加单引号
+    """
+    if pd.isna(value):
+        return 'NULL'
+    elif isinstance(value, (int, float)):
+        return str(value)
+    else:
+        return f"'{value}'"
+
 def find_sheet_by_keyword(file_path, keyword):
     """
     模糊查找包含关键字的 sheet 名称（支持 .xls 和 .xlsx）

zhejiang/crawl_gov_zhejiangi_full.py

@@ -17,7 +17,7 @@ from selenium.webdriver.support.ui import WebDriverWait
 import gov_commodity_zhejiang_city
 import gov_commodity_zhejiang_country
 import gov_commodity_zhejiang_import_export
-from com.zf.crawl import base_country_code, base_mysql
+from utils import base_country_code, base_mysql
 
 download_dir = base_country_code.download_dir
 Path(download_dir).mkdir(parents=True, exist_ok=True)

zhejiang/gov_commodity_zhejiang_city.py

@@ -1,11 +1,10 @@
 import time
 from pathlib import Path
 
-import pandas
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 city_code_map = {
     "杭州地区": "330100",
@@ -135,7 +134,7 @@ def process_folder(path):
         # 组装 SQL 语句
         sql = (f"INSERT INTO t_yujin_crossborder_prov_region_trade "
                f"(crossborder_year, crossborder_year_month, prov_code, prov_name, city_code, city_name, monthly_total, monthly_export, monthly_import,yoy_import_export, yoy_import, yoy_export, create_time) VALUES "
-               f"('{year}', '{year_month}', '330000', '浙江省', '{city_code}', '{city_name}', '{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
+               f"('{year}', '{year_month}', '330000', '浙江省', '{city_code}', '{city_name}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', '{yoy_export}', now());\n")
         sql_arr.append(sql)
 
     print(f"√ {year_month} prov_region_trade 成功生成 SQL 文件 size {len(sql_arr)} ")

zhejiang/gov_commodity_zhejiang_country.py

@@ -2,8 +2,8 @@ from pathlib import Path
 
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 # 排除地区名单
 EXCLUDE_REGIONS = ["亚洲", "非洲", "欧洲", "拉丁美洲", "北美洲", "大洋洲", "南极洲",
@@ -138,7 +138,7 @@ def process_folder(path):
             f"(crossborder_year, crossborder_year_month, prov_code, prov_name, country_code, country_name, "
             f"monthly_total, monthly_export, monthly_import, yoy_import_export, yoy_import, yoy_export, create_time) "
             f"VALUES ('{year}', '{year_month}', '330000', '浙江省', '{country_code}', '{country_name}', "
-            f"'{monthly_total}', '{monthly_export}', '{monthly_import}', '{yoy_import_export}', '{yoy_import}', "
+            f"{format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, '{yoy_import_export}', '{yoy_import}', "
             f"'{yoy_export}', NOW());"
         )
         sql_arr.append(sql)

zhejiang/gov_commodity_zhejiang_import_export.py

@@ -3,8 +3,8 @@ from pathlib import Path
 import re
 import pandas as pd
 
-from com.zf.crawl import base_country_code
-from com.zf.crawl import base_mysql
+from utils import base_country_code, base_mysql
+from utils.base_country_code import format_sql_value
 
 CUSTOM_COMMODITY_REPLACEMENTS = {
     '稻谷及大米': '稻谷、大米及大米粉',
@@ -182,7 +182,7 @@ def save_to_database(merged_df, year, month):
 
             sql = (f"INSERT INTO t_yujin_crossborder_prov_commodity_trade "
                    f"(crossborder_year, crossborder_year_month, prov_code, prov_name, commodity_code, commodity_name, monthly_total, monthly_export, monthly_import, create_time) VALUES "
-                   f"('{year}', '{year_month}', '330000', '浙江省', '{commodity_code}', '{commodity_name_fix}', {monthly_total}, {monthly_export}, {monthly_import}, now());")
+                   f"('{year}', '{year_month}', '330000', '浙江省', '{commodity_code}', '{commodity_name_fix}', {format_sql_value(monthly_total)}, {format_sql_value(monthly_export)}, {format_sql_value(monthly_import)}, now());")
             sql_arr.append(sql)
 
             processed_commodities.add(commodity_name_fix)