crawl remove sql file

crossborder/anhui/crawl_gov_anhui_full.py

@@ -17,10 +17,8 @@ from crossborder.anhui import gov_commodity_anhui_city, download_dir
 from crossborder.anhui import gov_commodity_anhui_country
 from crossborder.anhui import gov_commodity_anhui_import_export
 from crossborder.utils import base_country_code, base_mysql
-from crossborder.utils.base_country_code import get_last_month
-from crossborder.utils.base_mysql import get_commodity_trade_by_prov_year_month
 from crossborder.utils.dingtalk import send_dingtalk_message
-from crossborder.utils.log import  get_logger
+from crossborder.utils.log import get_logger
 
 log = get_logger(__name__)
 
@@ -344,12 +342,6 @@ def main():
                 minutes, seconds = divmod(duration, 60)
                 send_dingtalk_message(f'【安徽省海关】增量数据采集完成，{int(minutes)}分{seconds:.1f}秒')
 
-                res = get_commodity_trade_by_prov_year_month('安徽省', get_last_month())
-                if res is not None:
-                    send_dingtalk_message(f"【安徽省海关】 commodity_trade 查询到 {len(res)} 条记录，文件已生成")
-                else:
-                    send_dingtalk_message("【安徽省海关】 未查询到任何记录或发生错误")
-
     except Exception as e:
         send_dingtalk_message(f'【安徽省海关】发生错误：{e}')
 

crossborder/hebei/crawl_gov_hebei_full.py

@@ -19,7 +19,6 @@ from crossborder.hebei import gov_commodity_hebei_country
 from crossborder.hebei import gov_commodity_hebei_import_export
 from crossborder.utils import base_country_code, base_mysql
 from crossborder.utils.base_country_code import get_last_month
-from crossborder.utils.base_mysql import get_commodity_trade_by_prov_year_month
 from crossborder.utils.dingtalk import send_dingtalk_message
 from crossborder.utils.log import  get_logger
 
@@ -322,12 +321,6 @@ def main():
                 minutes, seconds = divmod(duration, 60)
                 send_dingtalk_message(f'【河北省海关】增量数据采集完成，耗时 {int(minutes)}分{seconds:.1f}秒')
 
-                res = get_commodity_trade_by_prov_year_month('河北省', get_last_month())
-                if res is not None:
-                    send_dingtalk_message(f"【河北省海关】 commodity_trade 查询到 {len(res)} 条记录，文件已生成")
-                else:
-                    send_dingtalk_message("【河北省海关】 未查询到任何记录或发生错误")
-
     except Exception as e:
         send_dingtalk_message(f"【河北省海关】发生错误：{e}")
 

crossborder/jiangsu/crawl_gov_jiangsu_full.py

@@ -24,7 +24,6 @@ from crossborder.jiangsu import gov_commodity_jiangsu_import_export
 
 from crossborder.utils import base_country_code, base_mysql
 from crossborder.utils.base_country_code import get_last_month
-from crossborder.utils.base_mysql import get_commodity_trade_by_prov_year_month
 from crossborder.utils.dingtalk import send_dingtalk_message
 from crossborder.utils.log import  get_logger
 
@@ -359,11 +358,6 @@ def main():
                 minutes, seconds = divmod(duration, 60)
                 send_dingtalk_message(f'【江苏省海关】增量数据采集完成，耗时 {int(minutes)}分{seconds:.1f}秒')
 
-                res = get_commodity_trade_by_prov_year_month('江苏省', get_last_month())
-                if res is not None:
-                    send_dingtalk_message(f"【江苏省海关】 commodity_trade 查询到 {len(res)} 条记录，文件已生成")
-                else:
-                    send_dingtalk_message("【江苏省海关】 未查询到任何记录或发生错误")
     except Exception as e:
         send_dingtalk_message(f"【江苏省海关】发生错误：{e}")
 

crossborder/utils/base_mysql.py

@@ -4,7 +4,7 @@ import pymysql
 from sqlalchemy import text, create_engine
 
 from crossborder.utils.crypto_utils import AESCryptor
-from crossborder.utils.log import  get_logger
+from crossborder.utils.log import get_logger
 
 log = get_logger(__name__)
 
@@ -336,65 +336,9 @@ def _update_shandong_new_yoy_origin(region_name):
         log.info(f"{region_name} 新数据更新数: {result.rowcount}")
         return result.rowcount
 
-def generate_sql_file(results, prov_name, year_month):
-    filename = f"export_{prov_name}_{year_month}.sql"
-
-    with open(filename, 'w', encoding='utf-8') as f:
-        for row in results:
-            # 过滤掉不需要的字段（如 id）
-            filtered_row = {k: v for k, v in row.items() if k != 'id'}
-
-            # 构建INSERT语句
-            columns = ', '.join([f"`{k}`" for k in filtered_row.keys()])
-            values = ', '.join([
-                f"'{v}'" if isinstance(v, (str,)) else str(v) if v is not None else 'NULL'
-                for v in filtered_row.values()
-            ])
-
-            sql = f"INSERT INTO `t_yujin_crossborder_prov_commodity_trade` ({columns}) VALUES ({values});\n"
-            f.write(sql)
-
-    log.info(f"成功生成SQL文件：{filename}")
-
-def get_commodity_trade_by_prov_year_month(prov_name, year_month):
-    connection = None
-    try:
-        # 连接数据库
-        db_config = DB_CONFIG.copy()
-        db_config['password'] = get_decrypted_password()
-        connection = pymysql.connect(**db_config)
-
-        # 使用 DictCursor 来获取字典形式的结果
-        with connection.cursor(pymysql.cursors.DictCursor) as cursor:
-            # 执行查询
-            sql = "SELECT * FROM t_yujin_crossborder_prov_commodity_trade WHERE prov_name = %s AND crossborder_year_month = %s"
-            cursor.execute(sql, (prov_name, year_month))
-
-            # 获取结果
-            results = cursor.fetchall()
-
-            # 生成SQL文件
-            if results:
-                generate_sql_file(results, prov_name, year_month)
-
-            return results
-
-    except Exception as e:
-        log.error(f"get_commodity_trade_by_prov_year_month error: {str(e)}")
-        return None
-    finally:
-        if connection:
-            connection.close()
-
 if __name__ == '__main__':
-    res = get_commodity_trade_by_prov_year_month('江苏省','2025-04')
-    if res is not None:
-        print(f"查询到 {len(res)} 条记录")
-    else:
-        print("未查询到任何记录或发生错误")
-
-    # commodity_code, commodity_name_fix = get_commodity_id('农产品')
-    # print(commodity_code, commodity_name_fix)
+    commodity_code, commodity_name_fix = get_commodity_id('农产品')
+    print(commodity_code, commodity_name_fix)
     # check_year, check_month = 2024, 4
     # count = get_code_exist(f'{check_year}-{check_month:02d}', "340000")
     # print(count)

crossborder/zhejiang/crawl_gov_zhejiang_full.py

@@ -17,7 +17,6 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
 from crossborder.utils.base_country_code import get_last_month
-from crossborder.utils.base_mysql import get_commodity_trade_by_prov_year_month
 from crossborder.utils.dingtalk import send_dingtalk_message
 from crossborder.zhejiang import download_dir
 from crossborder.zhejiang import gov_commodity_zhejiang_city
@@ -406,11 +405,6 @@ def main():
                 minutes, seconds = divmod(duration, 60)
                 send_dingtalk_message(f'【浙江省海关】增量数据采集完成，耗时 {int(minutes)}分{seconds:.1f}秒')
 
-                res = get_commodity_trade_by_prov_year_month('浙江省', get_last_month())
-                if res is not None:
-                    send_dingtalk_message(f"【浙江省海关】 commodity_trade 查询到 {len(res)} 条记录，文件已生成")
-                else:
-                    send_dingtalk_message("【浙江省海关】 未查询到任何记录或发生错误")
     except Exception as e:
         send_dingtalk_message(f"【浙江省海关】发生错误：{e}")