mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-03-02 12:10:46 +08:00
i18n: translate all Chinese comments, docstrings, and logger messages to English
Comprehensive translation of Chinese text to English across the entire codebase: - api/: FastAPI server documentation and logger messages - cache/: Cache abstraction layer comments and docstrings - database/: Database models and MongoDB store documentation - media_platform/: All platform crawlers (Bilibili, Douyin, Kuaishou, Tieba, Weibo, Xiaohongshu, Zhihu) - model/: Data model documentation - proxy/: Proxy pool and provider documentation - store/: Data storage layer comments - tools/: Utility functions and browser automation - test/: Test file documentation Preserved: Chinese disclaimer header (lines 10-18) for legal compliance 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
程序员阿江(Relakkes)
@@ -18,32 +18,32 @@
|
||||
|
||||
# @Author : persist-1<persist1@126.com>
|
||||
# @Time : 2025/9/8 00:02
|
||||
# @Desc : 用于将orm映射模型（database/models.py）与两种数据库实际结构进行对比，并进行更新操作（连接数据库->结构比对->差异报告->交互式同步）
|
||||
# @Tips : 该脚本需要安装依赖'pymysql==1.1.0'
|
||||
# @Desc : Used to compare ORM mapping model (database/models.py) with actual database structure and perform update operations (connect database -> structure comparison -> difference report -> interactive synchronization)
|
||||
# @Tips : This script requires dependency 'pymysql==1.1.0'
|
||||
|
||||
import os
|
||||
import sys
|
||||
from sqlalchemy import create_engine, inspect as sqlalchemy_inspect
|
||||
from sqlalchemy.schema import MetaData
|
||||
|
||||
# 将项目根目录添加到 sys.path
|
||||
# Add project root directory to sys.path
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from config.db_config import mysql_db_config, sqlite_db_config
|
||||
from database.models import Base
|
||||
|
||||
def get_mysql_engine():
|
||||
"""创建并返回一个MySQL数据库引擎"""
|
||||
"""Create and return a MySQL database engine"""
|
||||
conn_str = f"mysql+pymysql://{mysql_db_config['user']}:{mysql_db_config['password']}@{mysql_db_config['host']}:{mysql_db_config['port']}/{mysql_db_config['db_name']}"
|
||||
return create_engine(conn_str)
|
||||
|
||||
def get_sqlite_engine():
|
||||
"""创建并返回一个SQLite数据库引擎"""
|
||||
"""Create and return a SQLite database engine"""
|
||||
conn_str = f"sqlite:///{sqlite_db_config['db_path']}"
|
||||
return create_engine(conn_str)
|
||||
|
||||
def get_db_schema(engine):
|
||||
"""获取数据库的当前表结构"""
|
||||
"""Get current table structure of the database"""
|
||||
inspector = sqlalchemy_inspect(engine)
|
||||
schema = {}
|
||||
for table_name in inspector.get_table_names():
|
||||
@@ -54,7 +54,7 @@ def get_db_schema(engine):
|
||||
return schema
|
||||
|
||||
def get_orm_schema():
|
||||
"""获取ORM模型的表结构"""
|
||||
"""Get table structure of ORM model"""
|
||||
schema = {}
|
||||
for table_name, table in Base.metadata.tables.items():
|
||||
columns = {}
|
||||
@@ -64,7 +64,7 @@ def get_orm_schema():
|
||||
return schema
|
||||
|
||||
def compare_schemas(db_schema, orm_schema):
|
||||
"""比较数据库结构和ORM模型结构，返回差异"""
|
||||
"""Compare database structure with ORM model structure and return differences"""
|
||||
db_tables = set(db_schema.keys())
|
||||
orm_tables = set(orm_schema.keys())
|
||||
|
||||
@@ -99,42 +99,42 @@ def compare_schemas(db_schema, orm_schema):
|
||||
}
|
||||
|
||||
def print_diff(db_name, diff):
|
||||
"""打印差异报告"""
|
||||
print(f"--- {db_name} 数据库结构差异报告 ---")
|
||||
"""Print difference report"""
|
||||
print(f"--- {db_name} Database Structure Difference Report ---")
|
||||
if not any(diff.values()):
|
||||
print("数据库结构与ORM模型一致，无需同步。")
|
||||
print("Database structure matches ORM model, no synchronization needed.")
|
||||
return
|
||||
|
||||
if diff.get("added_tables"):
|
||||
print("\n[+] 新增的表:")
|
||||
print("\n[+] Added tables:")
|
||||
for table in diff["added_tables"]:
|
||||
print(f" - {table}")
|
||||
|
||||
if diff.get("deleted_tables"):
|
||||
print("\n[-] 删除的表:")
|
||||
print("\n[-] Deleted tables:")
|
||||
for table in diff["deleted_tables"]:
|
||||
print(f" - {table}")
|
||||
|
||||
if diff.get("changed_tables"):
|
||||
print("\n[*] 变动的表:")
|
||||
print("\n[*] Changed tables:")
|
||||
for table, changes in diff["changed_tables"].items():
|
||||
print(f" - {table}:")
|
||||
if changes.get("added"):
|
||||
print(" [+] 新增字段:", ", ".join(changes["added"]))
|
||||
print(" [+] Added fields:", ", ".join(changes["added"]))
|
||||
if changes.get("deleted"):
|
||||
print(" [-] 删除字段:", ", ".join(changes["deleted"]))
|
||||
print(" [-] Deleted fields:", ", ".join(changes["deleted"]))
|
||||
if changes.get("modified"):
|
||||
print(" [*] 修改字段:")
|
||||
print(" [*] Modified fields:")
|
||||
for col, types in changes["modified"].items():
|
||||
print(f" - {col}: {types[0]} -> {types[1]}")
|
||||
print("--- 报告结束 ---")
|
||||
print("--- End of Report ---")
|
||||
|
||||
|
||||
def sync_database(engine, diff):
|
||||
"""将ORM模型同步到数据库"""
|
||||
"""Synchronize ORM model to database"""
|
||||
metadata = Base.metadata
|
||||
|
||||
# Alembic的上下文配置
|
||||
# Alembic context configuration
|
||||
from alembic.migration import MigrationContext
|
||||
from alembic.operations import Operations
|
||||
|
||||
@@ -142,109 +142,109 @@ def sync_database(engine, diff):
|
||||
ctx = MigrationContext.configure(conn)
|
||||
op = Operations(ctx)
|
||||
|
||||
# 处理删除的表
|
||||
# Handle deleted tables
|
||||
for table_name in diff['deleted_tables']:
|
||||
op.drop_table(table_name)
|
||||
print(f"已删除表: {table_name}")
|
||||
print(f"Deleted table: {table_name}")
|
||||
|
||||
# 处理新增的表
|
||||
# Handle added tables
|
||||
for table_name in diff['added_tables']:
|
||||
table = metadata.tables.get(table_name)
|
||||
if table is not None:
|
||||
table.create(engine)
|
||||
print(f"已创建表: {table_name}")
|
||||
print(f"Created table: {table_name}")
|
||||
|
||||
# 处理字段变更
|
||||
# Handle field changes
|
||||
for table_name, changes in diff['changed_tables'].items():
|
||||
# 删除字段
|
||||
# Delete fields
|
||||
for col_name in changes['deleted']:
|
||||
op.drop_column(table_name, col_name)
|
||||
print(f"在表 {table_name} 中已删除字段: {col_name}")
|
||||
# 新增字段
|
||||
print(f"Deleted field in table {table_name}: {col_name}")
|
||||
# Add fields
|
||||
for col_name in changes['added']:
|
||||
table = metadata.tables.get(table_name)
|
||||
column = table.columns.get(col_name)
|
||||
if column is not None:
|
||||
op.add_column(table_name, column)
|
||||
print(f"在表 {table_name} 中已新增字段: {col_name}")
|
||||
print(f"Added field in table {table_name}: {col_name}")
|
||||
|
||||
# 修改字段
|
||||
# Modify fields
|
||||
for col_name, types in changes['modified'].items():
|
||||
table = metadata.tables.get(table_name)
|
||||
if table is not None:
|
||||
column = table.columns.get(col_name)
|
||||
if column is not None:
|
||||
op.alter_column(table_name, col_name, type_=column.type)
|
||||
print(f"在表 {table_name} 中已修改字段: {col_name} (类型变为 {column.type})")
|
||||
print(f"Modified field in table {table_name}: {col_name} (type changed to {column.type})")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
"""Main function"""
|
||||
orm_schema = get_orm_schema()
|
||||
|
||||
# 处理 MySQL
|
||||
# Handle MySQL
|
||||
try:
|
||||
mysql_engine = get_mysql_engine()
|
||||
mysql_schema = get_db_schema(mysql_engine)
|
||||
mysql_diff = compare_schemas(mysql_schema, orm_schema)
|
||||
print_diff("MySQL", mysql_diff)
|
||||
if any(mysql_diff.values()):
|
||||
choice = input(">>> 需要人工确认：是否要将ORM模型同步到MySQL数据库? (y/N): ")
|
||||
choice = input(">>> Manual confirmation required: Synchronize ORM model to MySQL database? (y/N): ")
|
||||
if choice.lower() == 'y':
|
||||
sync_database(mysql_engine, mysql_diff)
|
||||
print("MySQL数据库同步完成。")
|
||||
print("MySQL database synchronization completed.")
|
||||
except Exception as e:
|
||||
print(f"处理MySQL时出错: {e}")
|
||||
print(f"Error processing MySQL: {e}")
|
||||
|
||||
|
||||
# 处理 SQLite
|
||||
# Handle SQLite
|
||||
try:
|
||||
sqlite_engine = get_sqlite_engine()
|
||||
sqlite_schema = get_db_schema(sqlite_engine)
|
||||
sqlite_diff = compare_schemas(sqlite_schema, orm_schema)
|
||||
print_diff("SQLite", sqlite_diff)
|
||||
if any(sqlite_diff.values()):
|
||||
choice = input(">>> 需要人工确认：是否要将ORM模型同步到SQLite数据库? (y/N): ")
|
||||
choice = input(">>> Manual confirmation required: Synchronize ORM model to SQLite database? (y/N): ")
|
||||
if choice.lower() == 'y':
|
||||
# 注意：SQLite不支持ALTER COLUMN来修改字段类型，这里简化处理
|
||||
print("警告：SQLite的字段修改支持有限，此脚本不会执行修改字段类型的操作。")
|
||||
# Note: SQLite does not support ALTER COLUMN to modify field types, simplified handling here
|
||||
print("Warning: SQLite has limited support for field modifications, this script will not execute field type modification operations.")
|
||||
sync_database(sqlite_engine, sqlite_diff)
|
||||
print("SQLite数据库同步完成。")
|
||||
print("SQLite database synchronization completed.")
|
||||
except Exception as e:
|
||||
print(f"处理SQLite时出错: {e}")
|
||||
print(f"Error processing SQLite: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
######################### Feedback example #########################
|
||||
# [*] 变动的表:
|
||||
# [*] Changed tables:
|
||||
# - kuaishou_video:
|
||||
# [*] 修改字段:
|
||||
# [*] Modified fields:
|
||||
# - user_id: TEXT -> VARCHAR(64)
|
||||
# - xhs_note_comment:
|
||||
# [*] 修改字段:
|
||||
# [*] Modified fields:
|
||||
# - comment_id: BIGINT -> VARCHAR(255)
|
||||
# - zhihu_content:
|
||||
# [*] 修改字段:
|
||||
# [*] Modified fields:
|
||||
# - created_time: BIGINT -> VARCHAR(32)
|
||||
# - content_id: BIGINT -> VARCHAR(64)
|
||||
# - zhihu_creator:
|
||||
# [*] 修改字段:
|
||||
# [*] Modified fields:
|
||||
# - user_id: INTEGER -> VARCHAR(64)
|
||||
# - tieba_note:
|
||||
# [*] 修改字段:
|
||||
# [*] Modified fields:
|
||||
# - publish_time: BIGINT -> VARCHAR(255)
|
||||
# - tieba_id: INTEGER -> VARCHAR(255)
|
||||
# - note_id: BIGINT -> VARCHAR(644)
|
||||
# --- 报告结束 ---
|
||||
# >>> 需要人工确认：是否要将ORM模型同步到MySQL数据库? (y/N): y
|
||||
# 在表 kuaishou_video 中已修改字段: user_id (类型变为 VARCHAR(64))
|
||||
# 在表 xhs_note_comment 中已修改字段: comment_id (类型变为 VARCHAR(255))
|
||||
# 在表 zhihu_content 中已修改字段: created_time (类型变为 VARCHAR(32))
|
||||
# 在表 zhihu_content 中已修改字段: content_id (类型变为 VARCHAR(64))
|
||||
# 在表 zhihu_creator 中已修改字段: user_id (类型变为 VARCHAR(64))
|
||||
# 在表 tieba_note 中已修改字段: publish_time (类型变为 VARCHAR(255))
|
||||
# 在表 tieba_note 中已修改字段: tieba_id (类型变为 VARCHAR(255))
|
||||
# 在表 tieba_note 中已修改字段: note_id (类型变为 VARCHAR(644))
|
||||
# MySQL数据库同步完成。
|
||||
# --- End of Report ---
|
||||
# >>> Manual confirmation required: Synchronize ORM model to MySQL database? (y/N): y
|
||||
# Modified field in table kuaishou_video: user_id (type changed to VARCHAR(64))
|
||||
# Modified field in table xhs_note_comment: comment_id (type changed to VARCHAR(255))
|
||||
# Modified field in table zhihu_content: created_time (type changed to VARCHAR(32))
|
||||
# Modified field in table zhihu_content: content_id (type changed to VARCHAR(64))
|
||||
# Modified field in table zhihu_creator: user_id (type changed to VARCHAR(64))
|
||||
# Modified field in table tieba_note: publish_time (type changed to VARCHAR(255))
|
||||
# Modified field in table tieba_note: tieba_id (type changed to VARCHAR(255))
|
||||
# Modified field in table tieba_note: note_id (type changed to VARCHAR(644))
|
||||
# MySQL database synchronization completed.
|
||||
|
||||
Reference in New Issue
Block a user