mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-03-04 21:20:47 +08:00
feat: 添加SQLite数据库支持核心文件\n\n- 新增 async_sqlite_db.py: SQLite异步数据库操作封装\n- 新增 schema/sqlite_tables.sql: SQLite数据库表结构定义\n- 新增 schema/sqlite_tables.db: SQLite数据库文件
This commit is contained in:
买定不离手
111
async_sqlite_db.py
Normal file
111
async_sqlite_db.py
Normal file
@@ -0,0 +1,111 @@
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Author : relakkes@gmail.com
|
||||
# @Time : 2024/4/6 14:21
|
||||
# @Desc : 异步SQLite的增删改查封装
|
||||
from typing import Any, Dict, List, Union
|
||||
|
||||
import aiosqlite
|
||||
|
||||
|
||||
class AsyncSqliteDB:
|
||||
def __init__(self, db_path: str) -> None:
|
||||
self.__db_path = db_path
|
||||
|
||||
async def query(self, sql: str, *args: Union[str, int]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
从给定的 SQL 中查询记录，返回的是一个列表
|
||||
:param sql: 查询的sql
|
||||
:param args: sql中传递动态参数列表
|
||||
:return:
|
||||
"""
|
||||
async with aiosqlite.connect(self.__db_path) as conn:
|
||||
conn.row_factory = aiosqlite.Row
|
||||
async with conn.execute(sql, args) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
return [dict(row) for row in rows] if rows else []
|
||||
|
||||
async def get_first(self, sql: str, *args: Union[str, int]) -> Union[Dict[str, Any], None]:
|
||||
"""
|
||||
从给定的 SQL 中查询记录，返回的是符合条件的第一个结果
|
||||
:param sql: 查询的sql
|
||||
:param args:sql中传递动态参数列表
|
||||
:return:
|
||||
"""
|
||||
async with aiosqlite.connect(self.__db_path) as conn:
|
||||
conn.row_factory = aiosqlite.Row
|
||||
async with conn.execute(sql, args) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
async def item_to_table(self, table_name: str, item: Dict[str, Any]) -> int:
|
||||
"""
|
||||
表中插入数据
|
||||
:param table_name: 表名
|
||||
:param item: 一条记录的字典信息
|
||||
:return:
|
||||
"""
|
||||
fields = list(item.keys())
|
||||
values = list(item.values())
|
||||
fieldstr = ','.join(fields)
|
||||
valstr = ','.join(['?'] * len(item))
|
||||
sql = f"INSERT INTO {table_name} ({fieldstr}) VALUES({valstr})"
|
||||
async with aiosqlite.connect(self.__db_path) as conn:
|
||||
async with conn.execute(sql, values) as cursor:
|
||||
await conn.commit()
|
||||
return cursor.lastrowid
|
||||
|
||||
async def update_table(self, table_name: str, updates: Dict[str, Any], field_where: str,
|
||||
value_where: Union[str, int, float]) -> int:
|
||||
"""
|
||||
更新指定表的记录
|
||||
:param table_name: 表名
|
||||
:param updates: 需要更新的字段和值的 key - value 映射
|
||||
:param field_where: update 语句 where 条件中的字段名
|
||||
:param value_where: update 语句 where 条件中的字段值
|
||||
:return:
|
||||
"""
|
||||
upsets = []
|
||||
values = []
|
||||
for k, v in updates.items():
|
||||
upsets.append(f'{k}=?')
|
||||
values.append(v)
|
||||
upsets_str = ','.join(upsets)
|
||||
values.append(value_where)
|
||||
sql = f'UPDATE {table_name} SET {upsets_str} WHERE {field_where}=?'
|
||||
async with aiosqlite.connect(self.__db_path) as conn:
|
||||
async with conn.execute(sql, values) as cursor:
|
||||
await conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
async def execute(self, sql: str, *args: Union[str, int]) -> int:
|
||||
"""
|
||||
需要更新、写入等操作的 excute 执行语句
|
||||
:param sql:
|
||||
:param args:
|
||||
:return:
|
||||
"""
|
||||
async with aiosqlite.connect(self.__db_path) as conn:
|
||||
async with conn.execute(sql, args) as cursor:
|
||||
await conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
async def executescript(self, sql_script: str) -> None:
|
||||
"""
|
||||
执行SQL脚本，用于初始化数据库表结构
|
||||
:param sql_script: SQL脚本内容
|
||||
:return:
|
||||
"""
|
||||
async with aiosqlite.connect(self.__db_path) as conn:
|
||||
await conn.executescript(sql_script)
|
||||
await conn.commit()
|
||||
BIN
schema/sqlite_tables.db
Normal file
BIN
schema/sqlite_tables.db
Normal file
Binary file not shown.
567
schema/sqlite_tables.sql
Normal file
567
schema/sqlite_tables.sql
Normal file
@@ -0,0 +1,567 @@
|
||||
-- SQLite版本的MediaCrawler数据库表结构
|
||||
-- 从MySQL tables.sql转换而来，适配SQLite语法
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for bilibili_video
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS bilibili_video;
|
||||
CREATE TABLE bilibili_video (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
video_id TEXT NOT NULL,
|
||||
video_type TEXT NOT NULL,
|
||||
title TEXT DEFAULT NULL,
|
||||
desc TEXT,
|
||||
create_time INTEGER NOT NULL,
|
||||
liked_count TEXT DEFAULT NULL,
|
||||
disliked_count TEXT DEFAULT NULL,
|
||||
video_play_count TEXT DEFAULT NULL,
|
||||
video_favorite_count TEXT DEFAULT NULL,
|
||||
video_share_count TEXT DEFAULT NULL,
|
||||
video_coin_count TEXT DEFAULT NULL,
|
||||
video_danmaku TEXT DEFAULT NULL,
|
||||
video_comment TEXT DEFAULT NULL,
|
||||
video_url TEXT DEFAULT NULL,
|
||||
video_cover_url TEXT DEFAULT NULL,
|
||||
source_keyword TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX idx_bilibili_vi_video_i_31c36e ON bilibili_video(video_id);
|
||||
CREATE INDEX idx_bilibili_vi_create__73e0ec ON bilibili_video(create_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for bilibili_video_comment
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS bilibili_video_comment;
|
||||
CREATE TABLE bilibili_video_comment (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
sex TEXT DEFAULT NULL,
|
||||
sign TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
comment_id TEXT NOT NULL,
|
||||
video_id TEXT NOT NULL,
|
||||
content TEXT,
|
||||
create_time INTEGER NOT NULL,
|
||||
sub_comment_count TEXT NOT NULL,
|
||||
parent_comment_id TEXT DEFAULT NULL,
|
||||
like_count TEXT NOT NULL DEFAULT '0'
|
||||
);
|
||||
|
||||
CREATE INDEX idx_bilibili_vi_comment_41c34e ON bilibili_video_comment(comment_id);
|
||||
CREATE INDEX idx_bilibili_vi_video_i_f22873 ON bilibili_video_comment(video_id);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for bilibili_up_info
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS bilibili_up_info;
|
||||
CREATE TABLE bilibili_up_info (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
sex TEXT DEFAULT NULL,
|
||||
sign TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
total_fans INTEGER DEFAULT NULL,
|
||||
total_liked INTEGER DEFAULT NULL,
|
||||
user_rank INTEGER DEFAULT NULL,
|
||||
is_official INTEGER DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_bilibili_vi_user_123456 ON bilibili_up_info(user_id);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for bilibili_contact_info
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS bilibili_contact_info;
|
||||
CREATE TABLE bilibili_contact_info (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
up_id TEXT DEFAULT NULL,
|
||||
fan_id TEXT DEFAULT NULL,
|
||||
up_name TEXT DEFAULT NULL,
|
||||
fan_name TEXT DEFAULT NULL,
|
||||
up_sign TEXT DEFAULT NULL,
|
||||
fan_sign TEXT DEFAULT NULL,
|
||||
up_avatar TEXT DEFAULT NULL,
|
||||
fan_avatar TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_bilibili_contact_info_up_id ON bilibili_contact_info(up_id);
|
||||
CREATE INDEX idx_bilibili_contact_info_fan_id ON bilibili_contact_info(fan_id);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for bilibili_up_dynamic
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS bilibili_up_dynamic;
|
||||
CREATE TABLE bilibili_up_dynamic (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
dynamic_id TEXT DEFAULT NULL,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
user_name TEXT DEFAULT NULL,
|
||||
text TEXT DEFAULT NULL,
|
||||
type TEXT DEFAULT NULL,
|
||||
pub_ts INTEGER DEFAULT NULL,
|
||||
total_comments INTEGER DEFAULT NULL,
|
||||
total_forwards INTEGER DEFAULT NULL,
|
||||
total_liked INTEGER DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_bilibili_up_dynamic_dynamic_id ON bilibili_up_dynamic(dynamic_id);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for douyin_aweme
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS douyin_aweme;
|
||||
CREATE TABLE douyin_aweme (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
sec_uid TEXT DEFAULT NULL,
|
||||
short_user_id TEXT DEFAULT NULL,
|
||||
user_unique_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
user_signature TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
aweme_id TEXT NOT NULL,
|
||||
aweme_type TEXT NOT NULL,
|
||||
title TEXT DEFAULT NULL,
|
||||
desc TEXT,
|
||||
create_time INTEGER NOT NULL,
|
||||
liked_count TEXT DEFAULT NULL,
|
||||
comment_count TEXT DEFAULT NULL,
|
||||
share_count TEXT DEFAULT NULL,
|
||||
collected_count TEXT DEFAULT NULL,
|
||||
aweme_url TEXT DEFAULT NULL,
|
||||
cover_url TEXT DEFAULT NULL,
|
||||
video_download_url TEXT DEFAULT NULL,
|
||||
source_keyword TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX idx_douyin_awem_aweme_i_6f7bc6 ON douyin_aweme(aweme_id);
|
||||
CREATE INDEX idx_douyin_awem_create__299dfe ON douyin_aweme(create_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for douyin_aweme_comment
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS douyin_aweme_comment;
|
||||
CREATE TABLE douyin_aweme_comment (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
sec_uid TEXT DEFAULT NULL,
|
||||
short_user_id TEXT DEFAULT NULL,
|
||||
user_unique_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
user_signature TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
comment_id TEXT NOT NULL,
|
||||
aweme_id TEXT NOT NULL,
|
||||
content TEXT,
|
||||
create_time INTEGER NOT NULL,
|
||||
sub_comment_count TEXT NOT NULL,
|
||||
parent_comment_id TEXT DEFAULT NULL,
|
||||
like_count TEXT NOT NULL DEFAULT '0',
|
||||
pictures TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX idx_douyin_awem_comment_fcd7e4 ON douyin_aweme_comment(comment_id);
|
||||
CREATE INDEX idx_douyin_awem_aweme_i_c50049 ON douyin_aweme_comment(aweme_id);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for dy_creator
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS dy_creator;
|
||||
CREATE TABLE dy_creator (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
desc TEXT,
|
||||
gender TEXT DEFAULT NULL,
|
||||
follows TEXT DEFAULT NULL,
|
||||
fans TEXT DEFAULT NULL,
|
||||
interaction TEXT DEFAULT NULL,
|
||||
videos_count TEXT DEFAULT NULL
|
||||
);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for kuaishou_video
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS kuaishou_video;
|
||||
CREATE TABLE kuaishou_video (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
video_id TEXT NOT NULL,
|
||||
video_type TEXT NOT NULL,
|
||||
title TEXT DEFAULT NULL,
|
||||
desc TEXT,
|
||||
create_time INTEGER NOT NULL,
|
||||
liked_count TEXT DEFAULT NULL,
|
||||
viewd_count TEXT DEFAULT NULL,
|
||||
video_url TEXT DEFAULT NULL,
|
||||
video_cover_url TEXT DEFAULT NULL,
|
||||
video_play_url TEXT DEFAULT NULL,
|
||||
source_keyword TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX idx_kuaishou_vi_video_i_c5c6a6 ON kuaishou_video(video_id);
|
||||
CREATE INDEX idx_kuaishou_vi_create__a10dee ON kuaishou_video(create_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for kuaishou_video_comment
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS kuaishou_video_comment;
|
||||
CREATE TABLE kuaishou_video_comment (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
comment_id TEXT NOT NULL,
|
||||
video_id TEXT NOT NULL,
|
||||
content TEXT,
|
||||
create_time INTEGER NOT NULL,
|
||||
sub_comment_count TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_kuaishou_vi_comment_ed48fa ON kuaishou_video_comment(comment_id);
|
||||
CREATE INDEX idx_kuaishou_vi_video_i_e50914 ON kuaishou_video_comment(video_id);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for weibo_note
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS weibo_note;
|
||||
CREATE TABLE weibo_note (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
gender TEXT DEFAULT NULL,
|
||||
profile_url TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT '',
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
note_id TEXT NOT NULL,
|
||||
content TEXT,
|
||||
create_time INTEGER NOT NULL,
|
||||
create_date_time TEXT NOT NULL,
|
||||
liked_count TEXT DEFAULT NULL,
|
||||
comments_count TEXT DEFAULT NULL,
|
||||
shared_count TEXT DEFAULT NULL,
|
||||
note_url TEXT DEFAULT NULL,
|
||||
source_keyword TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX idx_weibo_note_note_id_f95b1a ON weibo_note(note_id);
|
||||
CREATE INDEX idx_weibo_note_create__692709 ON weibo_note(create_time);
|
||||
CREATE INDEX idx_weibo_note_create__d05ed2 ON weibo_note(create_date_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for weibo_note_comment
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS weibo_note_comment;
|
||||
CREATE TABLE weibo_note_comment (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT DEFAULT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
gender TEXT DEFAULT NULL,
|
||||
profile_url TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT '',
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
comment_id TEXT NOT NULL,
|
||||
note_id TEXT NOT NULL,
|
||||
content TEXT,
|
||||
create_time INTEGER NOT NULL,
|
||||
create_date_time TEXT NOT NULL,
|
||||
comment_like_count TEXT NOT NULL,
|
||||
sub_comment_count TEXT NOT NULL,
|
||||
parent_comment_id TEXT DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_weibo_note__comment_c7611c ON weibo_note_comment(comment_id);
|
||||
CREATE INDEX idx_weibo_note__note_id_24f108 ON weibo_note_comment(note_id);
|
||||
CREATE INDEX idx_weibo_note__create__667fe3 ON weibo_note_comment(create_date_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for weibo_creator
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS weibo_creator;
|
||||
CREATE TABLE weibo_creator (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
desc TEXT,
|
||||
gender TEXT DEFAULT NULL,
|
||||
follows TEXT DEFAULT NULL,
|
||||
fans TEXT DEFAULT NULL,
|
||||
tag_list TEXT
|
||||
);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for xhs_creator
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS xhs_creator;
|
||||
CREATE TABLE xhs_creator (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
desc TEXT,
|
||||
gender TEXT DEFAULT NULL,
|
||||
follows TEXT DEFAULT NULL,
|
||||
fans TEXT DEFAULT NULL,
|
||||
interaction TEXT DEFAULT NULL,
|
||||
tag_list TEXT
|
||||
);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for xhs_note
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS xhs_note;
|
||||
CREATE TABLE xhs_note (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
note_id TEXT NOT NULL,
|
||||
type TEXT DEFAULT NULL,
|
||||
title TEXT DEFAULT NULL,
|
||||
desc TEXT,
|
||||
video_url TEXT,
|
||||
time INTEGER NOT NULL,
|
||||
last_update_time INTEGER NOT NULL,
|
||||
liked_count TEXT DEFAULT NULL,
|
||||
collected_count TEXT DEFAULT NULL,
|
||||
comment_count TEXT DEFAULT NULL,
|
||||
share_count TEXT DEFAULT NULL,
|
||||
image_list TEXT,
|
||||
tag_list TEXT,
|
||||
note_url TEXT DEFAULT NULL,
|
||||
source_keyword TEXT DEFAULT '',
|
||||
xsec_token TEXT DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_xhs_note_note_id_209457 ON xhs_note(note_id);
|
||||
CREATE INDEX idx_xhs_note_time_eaa910 ON xhs_note(time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for xhs_note_comment
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS xhs_note_comment;
|
||||
CREATE TABLE xhs_note_comment (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
comment_id TEXT NOT NULL,
|
||||
create_time INTEGER NOT NULL,
|
||||
note_id TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
sub_comment_count INTEGER NOT NULL,
|
||||
pictures TEXT DEFAULT NULL,
|
||||
parent_comment_id TEXT DEFAULT NULL,
|
||||
like_count TEXT DEFAULT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_xhs_note_co_comment_8e8349 ON xhs_note_comment(comment_id);
|
||||
CREATE INDEX idx_xhs_note_co_create__204f8d ON xhs_note_comment(create_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for tieba_note
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS tieba_note;
|
||||
CREATE TABLE tieba_note (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
note_id TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
desc TEXT,
|
||||
note_url TEXT NOT NULL,
|
||||
publish_time TEXT NOT NULL,
|
||||
user_link TEXT DEFAULT '',
|
||||
user_nickname TEXT DEFAULT '',
|
||||
user_avatar TEXT DEFAULT '',
|
||||
tieba_id TEXT DEFAULT '',
|
||||
tieba_name TEXT NOT NULL,
|
||||
tieba_link TEXT NOT NULL,
|
||||
total_replay_num INTEGER DEFAULT 0,
|
||||
total_replay_page INTEGER DEFAULT 0,
|
||||
ip_location TEXT DEFAULT '',
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
source_keyword TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE INDEX idx_tieba_note_note_id ON tieba_note(note_id);
|
||||
CREATE INDEX idx_tieba_note_publish_time ON tieba_note(publish_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for tieba_comment
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS tieba_comment;
|
||||
CREATE TABLE tieba_comment (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
comment_id TEXT NOT NULL,
|
||||
parent_comment_id TEXT DEFAULT '',
|
||||
content TEXT NOT NULL,
|
||||
user_link TEXT DEFAULT '',
|
||||
user_nickname TEXT DEFAULT '',
|
||||
user_avatar TEXT DEFAULT '',
|
||||
tieba_id TEXT DEFAULT '',
|
||||
tieba_name TEXT NOT NULL,
|
||||
tieba_link TEXT NOT NULL,
|
||||
publish_time TEXT DEFAULT '',
|
||||
ip_location TEXT DEFAULT '',
|
||||
sub_comment_count INTEGER DEFAULT 0,
|
||||
note_id TEXT NOT NULL,
|
||||
note_url TEXT NOT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_tieba_comment_comment_id ON tieba_comment(comment_id);
|
||||
CREATE INDEX idx_tieba_comment_note_id ON tieba_comment(note_id);
|
||||
CREATE INDEX idx_tieba_comment_publish_time ON tieba_comment(publish_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for tieba_creator
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS tieba_creator;
|
||||
CREATE TABLE tieba_creator (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
user_name TEXT NOT NULL,
|
||||
nickname TEXT DEFAULT NULL,
|
||||
avatar TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL,
|
||||
gender TEXT DEFAULT NULL,
|
||||
follows TEXT DEFAULT NULL,
|
||||
fans TEXT DEFAULT NULL,
|
||||
registration_duration TEXT DEFAULT NULL
|
||||
);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for zhihu_content
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS zhihu_content;
|
||||
CREATE TABLE zhihu_content (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
content_id TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
content_text TEXT,
|
||||
content_url TEXT NOT NULL,
|
||||
question_id TEXT DEFAULT NULL,
|
||||
title TEXT NOT NULL,
|
||||
desc TEXT,
|
||||
created_time TEXT NOT NULL,
|
||||
updated_time TEXT NOT NULL,
|
||||
voteup_count INTEGER NOT NULL DEFAULT 0,
|
||||
comment_count INTEGER NOT NULL DEFAULT 0,
|
||||
source_keyword TEXT DEFAULT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
user_link TEXT NOT NULL,
|
||||
user_nickname TEXT NOT NULL,
|
||||
user_avatar TEXT NOT NULL,
|
||||
user_url_token TEXT NOT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_zhihu_content_content_id ON zhihu_content(content_id);
|
||||
CREATE INDEX idx_zhihu_content_created_time ON zhihu_content(created_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for zhihu_comment
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS zhihu_comment;
|
||||
CREATE TABLE zhihu_comment (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
comment_id TEXT NOT NULL,
|
||||
parent_comment_id TEXT DEFAULT NULL,
|
||||
content TEXT NOT NULL,
|
||||
publish_time TEXT NOT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
sub_comment_count INTEGER NOT NULL DEFAULT 0,
|
||||
like_count INTEGER NOT NULL DEFAULT 0,
|
||||
dislike_count INTEGER NOT NULL DEFAULT 0,
|
||||
content_id TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
user_link TEXT NOT NULL,
|
||||
user_nickname TEXT NOT NULL,
|
||||
user_avatar TEXT NOT NULL,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX idx_zhihu_comment_comment_id ON zhihu_comment(comment_id);
|
||||
CREATE INDEX idx_zhihu_comment_content_id ON zhihu_comment(content_id);
|
||||
CREATE INDEX idx_zhihu_comment_publish_time ON zhihu_comment(publish_time);
|
||||
|
||||
-- ----------------------------
|
||||
-- Table structure for zhihu_creator
|
||||
-- ----------------------------
|
||||
DROP TABLE IF EXISTS zhihu_creator;
|
||||
CREATE TABLE zhihu_creator (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL UNIQUE,
|
||||
user_link TEXT NOT NULL,
|
||||
user_nickname TEXT NOT NULL,
|
||||
user_avatar TEXT NOT NULL,
|
||||
url_token TEXT NOT NULL,
|
||||
gender TEXT DEFAULT NULL,
|
||||
ip_location TEXT DEFAULT NULL,
|
||||
follows INTEGER NOT NULL DEFAULT 0,
|
||||
fans INTEGER NOT NULL DEFAULT 0,
|
||||
anwser_count INTEGER NOT NULL DEFAULT 0,
|
||||
video_count INTEGER NOT NULL DEFAULT 0,
|
||||
question_count INTEGER NOT NULL DEFAULT 0,
|
||||
article_count INTEGER NOT NULL DEFAULT 0,
|
||||
column_count INTEGER NOT NULL DEFAULT 0,
|
||||
get_voteup_count INTEGER NOT NULL DEFAULT 0,
|
||||
add_ts INTEGER NOT NULL,
|
||||
last_modify_ts INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX idx_zhihu_creator_user_id ON zhihu_creator(user_id);
|
||||
Reference in New Issue
Block a user