diff --git a/async_sqlite_db.py b/async_sqlite_db.py new file mode 100644 index 0000000..d9409bd --- /dev/null +++ b/async_sqlite_db.py @@ -0,0 +1,111 @@ +# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则: +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + + +# -*- coding: utf-8 -*- +# @Author : relakkes@gmail.com +# @Time : 2024/4/6 14:21 +# @Desc : 异步SQLite的增删改查封装 +from typing import Any, Dict, List, Union + +import aiosqlite + + +class AsyncSqliteDB: + def __init__(self, db_path: str) -> None: + self.__db_path = db_path + + async def query(self, sql: str, *args: Union[str, int]) -> List[Dict[str, Any]]: + """ + 从给定的 SQL 中查询记录,返回的是一个列表 + :param sql: 查询的sql + :param args: sql中传递动态参数列表 + :return: + """ + async with aiosqlite.connect(self.__db_path) as conn: + conn.row_factory = aiosqlite.Row + async with conn.execute(sql, args) as cursor: + rows = await cursor.fetchall() + return [dict(row) for row in rows] if rows else [] + + async def get_first(self, sql: str, *args: Union[str, int]) -> Union[Dict[str, Any], None]: + """ + 从给定的 SQL 中查询记录,返回的是符合条件的第一个结果 + :param sql: 查询的sql + :param args:sql中传递动态参数列表 + :return: + """ + async with aiosqlite.connect(self.__db_path) as conn: + conn.row_factory = aiosqlite.Row + async with conn.execute(sql, args) as cursor: + row = await cursor.fetchone() + return dict(row) if row else None + + async def item_to_table(self, table_name: str, item: Dict[str, Any]) -> int: + """ + 表中插入数据 + :param table_name: 表名 + :param item: 一条记录的字典信息 + :return: + """ + fields = list(item.keys()) + values = list(item.values()) + fieldstr = ','.join(fields) + valstr = ','.join(['?'] * len(item)) + sql = f"INSERT INTO {table_name} ({fieldstr}) VALUES({valstr})" + async with aiosqlite.connect(self.__db_path) as conn: + async with conn.execute(sql, values) as cursor: + await conn.commit() + return cursor.lastrowid + + async def update_table(self, table_name: str, updates: Dict[str, Any], field_where: str, + value_where: Union[str, int, float]) -> int: + """ + 更新指定表的记录 + :param table_name: 表名 + :param updates: 需要更新的字段和值的 key - value 映射 + :param field_where: update 语句 where 条件中的字段名 + :param value_where: update 语句 where 条件中的字段值 + :return: + """ + upsets = [] + values = [] + for k, v in updates.items(): + upsets.append(f'{k}=?') + values.append(v) + upsets_str = ','.join(upsets) + values.append(value_where) + sql = f'UPDATE {table_name} SET {upsets_str} WHERE {field_where}=?' + async with aiosqlite.connect(self.__db_path) as conn: + async with conn.execute(sql, values) as cursor: + await conn.commit() + return cursor.rowcount + + async def execute(self, sql: str, *args: Union[str, int]) -> int: + """ + 需要更新、写入等操作的 excute 执行语句 + :param sql: + :param args: + :return: + """ + async with aiosqlite.connect(self.__db_path) as conn: + async with conn.execute(sql, args) as cursor: + await conn.commit() + return cursor.rowcount + + async def executescript(self, sql_script: str) -> None: + """ + 执行SQL脚本,用于初始化数据库表结构 + :param sql_script: SQL脚本内容 + :return: + """ + async with aiosqlite.connect(self.__db_path) as conn: + await conn.executescript(sql_script) + await conn.commit() \ No newline at end of file diff --git a/schema/sqlite_tables.db b/schema/sqlite_tables.db new file mode 100644 index 0000000..193ec5e Binary files /dev/null and b/schema/sqlite_tables.db differ diff --git a/schema/sqlite_tables.sql b/schema/sqlite_tables.sql new file mode 100644 index 0000000..dc5adb7 --- /dev/null +++ b/schema/sqlite_tables.sql @@ -0,0 +1,567 @@ +-- SQLite版本的MediaCrawler数据库表结构 +-- 从MySQL tables.sql转换而来,适配SQLite语法 + +-- ---------------------------- +-- Table structure for bilibili_video +-- ---------------------------- +DROP TABLE IF EXISTS bilibili_video; +CREATE TABLE bilibili_video ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + video_id TEXT NOT NULL, + video_type TEXT NOT NULL, + title TEXT DEFAULT NULL, + desc TEXT, + create_time INTEGER NOT NULL, + liked_count TEXT DEFAULT NULL, + disliked_count TEXT DEFAULT NULL, + video_play_count TEXT DEFAULT NULL, + video_favorite_count TEXT DEFAULT NULL, + video_share_count TEXT DEFAULT NULL, + video_coin_count TEXT DEFAULT NULL, + video_danmaku TEXT DEFAULT NULL, + video_comment TEXT DEFAULT NULL, + video_url TEXT DEFAULT NULL, + video_cover_url TEXT DEFAULT NULL, + source_keyword TEXT DEFAULT '' +); + +CREATE INDEX idx_bilibili_vi_video_i_31c36e ON bilibili_video(video_id); +CREATE INDEX idx_bilibili_vi_create__73e0ec ON bilibili_video(create_time); + +-- ---------------------------- +-- Table structure for bilibili_video_comment +-- ---------------------------- +DROP TABLE IF EXISTS bilibili_video_comment; +CREATE TABLE bilibili_video_comment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + sex TEXT DEFAULT NULL, + sign TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + comment_id TEXT NOT NULL, + video_id TEXT NOT NULL, + content TEXT, + create_time INTEGER NOT NULL, + sub_comment_count TEXT NOT NULL, + parent_comment_id TEXT DEFAULT NULL, + like_count TEXT NOT NULL DEFAULT '0' +); + +CREATE INDEX idx_bilibili_vi_comment_41c34e ON bilibili_video_comment(comment_id); +CREATE INDEX idx_bilibili_vi_video_i_f22873 ON bilibili_video_comment(video_id); + +-- ---------------------------- +-- Table structure for bilibili_up_info +-- ---------------------------- +DROP TABLE IF EXISTS bilibili_up_info; +CREATE TABLE bilibili_up_info ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + sex TEXT DEFAULT NULL, + sign TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + total_fans INTEGER DEFAULT NULL, + total_liked INTEGER DEFAULT NULL, + user_rank INTEGER DEFAULT NULL, + is_official INTEGER DEFAULT NULL +); + +CREATE INDEX idx_bilibili_vi_user_123456 ON bilibili_up_info(user_id); + +-- ---------------------------- +-- Table structure for bilibili_contact_info +-- ---------------------------- +DROP TABLE IF EXISTS bilibili_contact_info; +CREATE TABLE bilibili_contact_info ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + up_id TEXT DEFAULT NULL, + fan_id TEXT DEFAULT NULL, + up_name TEXT DEFAULT NULL, + fan_name TEXT DEFAULT NULL, + up_sign TEXT DEFAULT NULL, + fan_sign TEXT DEFAULT NULL, + up_avatar TEXT DEFAULT NULL, + fan_avatar TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL +); + +CREATE INDEX idx_bilibili_contact_info_up_id ON bilibili_contact_info(up_id); +CREATE INDEX idx_bilibili_contact_info_fan_id ON bilibili_contact_info(fan_id); + +-- ---------------------------- +-- Table structure for bilibili_up_dynamic +-- ---------------------------- +DROP TABLE IF EXISTS bilibili_up_dynamic; +CREATE TABLE bilibili_up_dynamic ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dynamic_id TEXT DEFAULT NULL, + user_id TEXT DEFAULT NULL, + user_name TEXT DEFAULT NULL, + text TEXT DEFAULT NULL, + type TEXT DEFAULT NULL, + pub_ts INTEGER DEFAULT NULL, + total_comments INTEGER DEFAULT NULL, + total_forwards INTEGER DEFAULT NULL, + total_liked INTEGER DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL +); + +CREATE INDEX idx_bilibili_up_dynamic_dynamic_id ON bilibili_up_dynamic(dynamic_id); + +-- ---------------------------- +-- Table structure for douyin_aweme +-- ---------------------------- +DROP TABLE IF EXISTS douyin_aweme; +CREATE TABLE douyin_aweme ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + sec_uid TEXT DEFAULT NULL, + short_user_id TEXT DEFAULT NULL, + user_unique_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + user_signature TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + aweme_id TEXT NOT NULL, + aweme_type TEXT NOT NULL, + title TEXT DEFAULT NULL, + desc TEXT, + create_time INTEGER NOT NULL, + liked_count TEXT DEFAULT NULL, + comment_count TEXT DEFAULT NULL, + share_count TEXT DEFAULT NULL, + collected_count TEXT DEFAULT NULL, + aweme_url TEXT DEFAULT NULL, + cover_url TEXT DEFAULT NULL, + video_download_url TEXT DEFAULT NULL, + source_keyword TEXT DEFAULT '' +); + +CREATE INDEX idx_douyin_awem_aweme_i_6f7bc6 ON douyin_aweme(aweme_id); +CREATE INDEX idx_douyin_awem_create__299dfe ON douyin_aweme(create_time); + +-- ---------------------------- +-- Table structure for douyin_aweme_comment +-- ---------------------------- +DROP TABLE IF EXISTS douyin_aweme_comment; +CREATE TABLE douyin_aweme_comment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + sec_uid TEXT DEFAULT NULL, + short_user_id TEXT DEFAULT NULL, + user_unique_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + user_signature TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + comment_id TEXT NOT NULL, + aweme_id TEXT NOT NULL, + content TEXT, + create_time INTEGER NOT NULL, + sub_comment_count TEXT NOT NULL, + parent_comment_id TEXT DEFAULT NULL, + like_count TEXT NOT NULL DEFAULT '0', + pictures TEXT NOT NULL DEFAULT '' +); + +CREATE INDEX idx_douyin_awem_comment_fcd7e4 ON douyin_aweme_comment(comment_id); +CREATE INDEX idx_douyin_awem_aweme_i_c50049 ON douyin_aweme_comment(aweme_id); + +-- ---------------------------- +-- Table structure for dy_creator +-- ---------------------------- +DROP TABLE IF EXISTS dy_creator; +CREATE TABLE dy_creator ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + desc TEXT, + gender TEXT DEFAULT NULL, + follows TEXT DEFAULT NULL, + fans TEXT DEFAULT NULL, + interaction TEXT DEFAULT NULL, + videos_count TEXT DEFAULT NULL +); + +-- ---------------------------- +-- Table structure for kuaishou_video +-- ---------------------------- +DROP TABLE IF EXISTS kuaishou_video; +CREATE TABLE kuaishou_video ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + video_id TEXT NOT NULL, + video_type TEXT NOT NULL, + title TEXT DEFAULT NULL, + desc TEXT, + create_time INTEGER NOT NULL, + liked_count TEXT DEFAULT NULL, + viewd_count TEXT DEFAULT NULL, + video_url TEXT DEFAULT NULL, + video_cover_url TEXT DEFAULT NULL, + video_play_url TEXT DEFAULT NULL, + source_keyword TEXT DEFAULT '' +); + +CREATE INDEX idx_kuaishou_vi_video_i_c5c6a6 ON kuaishou_video(video_id); +CREATE INDEX idx_kuaishou_vi_create__a10dee ON kuaishou_video(create_time); + +-- ---------------------------- +-- Table structure for kuaishou_video_comment +-- ---------------------------- +DROP TABLE IF EXISTS kuaishou_video_comment; +CREATE TABLE kuaishou_video_comment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + comment_id TEXT NOT NULL, + video_id TEXT NOT NULL, + content TEXT, + create_time INTEGER NOT NULL, + sub_comment_count TEXT NOT NULL +); + +CREATE INDEX idx_kuaishou_vi_comment_ed48fa ON kuaishou_video_comment(comment_id); +CREATE INDEX idx_kuaishou_vi_video_i_e50914 ON kuaishou_video_comment(video_id); + +-- ---------------------------- +-- Table structure for weibo_note +-- ---------------------------- +DROP TABLE IF EXISTS weibo_note; +CREATE TABLE weibo_note ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + gender TEXT DEFAULT NULL, + profile_url TEXT DEFAULT NULL, + ip_location TEXT DEFAULT '', + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + note_id TEXT NOT NULL, + content TEXT, + create_time INTEGER NOT NULL, + create_date_time TEXT NOT NULL, + liked_count TEXT DEFAULT NULL, + comments_count TEXT DEFAULT NULL, + shared_count TEXT DEFAULT NULL, + note_url TEXT DEFAULT NULL, + source_keyword TEXT DEFAULT '' +); + +CREATE INDEX idx_weibo_note_note_id_f95b1a ON weibo_note(note_id); +CREATE INDEX idx_weibo_note_create__692709 ON weibo_note(create_time); +CREATE INDEX idx_weibo_note_create__d05ed2 ON weibo_note(create_date_time); + +-- ---------------------------- +-- Table structure for weibo_note_comment +-- ---------------------------- +DROP TABLE IF EXISTS weibo_note_comment; +CREATE TABLE weibo_note_comment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT DEFAULT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + gender TEXT DEFAULT NULL, + profile_url TEXT DEFAULT NULL, + ip_location TEXT DEFAULT '', + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + comment_id TEXT NOT NULL, + note_id TEXT NOT NULL, + content TEXT, + create_time INTEGER NOT NULL, + create_date_time TEXT NOT NULL, + comment_like_count TEXT NOT NULL, + sub_comment_count TEXT NOT NULL, + parent_comment_id TEXT DEFAULT NULL +); + +CREATE INDEX idx_weibo_note__comment_c7611c ON weibo_note_comment(comment_id); +CREATE INDEX idx_weibo_note__note_id_24f108 ON weibo_note_comment(note_id); +CREATE INDEX idx_weibo_note__create__667fe3 ON weibo_note_comment(create_date_time); + +-- ---------------------------- +-- Table structure for weibo_creator +-- ---------------------------- +DROP TABLE IF EXISTS weibo_creator; +CREATE TABLE weibo_creator ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + desc TEXT, + gender TEXT DEFAULT NULL, + follows TEXT DEFAULT NULL, + fans TEXT DEFAULT NULL, + tag_list TEXT +); + +-- ---------------------------- +-- Table structure for xhs_creator +-- ---------------------------- +DROP TABLE IF EXISTS xhs_creator; +CREATE TABLE xhs_creator ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + desc TEXT, + gender TEXT DEFAULT NULL, + follows TEXT DEFAULT NULL, + fans TEXT DEFAULT NULL, + interaction TEXT DEFAULT NULL, + tag_list TEXT +); + +-- ---------------------------- +-- Table structure for xhs_note +-- ---------------------------- +DROP TABLE IF EXISTS xhs_note; +CREATE TABLE xhs_note ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + note_id TEXT NOT NULL, + type TEXT DEFAULT NULL, + title TEXT DEFAULT NULL, + desc TEXT, + video_url TEXT, + time INTEGER NOT NULL, + last_update_time INTEGER NOT NULL, + liked_count TEXT DEFAULT NULL, + collected_count TEXT DEFAULT NULL, + comment_count TEXT DEFAULT NULL, + share_count TEXT DEFAULT NULL, + image_list TEXT, + tag_list TEXT, + note_url TEXT DEFAULT NULL, + source_keyword TEXT DEFAULT '', + xsec_token TEXT DEFAULT NULL +); + +CREATE INDEX idx_xhs_note_note_id_209457 ON xhs_note(note_id); +CREATE INDEX idx_xhs_note_time_eaa910 ON xhs_note(time); + +-- ---------------------------- +-- Table structure for xhs_note_comment +-- ---------------------------- +DROP TABLE IF EXISTS xhs_note_comment; +CREATE TABLE xhs_note_comment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + comment_id TEXT NOT NULL, + create_time INTEGER NOT NULL, + note_id TEXT NOT NULL, + content TEXT NOT NULL, + sub_comment_count INTEGER NOT NULL, + pictures TEXT DEFAULT NULL, + parent_comment_id TEXT DEFAULT NULL, + like_count TEXT DEFAULT NULL +); + +CREATE INDEX idx_xhs_note_co_comment_8e8349 ON xhs_note_comment(comment_id); +CREATE INDEX idx_xhs_note_co_create__204f8d ON xhs_note_comment(create_time); + +-- ---------------------------- +-- Table structure for tieba_note +-- ---------------------------- +DROP TABLE IF EXISTS tieba_note; +CREATE TABLE tieba_note ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + note_id TEXT NOT NULL, + title TEXT NOT NULL, + desc TEXT, + note_url TEXT NOT NULL, + publish_time TEXT NOT NULL, + user_link TEXT DEFAULT '', + user_nickname TEXT DEFAULT '', + user_avatar TEXT DEFAULT '', + tieba_id TEXT DEFAULT '', + tieba_name TEXT NOT NULL, + tieba_link TEXT NOT NULL, + total_replay_num INTEGER DEFAULT 0, + total_replay_page INTEGER DEFAULT 0, + ip_location TEXT DEFAULT '', + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + source_keyword TEXT DEFAULT '' +); + +CREATE INDEX idx_tieba_note_note_id ON tieba_note(note_id); +CREATE INDEX idx_tieba_note_publish_time ON tieba_note(publish_time); + +-- ---------------------------- +-- Table structure for tieba_comment +-- ---------------------------- +DROP TABLE IF EXISTS tieba_comment; +CREATE TABLE tieba_comment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + comment_id TEXT NOT NULL, + parent_comment_id TEXT DEFAULT '', + content TEXT NOT NULL, + user_link TEXT DEFAULT '', + user_nickname TEXT DEFAULT '', + user_avatar TEXT DEFAULT '', + tieba_id TEXT DEFAULT '', + tieba_name TEXT NOT NULL, + tieba_link TEXT NOT NULL, + publish_time TEXT DEFAULT '', + ip_location TEXT DEFAULT '', + sub_comment_count INTEGER DEFAULT 0, + note_id TEXT NOT NULL, + note_url TEXT NOT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL +); + +CREATE INDEX idx_tieba_comment_comment_id ON tieba_comment(comment_id); +CREATE INDEX idx_tieba_comment_note_id ON tieba_comment(note_id); +CREATE INDEX idx_tieba_comment_publish_time ON tieba_comment(publish_time); + +-- ---------------------------- +-- Table structure for tieba_creator +-- ---------------------------- +DROP TABLE IF EXISTS tieba_creator; +CREATE TABLE tieba_creator ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + user_name TEXT NOT NULL, + nickname TEXT DEFAULT NULL, + avatar TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL, + gender TEXT DEFAULT NULL, + follows TEXT DEFAULT NULL, + fans TEXT DEFAULT NULL, + registration_duration TEXT DEFAULT NULL +); + +-- ---------------------------- +-- Table structure for zhihu_content +-- ---------------------------- +DROP TABLE IF EXISTS zhihu_content; +CREATE TABLE zhihu_content ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_id TEXT NOT NULL, + content_type TEXT NOT NULL, + content_text TEXT, + content_url TEXT NOT NULL, + question_id TEXT DEFAULT NULL, + title TEXT NOT NULL, + desc TEXT, + created_time TEXT NOT NULL, + updated_time TEXT NOT NULL, + voteup_count INTEGER NOT NULL DEFAULT 0, + comment_count INTEGER NOT NULL DEFAULT 0, + source_keyword TEXT DEFAULT NULL, + user_id TEXT NOT NULL, + user_link TEXT NOT NULL, + user_nickname TEXT NOT NULL, + user_avatar TEXT NOT NULL, + user_url_token TEXT NOT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL +); + +CREATE INDEX idx_zhihu_content_content_id ON zhihu_content(content_id); +CREATE INDEX idx_zhihu_content_created_time ON zhihu_content(created_time); + +-- ---------------------------- +-- Table structure for zhihu_comment +-- ---------------------------- +DROP TABLE IF EXISTS zhihu_comment; +CREATE TABLE zhihu_comment ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + comment_id TEXT NOT NULL, + parent_comment_id TEXT DEFAULT NULL, + content TEXT NOT NULL, + publish_time TEXT NOT NULL, + ip_location TEXT DEFAULT NULL, + sub_comment_count INTEGER NOT NULL DEFAULT 0, + like_count INTEGER NOT NULL DEFAULT 0, + dislike_count INTEGER NOT NULL DEFAULT 0, + content_id TEXT NOT NULL, + content_type TEXT NOT NULL, + user_id TEXT NOT NULL, + user_link TEXT NOT NULL, + user_nickname TEXT NOT NULL, + user_avatar TEXT NOT NULL, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL +); + +CREATE INDEX idx_zhihu_comment_comment_id ON zhihu_comment(comment_id); +CREATE INDEX idx_zhihu_comment_content_id ON zhihu_comment(content_id); +CREATE INDEX idx_zhihu_comment_publish_time ON zhihu_comment(publish_time); + +-- ---------------------------- +-- Table structure for zhihu_creator +-- ---------------------------- +DROP TABLE IF EXISTS zhihu_creator; +CREATE TABLE zhihu_creator ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL UNIQUE, + user_link TEXT NOT NULL, + user_nickname TEXT NOT NULL, + user_avatar TEXT NOT NULL, + url_token TEXT NOT NULL, + gender TEXT DEFAULT NULL, + ip_location TEXT DEFAULT NULL, + follows INTEGER NOT NULL DEFAULT 0, + fans INTEGER NOT NULL DEFAULT 0, + anwser_count INTEGER NOT NULL DEFAULT 0, + video_count INTEGER NOT NULL DEFAULT 0, + question_count INTEGER NOT NULL DEFAULT 0, + article_count INTEGER NOT NULL DEFAULT 0, + column_count INTEGER NOT NULL DEFAULT 0, + get_voteup_count INTEGER NOT NULL DEFAULT 0, + add_ts INTEGER NOT NULL, + last_modify_ts INTEGER NOT NULL +); + +CREATE UNIQUE INDEX idx_zhihu_creator_user_id ON zhihu_creator(user_id); \ No newline at end of file