rename old

old_py12306/helpers/cdn.py

@@ -0,0 +1,234 @@
+import random
+import json
+from datetime import timedelta
+from os import path
+
+from py12306.cluster.cluster import Cluster
+from py12306.config import Config
+from py12306.app import app_available_check
+from py12306.helpers.api import API_CHECK_CDN_AVAILABLE, HOST_URL_OF_12306
+from py12306.helpers.func import *
+from py12306.helpers.request import Request
+from py12306.log.common_log import CommonLog
+
+
+@singleton
+class Cdn:
+    """
+    CDN 管理
+    """
+    items = []
+    available_items = []
+    unavailable_items = []
+    recheck_available_items = []
+    recheck_unavailable_items = []
+    retry_time = 3
+    is_ready = False
+    is_finished = False
+    is_ready_num = 10  # 当可用超过 10，已准备好
+    is_alive = True
+    is_recheck = False
+
+    safe_stay_time = 0.2
+    retry_num = 1
+    thread_num = 5
+    check_time_out = 3
+
+    last_check_at = 0
+    save_second = 5
+    check_keep_second = 60 * 60 * 24
+
+    def __init__(self):
+        self.cluster = Cluster()
+        self.init_config()
+        create_thread_and_run(self, 'watch_cdn', False)
+
+    def init_data(self):
+        self.items = []
+        self.available_items = []
+        self.unavailable_items = []
+        self.is_finished = False
+        self.is_ready = False
+        self.is_recheck = False
+
+    def init_config(self):
+        self.check_time_out = Config().CDN_CHECK_TIME_OUT
+
+    def update_cdn_status(self, auto=False):
+        if auto:
+            self.init_config()
+            if Config().is_cdn_enabled():
+                self.run()
+            else:
+                self.destroy()
+
+    @classmethod
+    def run(cls):
+        self = cls()
+        app_available_check()
+        self.is_alive = True
+        self.start()
+        pass
+
+    def start(self):
+        if not Config.is_cdn_enabled(): return
+        self.load_items()
+        CommonLog.add_quick_log(CommonLog.MESSAGE_CDN_START_TO_CHECK.format(len(self.items))).flush()
+        self.restore_items()
+        for i in range(self.thread_num):  # 多线程
+            create_thread_and_run(jobs=self, callback_name='check_available', wait=False)
+
+    def load_items(self):
+        with open(Config().CDN_ITEM_FILE, encoding='utf-8') as f:
+            for line, val in enumerate(f):
+                self.items.append(val.rstrip('\n'))
+
+    def restore_items(self):
+        """
+        恢复已有数据
+        :return: bool
+        """
+        result = False
+        if path.exists(Config().CDN_ENABLED_AVAILABLE_ITEM_FILE):
+            with open(Config().CDN_ENABLED_AVAILABLE_ITEM_FILE, encoding='utf-8') as f:
+                result = f.read()
+                try:
+                    result = json.loads(result)
+                except json.JSONDecodeError as e:
+                    result = {}
+
+        # if Config.is_cluster_enabled(): # 集群不用同步 cdn
+        #     result = self.get_data_from_cluster()
+
+        if result:
+            self.last_check_at = result.get('last_check_at', '')
+            if self.last_check_at: self.last_check_at = str_to_time(self.last_check_at)
+            self.available_items = result.get('items', [])
+            self.unavailable_items = result.get('fail_items', [])
+            CommonLog.add_quick_log(CommonLog.MESSAGE_CDN_RESTORE_SUCCESS.format(self.last_check_at)).flush()
+            return True
+        return False
+
+    # def get_data_from_cluster(self):
+    #     available_items = self.cluster.session.smembers(Cluster.KEY_CDN_AVAILABLE_ITEMS)
+    #     last_time = self.cluster.session.get(Cluster.KEY_CDN_LAST_CHECK_AT, '')
+    #     if available_items and last_time:
+    #         return {'items': available_items, 'last_check_at': last_time}
+    #     return False
+
+    def is_need_to_recheck(self):
+        """
+        是否需要重新检查 cdn
+        :return:
+        """
+        if self.last_check_at and (
+                time_now() - self.last_check_at).seconds > self.check_keep_second:
+            return True
+        return False
+
+    def get_unchecked_item(self):
+        if not self.is_recheck:
+            items = list(set(self.items) - set(self.available_items) - set(self.unavailable_items))
+        else:
+            items = list(set(self.items) - set(self.recheck_available_items) - set(self.recheck_unavailable_items))
+        if items: return random.choice(items)
+        return None
+
+    def check_available(self):
+        while True and self.is_alive:
+            item = self.get_unchecked_item()
+            if not item: return self.check_did_finished()
+            self.check_item_available(item)
+
+    def watch_cdn(self):
+        """
+        监控 cdn 状态，自动重新检测
+        :return:
+        """
+        while True:
+            if self.is_alive and not self.is_recheck and self.is_need_to_recheck():  # 重新检测
+                self.is_recheck = True
+                self.is_finished = False
+                CommonLog.add_quick_log(
+                    CommonLog.MESSAGE_CDN_START_TO_RECHECK.format(len(self.items), time_now())).flush()
+                for i in range(self.thread_num):  # 多线程
+                    create_thread_and_run(jobs=self, callback_name='check_available', wait=False)
+            stay_second(self.retry_num)
+
+    def destroy(self):
+        """
+        关闭 CDN
+        :return:
+        """
+        CommonLog.add_quick_log(CommonLog.MESSAGE_CDN_CLOSED).flush()
+        self.is_alive = False
+        self.init_data()
+
+    def check_item_available(self, item, try_num=0):
+        session = Request()
+        response = session.get(API_CHECK_CDN_AVAILABLE.format(item), headers={'Host': HOST_URL_OF_12306},
+                               timeout=self.check_time_out,
+                               verify=False)
+
+        if response.status_code == 200:
+            if not self.is_recheck:
+                self.available_items.append(item)
+            else:
+                self.recheck_available_items.append(item)
+            if not self.is_ready: self.check_is_ready()
+        elif try_num < self.retry_num:  # 重试
+            stay_second(self.safe_stay_time)
+            return self.check_item_available(item, try_num + 1)
+        else:
+            if not self.is_recheck:
+                self.unavailable_items.append(item)
+            else:
+                self.recheck_unavailable_items.append(item)
+        if not self.is_recheck and (
+                not self.last_check_at or (time_now() - self.last_check_at).seconds > self.save_second):
+            self.save_available_items()
+        stay_second(self.safe_stay_time)
+
+    def check_did_finished(self):
+        self.is_ready = True
+        if not self.is_finished:
+            self.is_finished = True
+            if self.is_recheck:
+                self.is_recheck = False
+                self.available_items = self.recheck_available_items
+                self.unavailable_items = self.recheck_unavailable_items
+                self.recheck_available_items = []
+                self.recheck_unavailable_items = []
+            CommonLog.add_quick_log(CommonLog.MESSAGE_CDN_CHECKED_SUCCESS.format(len(self.available_items))).flush()
+            self.save_available_items()
+
+    def save_available_items(self):
+        self.last_check_at = time_now()
+        data = {'items': self.available_items, 'fail_items': self.unavailable_items,
+                'last_check_at': str(self.last_check_at)}
+        with open(Config().CDN_ENABLED_AVAILABLE_ITEM_FILE, 'w') as f:
+            f.write(json.dumps(data))
+
+        # if Config.is_master():
+        #     self.cluster.session.sadd(Cluster.KEY_CDN_AVAILABLE_ITEMS, self.available_items)
+        #     self.cluster.session.set(Cluster.KEY_CDN_LAST_CHECK_AT, time_now())
+
+    def check_is_ready(self):
+        if len(self.available_items) > self.is_ready_num:
+            self.is_ready = True
+        else:
+            self.is_ready = False
+
+    @classmethod
+    def get_cdn(cls):
+        self = cls()
+        if self.is_ready and self.available_items:
+            return random.choice(self.available_items)
+        return None
+
+
+if __name__ == '__main__':
+    # Const.IS_TEST = True
+    Cdn.run()
+    while not Cdn().is_finished:
+        stay_second(1)