rename old
This commit is contained in:
Jalin
0
old_py12306/cluster/__init__.py
Normal file
0
old_py12306/cluster/__init__.py
Normal file
274
old_py12306/cluster/cluster.py
Normal file
274
old_py12306/cluster/cluster.py
Normal file
@@ -0,0 +1,274 @@
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
import time
|
||||
|
||||
import redis
|
||||
from redis.client import PubSub
|
||||
|
||||
from py12306.cluster.redis import Redis
|
||||
from py12306.config import Config
|
||||
from py12306.helpers.func import *
|
||||
from py12306.log.cluster_log import ClusterLog
|
||||
|
||||
|
||||
@singleton
|
||||
class Cluster():
|
||||
KEY_PREFIX = 'py12306_' # 目前只能手动
|
||||
KEY_QUERY_COUNT = KEY_PREFIX + 'query_count'
|
||||
KEY_QUERY_LAST_TIME = KEY_PREFIX + 'query_last_time'
|
||||
KEY_CONFIGS = KEY_PREFIX + 'configs'
|
||||
KEY_NODES = KEY_PREFIX + 'nodes'
|
||||
KEY_CHANNEL_LOG = KEY_PREFIX + 'channel_log'
|
||||
KEY_CHANNEL_EVENT = KEY_PREFIX + 'channel_even'
|
||||
KEY_USER_COOKIES = KEY_PREFIX + 'user_cookies'
|
||||
KEY_USER_INFOS = KEY_PREFIX + 'user_infos'
|
||||
KEY_USER_LAST_HEARTBEAT = KEY_PREFIX + 'user_last_heartbeat'
|
||||
KEY_NODES_ALIVE_PREFIX = KEY_PREFIX + 'nodes_alive_'
|
||||
|
||||
KEY_CDN_AVAILABLE_ITEMS = KEY_PREFIX + 'cdn_available_items'
|
||||
KEY_CDN_LAST_CHECK_AT = KEY_PREFIX + 'cdn_last_check_at'
|
||||
|
||||
# 锁
|
||||
KEY_LOCK_INIT_USER = KEY_PREFIX + 'lock_init_user' # 暂未使用
|
||||
KEY_LOCK_DO_ORDER = KEY_PREFIX + 'lock_do_order' # 订单锁
|
||||
lock_do_order_time = 60 * 1 # 订单锁超时时间
|
||||
|
||||
lock_prefix = KEY_PREFIX + 'lock_' # 锁键前缀
|
||||
lock_info_prefix = KEY_PREFIX + 'info_'
|
||||
|
||||
KEY_MASTER = 1
|
||||
KEY_SLAVE = 0
|
||||
|
||||
session: Redis = None
|
||||
pubsub: PubSub = None
|
||||
refresh_channel_time = 0.5
|
||||
retry_time = 2
|
||||
keep_alive_time = 3 # 报告存活间隔
|
||||
lost_alive_time = keep_alive_time * 2
|
||||
|
||||
nodes = {}
|
||||
node_name = None
|
||||
is_ready = False
|
||||
is_master = False
|
||||
|
||||
def __init__(self, *args):
|
||||
if Config.is_cluster_enabled():
|
||||
self.session = Redis()
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def run(cls):
|
||||
self = cls()
|
||||
self.start()
|
||||
|
||||
def start(self):
|
||||
self.pubsub = self.session.pubsub()
|
||||
self.pubsub.subscribe(self.KEY_CHANNEL_LOG, self.KEY_CHANNEL_EVENT)
|
||||
create_thread_and_run(self, 'subscribe', wait=False)
|
||||
self.is_ready = True
|
||||
self.get_nodes() # 提前获取节点列表
|
||||
self.check_nodes() # 防止 节点列表未清空
|
||||
self.join_cluster()
|
||||
create_thread_and_run(self, 'keep_alive', wait=False)
|
||||
create_thread_and_run(self, 'refresh_data', wait=False)
|
||||
|
||||
def join_cluster(self):
|
||||
"""
|
||||
加入到集群
|
||||
:return:
|
||||
"""
|
||||
self.node_name = node_name = Config().NODE_NAME
|
||||
|
||||
if Config().NODE_IS_MASTER:
|
||||
if self.node_name in self.nodes: # 重复运行主节点
|
||||
ClusterLog.add_quick_log(ClusterLog.MESSAGE_MASTER_NODE_ALREADY_RUN.format(node_name)).flush(
|
||||
publish=False)
|
||||
os._exit(1)
|
||||
if self.have_master(): # 子节点提升为主节点情况，交回控制
|
||||
message = ClusterLog.MESSAGE_NODE_BECOME_MASTER_AGAIN.format(node_name)
|
||||
self.publish_log_message(message)
|
||||
self.make_nodes_as_slave()
|
||||
elif not self.have_master(): # 只能通过主节点启动
|
||||
ClusterLog.add_quick_log(ClusterLog.MESSAGE_MASTER_NODE_NOT_FOUND).flush(publish=False)
|
||||
os._exit(1)
|
||||
|
||||
if node_name in self.nodes:
|
||||
self.node_name = node_name = node_name + '_' + str(dict_count_key_num(self.nodes, node_name))
|
||||
ClusterLog.add_quick_log(ClusterLog.MESSAGE_NODE_ALREADY_IN_CLUSTER.format(node_name)).flush()
|
||||
|
||||
self.session.hset(self.KEY_NODES, node_name, Config().NODE_IS_MASTER)
|
||||
message = ClusterLog.MESSAGE_JOIN_CLUSTER_SUCCESS.format(self.node_name, ClusterLog.get_print_nodes(
|
||||
self.get_nodes())) # 手动 get nodes
|
||||
self.publish_log_message(message)
|
||||
|
||||
def left_cluster(self, node_name=None):
|
||||
node_name = node_name if node_name else self.node_name
|
||||
self.session.hdel(self.KEY_NODES, node_name)
|
||||
message = ClusterLog.MESSAGE_LEFT_CLUSTER.format(node_name, ClusterLog.get_print_nodes(self.get_nodes()))
|
||||
self.publish_log_message(message, node_name)
|
||||
|
||||
def make_nodes_as_slave(self):
|
||||
"""
|
||||
将所有节点设为主节点
|
||||
:return:
|
||||
"""
|
||||
for node in self.nodes:
|
||||
self.session.hset(self.KEY_NODES, node, self.KEY_SLAVE)
|
||||
|
||||
def publish_log_message(self, message, node_name=None):
|
||||
"""
|
||||
发布订阅消息
|
||||
:return:
|
||||
"""
|
||||
node_name = node_name if node_name else self.node_name
|
||||
message = ClusterLog.MESSAGE_SUBSCRIBE_NOTIFICATION.format(node_name, message)
|
||||
self.session.publish(self.KEY_CHANNEL_LOG, message)
|
||||
|
||||
def publish_event(self, name, data={}):
|
||||
"""
|
||||
发布事件消息
|
||||
:return:
|
||||
"""
|
||||
data = {'event': name, 'data': data}
|
||||
self.session.publish(self.KEY_CHANNEL_EVENT, json.dumps(data))
|
||||
|
||||
def get_nodes(self) -> dict:
|
||||
res = self.session.hgetall(self.KEY_NODES)
|
||||
res = res if res else {}
|
||||
self.nodes = res
|
||||
return res
|
||||
|
||||
def refresh_data(self):
|
||||
"""
|
||||
单独进程处理数据同步
|
||||
:return:
|
||||
"""
|
||||
while True:
|
||||
self.get_nodes()
|
||||
self.check_locks()
|
||||
self.check_nodes()
|
||||
self.check_master()
|
||||
stay_second(self.retry_time)
|
||||
|
||||
def check_master(self):
|
||||
"""
|
||||
检测主节点是否可用
|
||||
:return:
|
||||
"""
|
||||
master = self.have_master()
|
||||
if master == self.node_name: # 动态提升
|
||||
self.is_master = True
|
||||
else:
|
||||
self.is_master = False
|
||||
|
||||
if not master:
|
||||
if Config().NODE_SLAVE_CAN_BE_MASTER:
|
||||
# 提升子节点为主节点
|
||||
slave = list(self.nodes)[0]
|
||||
self.session.hset(self.KEY_NODES, slave, self.KEY_MASTER)
|
||||
self.publish_log_message(ClusterLog.MESSAGE_ASCENDING_MASTER_NODE.format(slave,
|
||||
ClusterLog.get_print_nodes(
|
||||
self.get_nodes())))
|
||||
return True
|
||||
else:
|
||||
self.publish_log_message(ClusterLog.MESSAGE_MASTER_DID_LOST.format(self.retry_time))
|
||||
stay_second(self.retry_time)
|
||||
os._exit(1) # 退出整个程序
|
||||
|
||||
def have_master(self):
|
||||
return dict_find_key_by_value(self.nodes, str(self.KEY_MASTER), False)
|
||||
|
||||
def check_nodes(self):
|
||||
"""
|
||||
检查节点是否存活
|
||||
:return:
|
||||
"""
|
||||
for node in self.nodes:
|
||||
if not self.session.exists(self.KEY_NODES_ALIVE_PREFIX + node):
|
||||
self.left_cluster(node)
|
||||
|
||||
# def kick_out_from_nodes(self, node_name):
|
||||
# pass
|
||||
|
||||
def keep_alive(self):
|
||||
while True:
|
||||
if self.node_name not in self.get_nodes(): # 已经被 kict out 重新加下
|
||||
self.join_cluster()
|
||||
self.session.set(self.KEY_NODES_ALIVE_PREFIX + self.node_name, Config().NODE_IS_MASTER, ex=self.lost_alive_time)
|
||||
stay_second(self.keep_alive_time)
|
||||
|
||||
def subscribe(self):
|
||||
while True:
|
||||
try:
|
||||
message = self.pubsub.get_message()
|
||||
except RuntimeError as err:
|
||||
if 'args' in dir(err) and err.args[0].find('pubsub connection not set') >= 0: # 失去重连
|
||||
self.pubsub.subscribe(self.KEY_CHANNEL_LOG, self.KEY_CHANNEL_EVENT)
|
||||
continue
|
||||
if message:
|
||||
if message.get('type') == 'message' and message.get('channel') == self.KEY_CHANNEL_LOG and message.get(
|
||||
'data'):
|
||||
msg = message.get('data')
|
||||
if self.node_name:
|
||||
msg = msg.replace(ClusterLog.MESSAGE_SUBSCRIBE_NOTIFICATION_PREFIX.format(self.node_name), '')
|
||||
ClusterLog.add_quick_log(msg).flush(publish=False)
|
||||
elif message.get('channel') == self.KEY_CHANNEL_EVENT:
|
||||
create_thread_and_run(self, 'handle_events', args=(message,))
|
||||
stay_second(self.refresh_channel_time)
|
||||
|
||||
def handle_events(self, message):
|
||||
# 这里应该分开处理，先都在这处理了
|
||||
if message.get('type') != 'message': return
|
||||
result = json.loads(message.get('data', {}))
|
||||
event_name = result.get('event')
|
||||
data = result.get('data')
|
||||
from py12306.helpers.event import Event
|
||||
method = getattr(Event(), event_name)
|
||||
if method:
|
||||
create_thread_and_run(Event(), event_name, Const.IS_TEST, kwargs={'data': data, 'callback': True})
|
||||
|
||||
def get_lock(self, key: str, timeout=1, info={}):
|
||||
timeout = int(time.time()) + timeout
|
||||
res = self.session.setnx(key, timeout)
|
||||
if res:
|
||||
if info: self.session.set_dict(self.lock_info_prefix + key.replace(self.KEY_PREFIX, ''), info) # 存储额外信息
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_lock_info(self, key, default={}):
|
||||
return self.session.get_dict(self.lock_info_prefix + key.replace(self.KEY_PREFIX, ''), default=default)
|
||||
|
||||
def release_lock(self, key):
|
||||
self.session.delete(key)
|
||||
self.session.delete(self.lock_info_prefix + key.replace(self.KEY_PREFIX, ''))
|
||||
|
||||
def check_locks(self):
|
||||
locks = self.session.keys(self.lock_prefix + '*')
|
||||
for key in locks:
|
||||
val = self.session.get(key)
|
||||
if val and int(val) <= time_int():
|
||||
self.release_lock(key)
|
||||
|
||||
@classmethod
|
||||
def get_user_cookie(cls, key, default=None):
|
||||
self = cls()
|
||||
res = self.session.hget(Cluster.KEY_USER_COOKIES, key)
|
||||
return pickle.loads(res.encode()) if res else default
|
||||
|
||||
@classmethod
|
||||
def set_user_cookie(cls, key, value):
|
||||
self = cls()
|
||||
return self.session.hset(Cluster.KEY_USER_COOKIES, key, pickle.dumps(value, 0).decode())
|
||||
|
||||
@classmethod
|
||||
def set_user_info(cls, key, info):
|
||||
self = cls()
|
||||
return self.session.hset(Cluster.KEY_USER_INFOS, key, pickle.dumps(info, 0).decode())
|
||||
|
||||
@classmethod
|
||||
def get_user_info(cls, key, default=None):
|
||||
self = cls()
|
||||
res = self.session.hget(Cluster.KEY_USER_INFOS, key)
|
||||
return pickle.loads(res.encode()) if res else default
|
||||
59
old_py12306/cluster/redis.py
Normal file
59
old_py12306/cluster/redis.py
Normal file
@@ -0,0 +1,59 @@
|
||||
import json
|
||||
import pickle
|
||||
|
||||
import redis
|
||||
|
||||
from py12306.config import Config
|
||||
from py12306.helpers.func import *
|
||||
from py12306.log.redis_log import RedisLog
|
||||
from redis import Redis as PyRedis
|
||||
|
||||
|
||||
@singleton
|
||||
class Redis(PyRedis):
|
||||
# session = None
|
||||
|
||||
def __init__(self, *args):
|
||||
if Config.is_cluster_enabled():
|
||||
args = {
|
||||
'host': Config().REDIS_HOST,
|
||||
'port': Config().REDIS_PORT,
|
||||
'db': 0,
|
||||
'password': Config().REDIS_PASSWORD,
|
||||
'decode_responses': True
|
||||
}
|
||||
super().__init__(**args)
|
||||
RedisLog.add_quick_log(RedisLog.MESSAGE_REDIS_INIT_SUCCESS)
|
||||
else:
|
||||
super().__init__(**args)
|
||||
return self
|
||||
|
||||
def get(self, name, default=None):
|
||||
res = super().get(name)
|
||||
# if decode: res = res.decode()
|
||||
return res if res else default
|
||||
|
||||
def set(self, name, value, ex=None, px=None, nx=False, xx=False):
|
||||
return super().set(name, available_value(value), ex=ex, px=px, nx=nx, xx=xx)
|
||||
|
||||
def set_dict(self, name, value):
|
||||
return self.set_pickle(name, value)
|
||||
# return self.set(name, json.dumps(value))
|
||||
|
||||
def get_dict(self, name, default={}):
|
||||
return self.get_pickle(name, default)
|
||||
# res = self.get(name)
|
||||
# if res:
|
||||
# return json.loads(res)
|
||||
# return default
|
||||
|
||||
def set_pickle(self, name, value):
|
||||
return self.set(name, pickle.dumps(value, 0).decode())
|
||||
|
||||
def get_pickle(self, name, default=None):
|
||||
res = self.get(name)
|
||||
return pickle.loads(res.encode()) if res else default
|
||||
|
||||
# def smembers(self, name, default=[]):
|
||||
# res = super().smembers(name)
|
||||
# return [val.decode() for val in list(res)] if res else default
|
||||
Reference in New Issue
Block a user