From c6f5411efcb196974f0ea337966b99cf8e91c04f Mon Sep 17 00:00:00 2001 From: John Smith Date: Tue, 20 Apr 2021 21:30:57 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=8E=B7=E5=8F=96=E5=A4=B4?= =?UTF-8?q?=E5=83=8F=E9=97=B4=E9=9A=94=E3=80=81=E7=BC=93=E5=AD=98=E7=AD=89?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.py | 20 ++++++++++++++++---- data/config.example.ini | 34 ++++++++++++++++++++++++++-------- models/avatar.py | 12 +++++++++--- models/translate.py | 3 ++- 4 files changed, 53 insertions(+), 16 deletions(-) diff --git a/config.py b/config.py index 962339f..8fb2aa2 100644 --- a/config.py +++ b/config.py @@ -47,10 +47,16 @@ def get_config(): class AppConfig: def __init__(self): self.database_url = 'sqlite:///data/database.db' - self.enable_translate = True - self.allow_translate_rooms = {} self.tornado_xheaders = False self.loader_url = '' + + self.fetch_avatar_interval = 3.5 + self.fetch_avatar_max_queue_size = 2 + self.avatar_cache_size = 50000 + + self.enable_translate = True + self.allow_translate_rooms = set() + self.translation_cache_size = 50000 self.translator_configs = [] def load(self, path): @@ -68,11 +74,17 @@ class AppConfig: def _load_app_config(self, config): app_section = config['app'] self.database_url = app_section['database_url'] - self.enable_translate = app_section.getboolean('enable_translate') - self.allow_translate_rooms = _str_to_list(app_section['allow_translate_rooms'], int, set) self.tornado_xheaders = app_section.getboolean('tornado_xheaders') self.loader_url = app_section['loader_url'] + self.fetch_avatar_interval = app_section.getfloat('fetch_avatar_interval') + self.fetch_avatar_max_queue_size = app_section.getint('fetch_avatar_max_queue_size') + self.avatar_cache_size = app_section.getint('avatar_cache_size') + + self.enable_translate = app_section.getboolean('enable_translate') + self.allow_translate_rooms = _str_to_list(app_section['allow_translate_rooms'], int, set) + self.translation_cache_size = app_section.getint('translation_cache_size') + def _load_translator_configs(self, config): app_section = config['app'] section_names = _str_to_list(app_section['translator_configs']) diff --git a/data/config.example.ini b/data/config.example.ini index 770700a..27ac741 100644 --- a/data/config.example.ini +++ b/data/config.example.ini @@ -6,6 +6,29 @@ # See https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls database_url = sqlite:///data/database.db +# 如果使用了nginx之类的反向代理服务器，设置为true +# Set to true if you are using a reverse proxy server such as nginx +tornado_xheaders = false + +# 加载器URL，本地使用时加载器可以让你先运行OBS再运行blivechat。如果为空，不使用加载器 +# **自建服务器时强烈建议不使用加载器**，否则可能因为混合HTTP和HTTPS等原因加载不出来 +# Use a loader so that you can run OBS before blivechat. If empty, no loader is used +loader_url = https://xfgryujk.sinacloud.net/blivechat/loader.html + + +# 获取头像间隔时间（秒）。如果小于3秒有很大概率被服务器拉黑 +# Interval between fetching avatar (s). At least 3 seconds is recommended +fetch_avatar_interval = 3.5 + +# 获取头像最大队列长度，注意最长等待时间等于 最大队列长度 * 请求间隔时间 +# Maximum queue length for fetching avatar +fetch_avatar_max_queue_size = 2 + +# 头像缓存数量 +# Number of avatar caches +avatar_cache_size = 50000 + + # 允许自动翻译到日语 # Enable auto translate to Japanese enable_translate = true @@ -15,14 +38,9 @@ enable_translate = true # Example: allow_translate_rooms = 4895312,22347054,21693691 allow_translate_rooms = -# 如果使用了nginx之类的反向代理服务器，设置为true -# Set to true if you are using a reverse proxy server such as nginx -tornado_xheaders = false - -# 加载器URL，本地使用时加载器可以让你先运行OBS再运行blivechat。如果为空，不使用加载器 -# **自建服务器时强烈建议不使用加载器**，否则可能因为混合HTTP和HTTPS等原因加载不出来 -# Use a loader so that you can run OBS before blivechat. If empty, no loader is used -loader_url = https://xfgryujk.sinacloud.net/blivechat/loader.html +# 翻译缓存数量 +# Number of translation caches +translation_cache_size = 50000 # ------------------------------------------------------------------------------------------------- diff --git a/models/avatar.py b/models/avatar.py index 353f8ed..a8995c8 100644 --- a/models/avatar.py +++ b/models/avatar.py @@ -10,6 +10,7 @@ import aiohttp import sqlalchemy import sqlalchemy.exc +import config import models.database logger = logging.getLogger(__name__) @@ -24,12 +25,15 @@ _avatar_url_cache: Dict[int, str] = {} # 正在获取头像的Future，user_id -> Future _uid_fetch_future_map: Dict[int, asyncio.Future] = {} # 正在获取头像的user_id队列 -_uid_queue_to_fetch = asyncio.Queue(15) +_uid_queue_to_fetch = None # 上次被B站ban时间 _last_fetch_banned_time: Optional[datetime.datetime] = None def init(): + cfg = config.get_config() + global _uid_queue_to_fetch + _uid_queue_to_fetch = asyncio.Queue(cfg.fetch_avatar_max_queue_size) asyncio.ensure_future(_get_avatar_url_from_web_consumer()) @@ -124,7 +128,8 @@ async def _get_avatar_url_from_web_consumer(): asyncio.ensure_future(_get_avatar_url_from_web_coroutine(user_id, future)) # 限制频率，防止被B站ban - await asyncio.sleep(0.2) + cfg = config.get_config() + await asyncio.sleep(cfg.fetch_avatar_interval) except Exception: logger.exception('_get_avatar_url_from_web_consumer error:') @@ -178,7 +183,8 @@ def update_avatar_cache(user_id, avatar_url): def _update_avatar_cache_in_memory(user_id, avatar_url): _avatar_url_cache[user_id] = avatar_url - while len(_avatar_url_cache) > 50000: + cfg = config.get_config() + while len(_avatar_url_cache) > cfg.avatar_cache_size: _avatar_url_cache.pop(next(iter(_avatar_url_cache)), None) diff --git a/models/translate.py b/models/translate.py index 91c9389..7ca74a2 100644 --- a/models/translate.py +++ b/models/translate.py @@ -124,7 +124,8 @@ def _on_translate_done(key, future): if res is None: return _translate_cache[key] = res - while len(_translate_cache) > 50000: + cfg = config.get_config() + while len(_translate_cache) > cfg.translation_cache_size: _translate_cache.pop(next(iter(_translate_cache)), None)