From 8f66b909122d25f5f6e6b6ae0576b983b2b7fb02 Mon Sep 17 00:00:00 2001 From: Lost-MSth Date: Mon, 11 Mar 2024 16:31:21 +0800 Subject: [PATCH] [Enhance] Content bundle - Add support for content bundles - For Arcaea 5.4.0 play store version and iOS version --- latest version/core/bundle.py | 191 ++++++++++++++++++ latest version/core/config_manager.py | 7 +- latest version/core/constant.py | 4 + latest version/core/error.py | 7 + latest version/core/init.py | 20 +- latest version/core/operation.py | 11 + latest version/core/sql.py | 2 + latest version/main.py | 26 ++- latest version/server/func.py | 14 +- latest version/server/others.py | 34 ++-- .../templates/web/updatedatabase.html | 15 +- latest version/web/index.py | 14 +- 12 files changed, 313 insertions(+), 32 deletions(-) create mode 100644 latest version/core/bundle.py diff --git a/latest version/core/bundle.py b/latest version/core/bundle.py new file mode 100644 index 0000000..c4667c9 --- /dev/null +++ b/latest version/core/bundle.py @@ -0,0 +1,191 @@ +import json +import os +from time import time + +from flask import url_for + +from .constant import Constant +from .error import NoAccess, RateLimit +from .limiter import ArcLimiter + + +class ContentBundle: + + def __init__(self) -> None: + self.version: str = None + self.prev_version: str = None + self.app_version: str = None + self.uuid: str = None + + self.json_size: int = None + self.bundle_size: int = None + self.json_path: str = None # relative path + self.bundle_path: str = None # relative path + + self.json_url: str = None + self.bundle_url: str = None + + @staticmethod + def parse_version(version: str) -> tuple: + try: + r = tuple(map(int, version.split('.'))) + except AttributeError: + r = (0, 0, 0) + return r + + @property + def version_tuple(self) -> tuple: + return self.parse_version(self.version) + + @classmethod + def from_json(cls, json_data: dict) -> 'ContentBundle': + x = cls() + x.version = json_data['versionNumber'] + x.prev_version = json_data['previousVersionNumber'] + x.app_version = json_data['applicationVersionNumber'] + x.uuid = json_data['uuid'] + return x + + def to_dict(self) -> dict: + r = { + 'contentBundleVersion': self.version, + 'appVersion': self.app_version, + 'jsonSize': self.json_size, + 'bundleSize': self.bundle_size, + } + if self.json_url and self.bundle_url: + r['jsonUrl'] = self.json_url + r['bundleUrl'] = self.bundle_url + return r + + def calculate_size(self) -> None: + self.json_size = os.path.getsize(os.path.join( + Constant.CONTENT_BUNDLE_FOLDER_PATH, self.json_path)) + self.bundle_size = os.path.getsize(os.path.join( + Constant.CONTENT_BUNDLE_FOLDER_PATH, self.bundle_path)) + + +class BundleParser: + + # {app_version: [ List[ContentBundle] ]} + bundles: 'dict[str, list[ContentBundle]]' = {} + max_bundle_version: 'dict[str, str]' = {} + + def __init__(self) -> None: + self.parse() + + def re_init(self) -> None: + self.bundles.clear() + self.max_bundle_version.clear() + self.parse() + + def parse(self) -> None: + for root, dirs, files in os.walk(Constant.CONTENT_BUNDLE_FOLDER_PATH): + for file in files: + if not file.endswith('.json'): + continue + + json_path = os.path.join(root, file) + bundle_path = os.path.join(root, f'{file[:-5]}.cb') + + with open(json_path, 'rb') as f: + data = json.load(f) + + x = ContentBundle.from_json(data) + + x.json_path = os.path.relpath( + json_path, Constant.CONTENT_BUNDLE_FOLDER_PATH) + x.bundle_path = os.path.relpath( + bundle_path, Constant.CONTENT_BUNDLE_FOLDER_PATH) + + x.json_path = x.json_path.replace('\\', '/') + x.bundle_path = x.bundle_path.replace('\\', '/') + + if not os.path.isfile(bundle_path): + raise FileNotFoundError( + f'Bundle file not found: {bundle_path}') + x.calculate_size() + + if x.app_version not in self.bundles: + self.bundles[x.app_version] = [] + self.bundles[x.app_version].append(x) + + # sort by version + for k, v in self.bundles.items(): + v.sort(key=lambda x: x.version_tuple) + self.max_bundle_version[k] = v[-1].version + + +class BundleDownload: + + limiter = ArcLimiter( + Constant.BUNDLE_DOWNLOAD_TIMES_LIMIT, 'bundle_download') + + def __init__(self, c_m=None): + self.c_m = c_m + + self.client_app_version = None + self.client_bundle_version = None + self.device_id = None + + def set_client_info(self, app_version: str, bundle_version: str, device_id: str = None) -> None: + self.client_app_version = app_version + self.client_bundle_version = bundle_version + self.device_id = device_id + + def get_bundle_list(self) -> list: + bundles: 'list[ContentBundle]' = BundleParser.bundles.get( + self.client_app_version, []) + if not bundles: + return [] + + now = time() + + if Constant.BUNDLE_DOWNLOAD_LINK_PREFIX: + prefix = Constant.BUNDLE_DOWNLOAD_LINK_PREFIX + if prefix[-1] != '/': + prefix += '/' + + def url_func(x): return f'{prefix}{x}' + else: + def url_func(x): return url_for( + 'bundle_download', token=x, _external=True) + + sql_list = [] + r = [] + for x in bundles: + if x.version_tuple <= ContentBundle.parse_version(self.client_bundle_version): + continue + t1 = os.urandom(64).hex() + t2 = os.urandom(64).hex() + + x.json_url = url_func(t1) + x.bundle_url = url_func(t2) + + sql_list.append((t1, x.json_path, now, self.device_id)) + sql_list.append((t2, x.bundle_path, now, self.device_id)) + r.append(x.to_dict()) + + if not sql_list: + return [] + + self.c_m.executemany( + '''insert into bundle_download_token values (?, ?, ?, ?)''', sql_list) + + return r + + def get_path_by_token(self, token: str, ip: str) -> str: + r = self.c_m.execute( + '''select file_path, time, device_id from bundle_download_token where token = ?''', (token,)).fetchone() + if not r: + raise NoAccess('Invalid token.', status=403) + file_path, create_time, device_id = r + + if time() - create_time > Constant.BUNDLE_DOWNLOAD_TIME_GAP_LIMIT: + raise NoAccess('Expired token.', status=403) + + if file_path.endswith('.cb') and not self.limiter.hit(ip): + raise RateLimit( + f'Too many content bundle downloads, IP: {ip}, DeviceID: {device_id}', status=429) + + return file_path diff --git a/latest version/core/config_manager.py b/latest version/core/config_manager.py index 6e82c28..4f00992 100644 --- a/latest version/core/config_manager.py +++ b/latest version/core/config_manager.py @@ -42,7 +42,8 @@ class Config: API_TOKEN = '' - DOWNLOAD_LINK_PREFIX = '' + DOWNLOAD_LINK_PREFIX = '' # http(s)://host(:port)/download/ + BUNDLE_DOWNLOAD_LINK_PREFIX = '' # http(s)://host(:port)/bundle_download/ DOWNLOAD_USE_NGINX_X_ACCEL_REDIRECT = False NGINX_X_ACCEL_REDIRECT_PREFIX = '/nginx_download/' @@ -52,6 +53,9 @@ class Config: DOWNLOAD_FORBID_WHEN_NO_ITEM = False + BUNDLE_DOWNLOAD_TIMES_LIMIT = '100/60 minutes' + BUNDLE_DOWNLOAD_TIME_GAP_LIMIT = 3000 + LOGIN_DEVICE_NUMBER_LIMIT = 1 ALLOW_LOGIN_SAME_DEVICE = False ALLOW_BAN_MULTIDEVICE_USER_AUTO = True @@ -84,6 +88,7 @@ class Config: WORLD_MAP_FOLDER_PATH = './database/map/' SONG_FILE_FOLDER_PATH = './database/songs/' SONGLIST_FILE_PATH = './database/songs/songlist' + CONTENT_BUNDLE_FOLDER_PATH = './database/bundle/' SQLITE_DATABASE_PATH = './database/arcaea_database.db' SQLITE_DATABASE_BACKUP_FOLDER_PATH = './database/backup/' DATABASE_INIT_PATH = './database/init/' diff --git a/latest version/core/constant.py b/latest version/core/constant.py index 0d1caf4..93d329b 100644 --- a/latest version/core/constant.py +++ b/latest version/core/constant.py @@ -43,6 +43,7 @@ class Constant: WORLD_MAP_FOLDER_PATH = Config.WORLD_MAP_FOLDER_PATH SONG_FILE_FOLDER_PATH = Config.SONG_FILE_FOLDER_PATH SONGLIST_FILE_PATH = Config.SONGLIST_FILE_PATH + CONTENT_BUNDLE_FOLDER_PATH = Config.CONTENT_BUNDLE_FOLDER_PATH SQLITE_DATABASE_PATH = Config.SQLITE_DATABASE_PATH SQLITE_LOG_DATABASE_PATH = Config.SQLITE_LOG_DATABASE_PATH SQLITE_DATABASE_DELETED_PATH = Config.SQLITE_DATABASE_DELETED_PATH @@ -50,6 +51,9 @@ class Constant: DOWNLOAD_TIMES_LIMIT = Config.DOWNLOAD_TIMES_LIMIT DOWNLOAD_TIME_GAP_LIMIT = Config.DOWNLOAD_TIME_GAP_LIMIT DOWNLOAD_LINK_PREFIX = Config.DOWNLOAD_LINK_PREFIX + BUNDLE_DOWNLOAD_TIMES_LIMIT = Config.BUNDLE_DOWNLOAD_TIMES_LIMIT + BUNDLE_DOWNLOAD_TIME_GAP_LIMIT = Config.BUNDLE_DOWNLOAD_TIME_GAP_LIMIT + BUNDLE_DOWNLOAD_LINK_PREFIX = Config.BUNDLE_DOWNLOAD_LINK_PREFIX LINKPLAY_UNLOCK_LENGTH = 512 # Units: bytes LINKPLAY_TIMEOUT = 5 # Units: seconds diff --git a/latest version/core/error.py b/latest version/core/error.py index 439cdec..40913d3 100644 --- a/latest version/core/error.py +++ b/latest version/core/error.py @@ -101,6 +101,13 @@ class NoAccess(ArcError): super().__init__(message, error_code, api_error_code, extra_data, status) +class LowVersion(ArcError): + '''版本过低''' + + def __init__(self, message=None, error_code=5, api_error_code=-999, extra_data=None, status=403) -> None: + super().__init__(message, error_code, api_error_code, extra_data, status) + + class Timeout(ArcError): '''超时''' pass diff --git a/latest version/core/init.py b/latest version/core/init.py index efef5a5..a8152a9 100644 --- a/latest version/core/init.py +++ b/latest version/core/init.py @@ -6,6 +6,7 @@ from shutil import copy, copy2 from time import time from traceback import format_exc +from core.bundle import BundleParser from core.config_manager import Config from core.constant import ARCAEA_LOG_DATBASE_VERSION, ARCAEA_SERVER_VERSION from core.course import Course @@ -331,14 +332,27 @@ class FileChecker: DownloadList.initialize_cache() if not Config.SONG_FILE_HASH_PRE_CALCULATE: self.logger.info('Song file hash pre-calculate is disabled.') - self.logger.info('Complete!') + self.logger.info('Song data initialization is complete!') except Exception as e: self.logger.error(format_exc()) - self.logger.warning('Initialization error!') + self.logger.warning('Song data initialization error!') + f = False + return f + + def check_content_bundle(self) -> bool: + '''检查 content bundle 有关文件并初始化缓存''' + f = self.check_folder(Config.CONTENT_BUNDLE_FOLDER_PATH) + self.logger.info("Start to initialize content bundle data...") + try: + BundleParser() + self.logger.info('Content bundle data initialization is complete!') + except Exception as e: + self.logger.error(format_exc()) + self.logger.warning('Content bundle data initialization error!') f = False return f def check_before_run(self) -> bool: '''运行前检查，返回布尔值''' MemoryDatabase() # 初始化内存数据库 - return self.check_song_file() & self.check_update_database() + return self.check_song_file() and self.check_content_bundle() and self.check_update_database() diff --git a/latest version/core/operation.py b/latest version/core/operation.py index 3b6006d..741d4af 100644 --- a/latest version/core/operation.py +++ b/latest version/core/operation.py @@ -1,3 +1,4 @@ +from .bundle import BundleParser from .constant import Constant from .download import DownloadList from .error import NoData @@ -74,6 +75,16 @@ class RefreshSongFileCache(BaseOperation): DownloadList.initialize_cache() +class RefreshBundleCache(BaseOperation): + ''' + 刷新 bundle 缓存 + ''' + _name = 'refresh_content_bundle_cache' + + def run(self): + BundleParser().re_init() + + class SaveUpdateScore(BaseOperation): ''' 云存档更新成绩，是覆盖式更新 diff --git a/latest version/core/sql.py b/latest version/core/sql.py index a19088b..ee4388f 100644 --- a/latest version/core/sql.py +++ b/latest version/core/sql.py @@ -440,6 +440,8 @@ class MemoryDatabase: self.c.execute('''PRAGMA synchronous = 0''') self.c.execute('''create table if not exists download_token(user_id int, song_id text,file_name text,token text,time int,primary key(user_id, song_id, file_name));''') + self.c.execute('''create table if not exists bundle_download_token(token text primary key, + file_path text, time int, device_id text);''') self.c.execute( '''create index if not exists download_token_1 on download_token (song_id, file_name);''') self.conn.commit() diff --git a/latest version/main.py b/latest version/main.py index f05cc30..8f84d60 100644 --- a/latest version/main.py +++ b/latest version/main.py @@ -27,6 +27,7 @@ import server import web.index import web.login # import webapi +from core.bundle import BundleDownload from core.constant import Constant from core.download import UserDownload from core.error import ArcError, NoAccess, RateLimit @@ -81,7 +82,8 @@ def download(file_path): x.song_id, x.file_name = file_path.split('/', 1) x.select_for_check() if x.is_limited: - raise RateLimit('You have reached the download limit.', 903) + raise RateLimit( + f'User `{x.user.user_id}` has reached the download limit.', 903) if not x.is_valid: raise NoAccess('Expired token.') x.download_hit() @@ -99,6 +101,26 @@ def download(file_path): return error_return() +@app.route('/bundle_download/', methods=['GET']) # 热更新下载 +def bundle_download(token: str): + with Connect(in_memory=True) as c_m: + try: + file_path = BundleDownload(c_m).get_path_by_token( + token, request.remote_addr) + if Config.DOWNLOAD_USE_NGINX_X_ACCEL_REDIRECT: + # nginx X-Accel-Redirect + response = make_response() + response.headers['Content-Type'] = 'application/octet-stream' + response.headers['X-Accel-Redirect'] = Config.NGINX_X_ACCEL_REDIRECT_PREFIX + file_path + return response + return send_from_directory(Constant.CONTENT_BUNDLE_FOLDER_PATH, file_path, as_attachment=True, conditional=True) + except ArcError as e: + if Config.ALLOW_WARNING_LOG: + app.logger.warning(format_exc()) + return error_return(e) + return error_return() + + if Config.DEPLOY_MODE == 'waitress': # 给waitress加个日志 @app.after_request @@ -124,7 +146,7 @@ def tcp_server_run(): elif Config.DEPLOY_MODE == 'waitress': # waitress WSGI server import logging - from waitress import serve + from waitress import serve # type: ignore logger = logging.getLogger('waitress') logger.setLevel(logging.INFO) serve(app, host=Config.HOST, port=Config.PORT) diff --git a/latest version/server/func.py b/latest version/server/func.py index 5db87e7..061ad93 100644 --- a/latest version/server/func.py +++ b/latest version/server/func.py @@ -3,8 +3,9 @@ from traceback import format_exc from flask import current_app, g, jsonify +from core.bundle import BundleParser from core.config_manager import Config -from core.error import ArcError, NoAccess +from core.error import ArcError, LowVersion, NoAccess has_arc_hash = False try: @@ -22,7 +23,9 @@ def error_return(e: ArcError = default_error): # 错误返回 # -4 您的账号已在别处登录 # -3 无法连接至服务器 # 2 Arcaea服务器正在维护 + # 5 请更新 Arcaea 到最新版本 # 9 新版本请等待几分钟 + # 11 有游戏内容需要更新，即将返回主界面 # 100 无法在此ip地址下登录游戏 # 101 用户名占用 # 102 电子邮箱已注册 @@ -41,7 +44,7 @@ def error_return(e: ArcError = default_error): # 错误返回 # 150 非常抱歉您已被限制使用此功能 # 151 目前无法使用此功能 # 160 账户未邮箱认证，请检查邮箱 - # 161 账户认证过期，请重新注册 + # 161 账户认证过期，请重新注册 # 401 用户不存在 # 403 无法连接至服务器 # 501 502 -6 此物品目前无法获取 @@ -65,7 +68,7 @@ def error_return(e: ArcError = default_error): # 错误返回 # 9803 下载已取消 # 9905 没有在云端发现任何数据 # 9907 更新数据时发生了问题 - # 9908 服务器只支持最新的版本，请更新Arcaea + # 9908 服务器只支持最新的版本，请更新 Arcaea # 其它 发生未知错误 r = {"success": False, "error_code": e.error_code} if e.extra_data: @@ -106,7 +109,10 @@ def header_check(request) -> ArcError: headers = request.headers if Config.ALLOW_APPVERSION: # 版本检查 if 'AppVersion' not in headers or headers['AppVersion'] not in Config.ALLOW_APPVERSION: - return NoAccess('Invalid app version', 5) + return LowVersion('Invalid app version', 5) + + if 'ContentBundle' in headers and headers['ContentBundle'] != BundleParser.max_bundle_version[headers['AppVersion']]: + return LowVersion('Invalid content bundle version', 11) if has_arc_hash and not ArcHashChecker(request).check(): return NoAccess('Invalid request') diff --git a/latest version/server/others.py b/latest version/server/others.py index 27aac73..f433d36 100644 --- a/latest version/server/others.py +++ b/latest version/server/others.py @@ -4,6 +4,7 @@ from urllib.parse import parse_qs, urlparse from flask import Blueprint, jsonify, request from werkzeug.datastructures import ImmutableMultiDict +from core.bundle import BundleDownload from core.download import DownloadList from core.error import RateLimit from core.sql import Connect @@ -26,26 +27,19 @@ def game_info(): return success_return(GameInfo().to_dict()) -# @bp.route('/game/content_bundle', methods=['GET']) # 热更新 -# def game_content_bundle(): -# app_version = request.headers.get('AppVersion') -# bundle_version = request.headers.get('ContentBundle') -# import os -# if bundle_version != '5.4.0': -# r = {'orderedResults': [ -# { -# 'appVersion': '5.4.0', -# 'contentBundleVersion': '5.4.0', -# 'jsonUrl': 'http://192.168.0.110/bundle_download/bundle.json', -# 'jsonSize': os.path.getsize('./database/bundle/bundle.json'), -# 'bundleUrl': 'http://192.168.0.110/bundle_download/bundle', -# 'bundleSize': os.path.getsize('./database/bundle/bundle') -# }, -# ] -# } -# else: -# r = {} -# return success_return(r) +@bp.route('/game/content_bundle', methods=['GET']) # 热更新 +@arc_try +def game_content_bundle(): + # error code 5, 9 work + app_version = request.headers.get('AppVersion') + bundle_version = request.headers.get('ContentBundle') + device_id = request.headers.get('DeviceId') + with Connect(in_memory=True) as c_m: + x = BundleDownload(c_m) + x.set_client_info(app_version, bundle_version, device_id) + return success_return({ + 'orderedResults': x.get_bundle_list() + }) @bp.route('/serve/download/me/song', methods=['GET']) # 歌曲下载 diff --git a/latest version/templates/web/updatedatabase.html b/latest version/templates/web/updatedatabase.html index ddc48dd..9b1764a 100644 --- a/latest version/templates/web/updatedatabase.html +++ b/latest version/templates/web/updatedatabase.html @@ -13,7 +13,8 @@ 这里可以将旧版本的数据库同步到新版本的数据库，并刷新用户拥有的全角色列表。
可上传文件: arcaea_database.db
新数据库不存在的数据会被添加，存在的重复数据也会被改变。

- Here you can synchronize the old version of the database to the new version of the database and refresh the list of full + Here you can synchronize the old version of the database to the new version of the database and refresh the list of + full characters owned by players.
Uploadable files: arcaea_database.db
Data that does not exist in the new database will be added and the existing duplicate data will also be changed. @@ -40,4 +41,16 @@
Here you can refresh the ratings of all scores in the database. The purpose is to deal with the updating of songs' chart consts.
+
+
+
+
Refresh content bundles
+
+ + +
这里可以刷新储存在内存中的 bundle 文件夹下所有热更新文件的信息。目的是应对热更新文件的修改。
+
Here you can refresh the information of all the files stored in the bundle folder in memory. + The purpose + is to deal with the updating of content bundles.
+
{% endblock %} \ No newline at end of file diff --git a/latest version/web/index.py b/latest version/web/index.py index 456a063..d908402 100644 --- a/latest version/web/index.py +++ b/latest version/web/index.py @@ -2,7 +2,7 @@ import os import time from core.init import FileChecker -from core.operation import RefreshAllScoreRating, RefreshSongFileCache, SaveUpdateScore, UnlockUserItem, DeleteUserScore +from core.operation import RefreshAllScoreRating, RefreshBundleCache, RefreshSongFileCache, SaveUpdateScore, UnlockUserItem, DeleteUserScore from core.rank import RankList from core.sql import Connect from core.user import User @@ -299,6 +299,18 @@ def update_song_hash(): return render_template('web/updatedatabase.html') +@bp.route('/updatedatabase/refreshsbundle', methods=['POST']) +@login_required +def update_content_bundle(): + # 更新 bundle + try: + RefreshBundleCache().run() + flash('数据刷新成功 Success refresh data.') + except: + flash('Something error!') + return render_template('web/updatedatabase.html') + + @bp.route('/updatedatabase/refreshsongrating', methods=['POST']) @login_required def update_song_rating():