[Enhance] Content bundle

- Add support for content bundles
- For Arcaea 5.4.0 play store version and iOS version

latest version/core/bundle.py

@@ -0,0 +1,191 @@
+import json
+import os
+from time import time
+
+from flask import url_for
+
+from .constant import Constant
+from .error import NoAccess, RateLimit
+from .limiter import ArcLimiter
+
+
+class ContentBundle:
+
+    def __init__(self) -> None:
+        self.version: str = None
+        self.prev_version: str = None
+        self.app_version: str = None
+        self.uuid: str = None
+
+        self.json_size: int = None
+        self.bundle_size: int = None
+        self.json_path: str = None  # relative path
+        self.bundle_path: str = None  # relative path
+
+        self.json_url: str = None
+        self.bundle_url: str = None
+
+    @staticmethod
+    def parse_version(version: str) -> tuple:
+        try:
+            r = tuple(map(int, version.split('.')))
+        except AttributeError:
+            r = (0, 0, 0)
+        return r
+
+    @property
+    def version_tuple(self) -> tuple:
+        return self.parse_version(self.version)
+
+    @classmethod
+    def from_json(cls, json_data: dict) -> 'ContentBundle':
+        x = cls()
+        x.version = json_data['versionNumber']
+        x.prev_version = json_data['previousVersionNumber']
+        x.app_version = json_data['applicationVersionNumber']
+        x.uuid = json_data['uuid']
+        return x
+
+    def to_dict(self) -> dict:
+        r = {
+            'contentBundleVersion': self.version,
+            'appVersion': self.app_version,
+            'jsonSize': self.json_size,
+            'bundleSize': self.bundle_size,
+        }
+        if self.json_url and self.bundle_url:
+            r['jsonUrl'] = self.json_url
+            r['bundleUrl'] = self.bundle_url
+        return r
+
+    def calculate_size(self) -> None:
+        self.json_size = os.path.getsize(os.path.join(
+            Constant.CONTENT_BUNDLE_FOLDER_PATH, self.json_path))
+        self.bundle_size = os.path.getsize(os.path.join(
+            Constant.CONTENT_BUNDLE_FOLDER_PATH, self.bundle_path))
+
+
+class BundleParser:
+
+    # {app_version: [ List[ContentBundle] ]}
+    bundles: 'dict[str, list[ContentBundle]]' = {}
+    max_bundle_version: 'dict[str, str]' = {}
+
+    def __init__(self) -> None:
+        self.parse()
+
+    def re_init(self) -> None:
+        self.bundles.clear()
+        self.max_bundle_version.clear()
+        self.parse()
+
+    def parse(self) -> None:
+        for root, dirs, files in os.walk(Constant.CONTENT_BUNDLE_FOLDER_PATH):
+            for file in files:
+                if not file.endswith('.json'):
+                    continue
+
+                json_path = os.path.join(root, file)
+                bundle_path = os.path.join(root, f'{file[:-5]}.cb')
+
+                with open(json_path, 'rb') as f:
+                    data = json.load(f)
+
+                x = ContentBundle.from_json(data)
+
+                x.json_path = os.path.relpath(
+                    json_path, Constant.CONTENT_BUNDLE_FOLDER_PATH)
+                x.bundle_path = os.path.relpath(
+                    bundle_path, Constant.CONTENT_BUNDLE_FOLDER_PATH)
+
+                x.json_path = x.json_path.replace('\\', '/')
+                x.bundle_path = x.bundle_path.replace('\\', '/')
+
+                if not os.path.isfile(bundle_path):
+                    raise FileNotFoundError(
+                        f'Bundle file not found: {bundle_path}')
+                x.calculate_size()
+
+                if x.app_version not in self.bundles:
+                    self.bundles[x.app_version] = []
+                self.bundles[x.app_version].append(x)
+
+        # sort by version
+        for k, v in self.bundles.items():
+            v.sort(key=lambda x: x.version_tuple)
+            self.max_bundle_version[k] = v[-1].version
+
+
+class BundleDownload:
+
+    limiter = ArcLimiter(
+        Constant.BUNDLE_DOWNLOAD_TIMES_LIMIT, 'bundle_download')
+
+    def __init__(self, c_m=None):
+        self.c_m = c_m
+
+        self.client_app_version = None
+        self.client_bundle_version = None
+        self.device_id = None
+
+    def set_client_info(self, app_version: str, bundle_version: str, device_id: str = None) -> None:
+        self.client_app_version = app_version
+        self.client_bundle_version = bundle_version
+        self.device_id = device_id
+
+    def get_bundle_list(self) -> list:
+        bundles: 'list[ContentBundle]' = BundleParser.bundles.get(
+            self.client_app_version, [])
+        if not bundles:
+            return []
+
+        now = time()
+
+        if Constant.BUNDLE_DOWNLOAD_LINK_PREFIX:
+            prefix = Constant.BUNDLE_DOWNLOAD_LINK_PREFIX
+            if prefix[-1] != '/':
+                prefix += '/'
+
+            def url_func(x): return f'{prefix}{x}'
+        else:
+            def url_func(x): return url_for(
+                'bundle_download', token=x, _external=True)
+
+        sql_list = []
+        r = []
+        for x in bundles:
+            if x.version_tuple <= ContentBundle.parse_version(self.client_bundle_version):
+                continue
+            t1 = os.urandom(64).hex()
+            t2 = os.urandom(64).hex()
+
+            x.json_url = url_func(t1)
+            x.bundle_url = url_func(t2)
+
+            sql_list.append((t1, x.json_path, now, self.device_id))
+            sql_list.append((t2, x.bundle_path, now, self.device_id))
+            r.append(x.to_dict())
+
+        if not sql_list:
+            return []
+
+        self.c_m.executemany(
+            '''insert into bundle_download_token values (?, ?, ?, ?)''', sql_list)
+
+        return r
+
+    def get_path_by_token(self, token: str, ip: str) -> str:
+        r = self.c_m.execute(
+            '''select file_path, time, device_id from bundle_download_token where token = ?''', (token,)).fetchone()
+        if not r:
+            raise NoAccess('Invalid token.', status=403)
+        file_path, create_time, device_id = r
+
+        if time() - create_time > Constant.BUNDLE_DOWNLOAD_TIME_GAP_LIMIT:
+            raise NoAccess('Expired token.', status=403)
+
+        if file_path.endswith('.cb') and not self.limiter.hit(ip):
+            raise RateLimit(
+                f'Too many content bundle downloads, IP: {ip}, DeviceID: {device_id}', status=429)
+
+        return file_path

latest version/core/config_manager.py

@@ -42,7 +42,8 @@ class Config:
 
     API_TOKEN = ''
 
-    DOWNLOAD_LINK_PREFIX = ''
+    DOWNLOAD_LINK_PREFIX = ''  # http(s)://host(:port)/download/
+    BUNDLE_DOWNLOAD_LINK_PREFIX = ''  # http(s)://host(:port)/bundle_download/
 
     DOWNLOAD_USE_NGINX_X_ACCEL_REDIRECT = False
     NGINX_X_ACCEL_REDIRECT_PREFIX = '/nginx_download/'
@@ -52,6 +53,9 @@ class Config:
 
     DOWNLOAD_FORBID_WHEN_NO_ITEM = False
 
+    BUNDLE_DOWNLOAD_TIMES_LIMIT = '100/60 minutes'
+    BUNDLE_DOWNLOAD_TIME_GAP_LIMIT = 3000
+
     LOGIN_DEVICE_NUMBER_LIMIT = 1
     ALLOW_LOGIN_SAME_DEVICE = False
     ALLOW_BAN_MULTIDEVICE_USER_AUTO = True
@@ -84,6 +88,7 @@ class Config:
     WORLD_MAP_FOLDER_PATH = './database/map/'
     SONG_FILE_FOLDER_PATH = './database/songs/'
     SONGLIST_FILE_PATH = './database/songs/songlist'
+    CONTENT_BUNDLE_FOLDER_PATH = './database/bundle/'
     SQLITE_DATABASE_PATH = './database/arcaea_database.db'
     SQLITE_DATABASE_BACKUP_FOLDER_PATH = './database/backup/'
     DATABASE_INIT_PATH = './database/init/'

latest version/core/constant.py

@@ -43,6 +43,7 @@ class Constant:
     WORLD_MAP_FOLDER_PATH = Config.WORLD_MAP_FOLDER_PATH
     SONG_FILE_FOLDER_PATH = Config.SONG_FILE_FOLDER_PATH
     SONGLIST_FILE_PATH = Config.SONGLIST_FILE_PATH
+    CONTENT_BUNDLE_FOLDER_PATH = Config.CONTENT_BUNDLE_FOLDER_PATH
     SQLITE_DATABASE_PATH = Config.SQLITE_DATABASE_PATH
     SQLITE_LOG_DATABASE_PATH = Config.SQLITE_LOG_DATABASE_PATH
     SQLITE_DATABASE_DELETED_PATH = Config.SQLITE_DATABASE_DELETED_PATH
@@ -50,6 +51,9 @@ class Constant:
     DOWNLOAD_TIMES_LIMIT = Config.DOWNLOAD_TIMES_LIMIT
     DOWNLOAD_TIME_GAP_LIMIT = Config.DOWNLOAD_TIME_GAP_LIMIT
     DOWNLOAD_LINK_PREFIX = Config.DOWNLOAD_LINK_PREFIX
+    BUNDLE_DOWNLOAD_TIMES_LIMIT = Config.BUNDLE_DOWNLOAD_TIMES_LIMIT
+    BUNDLE_DOWNLOAD_TIME_GAP_LIMIT = Config.BUNDLE_DOWNLOAD_TIME_GAP_LIMIT
+    BUNDLE_DOWNLOAD_LINK_PREFIX = Config.BUNDLE_DOWNLOAD_LINK_PREFIX
 
     LINKPLAY_UNLOCK_LENGTH = 512  # Units: bytes
     LINKPLAY_TIMEOUT = 5  # Units: seconds

latest version/core/error.py

@@ -101,6 +101,13 @@ class NoAccess(ArcError):
         super().__init__(message, error_code, api_error_code, extra_data, status)
 
 
+class LowVersion(ArcError):
+    '''版本过低'''
+
+    def __init__(self, message=None, error_code=5, api_error_code=-999, extra_data=None, status=403) -> None:
+        super().__init__(message, error_code, api_error_code, extra_data, status)
+
+
 class Timeout(ArcError):
     '''超时'''
     pass

latest version/core/init.py

@@ -6,6 +6,7 @@ from shutil import copy, copy2
 from time import time
 from traceback import format_exc
 
+from core.bundle import BundleParser
 from core.config_manager import Config
 from core.constant import ARCAEA_LOG_DATBASE_VERSION, ARCAEA_SERVER_VERSION
 from core.course import Course
@@ -331,14 +332,27 @@ class FileChecker:
             DownloadList.initialize_cache()
             if not Config.SONG_FILE_HASH_PRE_CALCULATE:
                 self.logger.info('Song file hash pre-calculate is disabled.')
-            self.logger.info('Complete!')
+            self.logger.info('Song data initialization is complete!')
         except Exception as e:
             self.logger.error(format_exc())
-            self.logger.warning('Initialization error!')
+            self.logger.warning('Song data initialization error!')
+            f = False
+        return f
+
+    def check_content_bundle(self) -> bool:
+        '''检查 content bundle 有关文件并初始化缓存'''
+        f = self.check_folder(Config.CONTENT_BUNDLE_FOLDER_PATH)
+        self.logger.info("Start to initialize content bundle data...")
+        try:
+            BundleParser()
+            self.logger.info('Content bundle data initialization is complete!')
+        except Exception as e:
+            self.logger.error(format_exc())
+            self.logger.warning('Content bundle data initialization error!')
             f = False
         return f
 
     def check_before_run(self) -> bool:
         '''运行前检查，返回布尔值'''
         MemoryDatabase()  # 初始化内存数据库
-        return self.check_song_file() & self.check_update_database()
+        return self.check_song_file() and self.check_content_bundle() and self.check_update_database()

latest version/core/operation.py

@@ -1,3 +1,4 @@
+from .bundle import BundleParser
 from .constant import Constant
 from .download import DownloadList
 from .error import NoData
@@ -74,6 +75,16 @@ class RefreshSongFileCache(BaseOperation):
         DownloadList.initialize_cache()
 
 
+class RefreshBundleCache(BaseOperation):
+    '''
+        刷新 bundle 缓存
+    '''
+    _name = 'refresh_content_bundle_cache'
+
+    def run(self):
+        BundleParser().re_init()
+
+
 class SaveUpdateScore(BaseOperation):
     '''
         云存档更新成绩，是覆盖式更新

latest version/core/sql.py

@@ -440,6 +440,8 @@ class MemoryDatabase:
         self.c.execute('''PRAGMA synchronous = 0''')
         self.c.execute('''create table if not exists download_token(user_id int,
         song_id text,file_name text,token text,time int,primary key(user_id, song_id, file_name));''')
+        self.c.execute('''create table if not exists bundle_download_token(token text primary key,
+                       file_path text, time int, device_id text);''')
         self.c.execute(
             '''create index if not exists download_token_1 on download_token (song_id, file_name);''')
         self.conn.commit()

latest version/main.py

@@ -27,6 +27,7 @@ import server
 import web.index
 import web.login
 # import webapi
+from core.bundle import BundleDownload
 from core.constant import Constant
 from core.download import UserDownload
 from core.error import ArcError, NoAccess, RateLimit
@@ -81,7 +82,8 @@ def download(file_path):
             x.song_id, x.file_name = file_path.split('/', 1)
             x.select_for_check()
             if x.is_limited:
-                raise RateLimit('You have reached the download limit.', 903)
+                raise RateLimit(
+                    f'User `{x.user.user_id}` has reached the download limit.', 903)
             if not x.is_valid:
                 raise NoAccess('Expired token.')
             x.download_hit()
@@ -99,6 +101,26 @@ def download(file_path):
     return error_return()
 
 
+@app.route('/bundle_download/<string:token>', methods=['GET'])  # 热更新下载
+def bundle_download(token: str):
+    with Connect(in_memory=True) as c_m:
+        try:
+            file_path = BundleDownload(c_m).get_path_by_token(
+                token, request.remote_addr)
+            if Config.DOWNLOAD_USE_NGINX_X_ACCEL_REDIRECT:
+                # nginx X-Accel-Redirect
+                response = make_response()
+                response.headers['Content-Type'] = 'application/octet-stream'
+                response.headers['X-Accel-Redirect'] = Config.NGINX_X_ACCEL_REDIRECT_PREFIX + file_path
+                return response
+            return send_from_directory(Constant.CONTENT_BUNDLE_FOLDER_PATH, file_path, as_attachment=True, conditional=True)
+        except ArcError as e:
+            if Config.ALLOW_WARNING_LOG:
+                app.logger.warning(format_exc())
+            return error_return(e)
+    return error_return()
+
+
 if Config.DEPLOY_MODE == 'waitress':
     # 给waitress加个日志
     @app.after_request
@@ -124,7 +146,7 @@ def tcp_server_run():
     elif Config.DEPLOY_MODE == 'waitress':
         # waitress WSGI server
         import logging
-        from waitress import serve
+        from waitress import serve  # type: ignore
         logger = logging.getLogger('waitress')
         logger.setLevel(logging.INFO)
         serve(app, host=Config.HOST, port=Config.PORT)

latest version/server/func.py

@@ -3,8 +3,9 @@ from traceback import format_exc
 
 from flask import current_app, g, jsonify
 
+from core.bundle import BundleParser
 from core.config_manager import Config
-from core.error import ArcError, NoAccess
+from core.error import ArcError, LowVersion, NoAccess
 
 has_arc_hash = False
 try:
@@ -22,7 +23,9 @@ def error_return(e: ArcError = default_error):  # 错误返回
     # -4 您的账号已在别处登录
     # -3 无法连接至服务器
     # 2 Arcaea服务器正在维护
+    # 5 请更新 Arcaea 到最新版本
     # 9 新版本请等待几分钟
+    # 11 有游戏内容需要更新，即将返回主界面
     # 100 无法在此ip地址下登录游戏
     # 101 用户名占用
     # 102 电子邮箱已注册
@@ -41,7 +44,7 @@ def error_return(e: ArcError = default_error):  # 错误返回
     # 150 非常抱歉您已被限制使用此功能
     # 151 目前无法使用此功能
     # 160 账户未邮箱认证，请检查邮箱
-    # 161  账户认证过期，请重新注册
+    # 161 账户认证过期，请重新注册
     # 401 用户不存在
     # 403 无法连接至服务器
     # 501 502 -6 此物品目前无法获取
@@ -65,7 +68,7 @@ def error_return(e: ArcError = default_error):  # 错误返回
     # 9803 下载已取消
     # 9905 没有在云端发现任何数据
     # 9907 更新数据时发生了问题
-    # 9908 服务器只支持最新的版本，请更新Arcaea
+    # 9908 服务器只支持最新的版本，请更新 Arcaea
     # 其它 发生未知错误
     r = {"success": False, "error_code": e.error_code}
     if e.extra_data:
@@ -106,7 +109,10 @@ def header_check(request) -> ArcError:
     headers = request.headers
     if Config.ALLOW_APPVERSION:  # 版本检查
         if 'AppVersion' not in headers or headers['AppVersion'] not in Config.ALLOW_APPVERSION:
-            return NoAccess('Invalid app version', 5)
+            return LowVersion('Invalid app version', 5)
+
+        if 'ContentBundle' in headers and headers['ContentBundle'] != BundleParser.max_bundle_version[headers['AppVersion']]:
+            return LowVersion('Invalid content bundle version', 11)
 
     if has_arc_hash and not ArcHashChecker(request).check():
         return NoAccess('Invalid request')

latest version/server/others.py

@@ -4,6 +4,7 @@ from urllib.parse import parse_qs, urlparse
 from flask import Blueprint, jsonify, request
 from werkzeug.datastructures import ImmutableMultiDict
 
+from core.bundle import BundleDownload
 from core.download import DownloadList
 from core.error import RateLimit
 from core.sql import Connect
@@ -26,26 +27,19 @@ def game_info():
     return success_return(GameInfo().to_dict())
 
 
-# @bp.route('/game/content_bundle', methods=['GET'])  # 热更新
+@bp.route('/game/content_bundle', methods=['GET'])  # 热更新
-# def game_content_bundle():
+@arc_try
-#     app_version = request.headers.get('AppVersion')
+def game_content_bundle():
-#     bundle_version = request.headers.get('ContentBundle')
+    # error code 5, 9 work
-#     import os
+    app_version = request.headers.get('AppVersion')
-#     if bundle_version != '5.4.0':
+    bundle_version = request.headers.get('ContentBundle')
-#         r = {'orderedResults': [
+    device_id = request.headers.get('DeviceId')
-#             {
+    with Connect(in_memory=True) as c_m:
-#                 'appVersion': '5.4.0',
+        x = BundleDownload(c_m)
-#                 'contentBundleVersion': '5.4.0',
+        x.set_client_info(app_version, bundle_version, device_id)
-#                 'jsonUrl': 'http://192.168.0.110/bundle_download/bundle.json',
+        return success_return({
-#                 'jsonSize': os.path.getsize('./database/bundle/bundle.json'),
+            'orderedResults': x.get_bundle_list()
-#                 'bundleUrl': 'http://192.168.0.110/bundle_download/bundle',
+        })
-#                 'bundleSize': os.path.getsize('./database/bundle/bundle')
-#             },
-#         ]
-#         }
-#     else:
-#         r = {}
-#     return success_return(r)
 
 
 @bp.route('/serve/download/me/song', methods=['GET'])  # 歌曲下载

latest version/templates/web/updatedatabase.html

@@ -13,7 +13,8 @@
     这里可以将旧版本的数据库同步到新版本的数据库，并刷新用户拥有的全角色列表。<br />
     可上传文件: arcaea_database.db<br />
     新数据库不存在的数据会被添加，存在的重复数据也会被改变。<br /><br />
-    Here you can synchronize the old version of the database to the new version of the database and refresh the list of full
+    Here you can synchronize the old version of the database to the new version of the database and refresh the list of
+    full
     characters owned by players.<br />
     Uploadable files: arcaea_database.db<br />
     Data that does not exist in the new database will be added and the existing duplicate data will also be changed.
@@ -40,4 +41,16 @@
     <div class="content">Here you can refresh the ratings of all scores in the database. The purpose is to deal
         with the updating of songs' chart consts.</div>
 </form>
+<br />
+<hr />
+<form action="/web/updatedatabase/refreshsbundle" method="post">
+    <div class="title">Refresh content bundles</div>
+    <br />
+    <input type="submit" value="Refresh">
+
+    <div class="content">这里可以刷新储存在内存中的 bundle 文件夹下所有热更新文件的信息。目的是应对热更新文件的修改。</div>
+    <div class="content">Here you can refresh the information of all the files stored in the bundle folder in memory.
+        The purpose
+        is to deal with the updating of content bundles.</div>
+</form>
 {% endblock %}

latest version/web/index.py

@@ -2,7 +2,7 @@ import os
 import time
 
 from core.init import FileChecker
-from core.operation import RefreshAllScoreRating, RefreshSongFileCache, SaveUpdateScore, UnlockUserItem, DeleteUserScore
+from core.operation import RefreshAllScoreRating, RefreshBundleCache, RefreshSongFileCache, SaveUpdateScore, UnlockUserItem, DeleteUserScore
 from core.rank import RankList
 from core.sql import Connect
 from core.user import User
@@ -299,6 +299,18 @@ def update_song_hash():
     return render_template('web/updatedatabase.html')
 
 
+@bp.route('/updatedatabase/refreshsbundle', methods=['POST'])
+@login_required
+def update_content_bundle():
+    # 更新 bundle
+    try:
+        RefreshBundleCache().run()
+        flash('数据刷新成功 Success refresh data.')
+    except:
+        flash('Something error!')
+    return render_template('web/updatedatabase.html')
+
+
 @bp.route('/updatedatabase/refreshsongrating', methods=['POST'])
 @login_required
 def update_song_rating():