Bennett/Arcaea-server
1
0
Fork 0
mirror of https://github.com/Lost-MSth/Arcaea-server.git synced 2026-02-12 02:57:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Enhance] Content bundle

Browse Source 
- Add support for content bundles
- For Arcaea 5.4.0 play store version and iOS version
This commit is contained in:
Lost-MSth
2024-03-11 16:31:21 +08:00
parent d65cc3bcbe
commit 8f66b90912
12 changed files with 313 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
latest version/core/bundle.py Normal file
View File

@@ -0,0 +1,191 @@
import json
import os
from time import time
from flask import url_for
from .constant import Constant
from .error import NoAccess, RateLimit
from .limiter import ArcLimiter
class ContentBundle:
def __init__(self) -> None:
self.version: str = None
self.prev_version: str = None
self.app_version: str = None
self.uuid: str = None
self.json_size: int = None
self.bundle_size: int = None
self.json_path: str = None # relative path
self.bundle_path: str = None # relative path
self.json_url: str = None
self.bundle_url: str = None
@staticmethod
def parse_version(version: str) -> tuple:
try:
r = tuple(map(int, version.split('.')))
except AttributeError:
r = (0, 0, 0)
return r
@property
def version_tuple(self) -> tuple:
return self.parse_version(self.version)
@classmethod
def from_json(cls, json_data: dict) -> 'ContentBundle':
x = cls()
x.version = json_data['versionNumber']
x.prev_version = json_data['previousVersionNumber']
x.app_version = json_data['applicationVersionNumber']
x.uuid = json_data['uuid']
return x
def to_dict(self) -> dict:
r = {
'contentBundleVersion': self.version,
'appVersion': self.app_version,
'jsonSize': self.json_size,
'bundleSize': self.bundle_size,
}
if self.json_url and self.bundle_url:
r['jsonUrl'] = self.json_url
r['bundleUrl'] = self.bundle_url
return r
def calculate_size(self) -> None:
self.json_size = os.path.getsize(os.path.join(
Constant.CONTENT_BUNDLE_FOLDER_PATH, self.json_path))
self.bundle_size = os.path.getsize(os.path.join(
Constant.CONTENT_BUNDLE_FOLDER_PATH, self.bundle_path))
class BundleParser:
# {app_version: [ List[ContentBundle] ]}
bundles: 'dict[str, list[ContentBundle]]' = {}
max_bundle_version: 'dict[str, str]' = {}
def __init__(self) -> None:
self.parse()
def re_init(self) -> None:
self.bundles.clear()
self.max_bundle_version.clear()
self.parse()
def parse(self) -> None:
for root, dirs, files in os.walk(Constant.CONTENT_BUNDLE_FOLDER_PATH):
for file in files:
if not file.endswith('.json'):
continue
json_path = os.path.join(root, file)
bundle_path = os.path.join(root, f'{file[:-5]}.cb')
with open(json_path, 'rb') as f:
data = json.load(f)
x = ContentBundle.from_json(data)
x.json_path = os.path.relpath(
json_path, Constant.CONTENT_BUNDLE_FOLDER_PATH)
x.bundle_path = os.path.relpath(
bundle_path, Constant.CONTENT_BUNDLE_FOLDER_PATH)
x.json_path = x.json_path.replace('\\', '/')
x.bundle_path = x.bundle_path.replace('\\', '/')
if not os.path.isfile(bundle_path):
raise FileNotFoundError(
f'Bundle file not found: {bundle_path}')
x.calculate_size()
if x.app_version not in self.bundles:
self.bundles[x.app_version] = []
self.bundles[x.app_version].append(x)
# sort by version
for k, v in self.bundles.items():
v.sort(key=lambda x: x.version_tuple)
self.max_bundle_version[k] = v[-1].version
class BundleDownload:
limiter = ArcLimiter(
Constant.BUNDLE_DOWNLOAD_TIMES_LIMIT, 'bundle_download')
def __init__(self, c_m=None):
self.c_m = c_m
self.client_app_version = None
self.client_bundle_version = None
self.device_id = None
def set_client_info(self, app_version: str, bundle_version: str, device_id: str = None) -> None:
self.client_app_version = app_version
self.client_bundle_version = bundle_version
self.device_id = device_id
def get_bundle_list(self) -> list:
bundles: 'list[ContentBundle]' = BundleParser.bundles.get(
self.client_app_version, [])
if not bundles:
return []
now = time()
if Constant.BUNDLE_DOWNLOAD_LINK_PREFIX:
prefix = Constant.BUNDLE_DOWNLOAD_LINK_PREFIX
if prefix[-1] != '/':
prefix += '/'
def url_func(x): return f'{prefix}{x}'
else:
def url_func(x): return url_for(
'bundle_download', token=x, _external=True)
sql_list = []
r = []
for x in bundles:
if x.version_tuple <= ContentBundle.parse_version(self.client_bundle_version):
continue
t1 = os.urandom(64).hex()
t2 = os.urandom(64).hex()
x.json_url = url_func(t1)
x.bundle_url = url_func(t2)
sql_list.append((t1, x.json_path, now, self.device_id))
sql_list.append((t2, x.bundle_path, now, self.device_id))
r.append(x.to_dict())
if not sql_list:
return []
self.c_m.executemany(
'''insert into bundle_download_token values (?, ?, ?, ?)''', sql_list)
return r
def get_path_by_token(self, token: str, ip: str) -> str:
r = self.c_m.execute(
'''select file_path, time, device_id from bundle_download_token where token = ?''', (token,)).fetchone()
if not r:
raise NoAccess('Invalid token.', status=403)
file_path, create_time, device_id = r
if time() - create_time > Constant.BUNDLE_DOWNLOAD_TIME_GAP_LIMIT:
raise NoAccess('Expired token.', status=403)
if file_path.endswith('.cb') and not self.limiter.hit(ip):
raise RateLimit(
f'Too many content bundle downloads, IP: {ip}, DeviceID: {device_id}', status=429)
return file_path

7
latest version/core/config_manager.py
View File

@@ -42,7 +42,8 @@ class Config:
API_TOKEN = ''
DOWNLOAD_LINK_PREFIX = ''
DOWNLOAD_LINK_PREFIX = '' # http(s)://host(:port)/download/
BUNDLE_DOWNLOAD_LINK_PREFIX = '' # http(s)://host(:port)/bundle_download/
DOWNLOAD_USE_NGINX_X_ACCEL_REDIRECT = False
NGINX_X_ACCEL_REDIRECT_PREFIX = '/nginx_download/'
@@ -52,6 +53,9 @@ class Config:
DOWNLOAD_FORBID_WHEN_NO_ITEM = False
BUNDLE_DOWNLOAD_TIMES_LIMIT = '100/60 minutes'
BUNDLE_DOWNLOAD_TIME_GAP_LIMIT = 3000
LOGIN_DEVICE_NUMBER_LIMIT = 1
ALLOW_LOGIN_SAME_DEVICE = False
ALLOW_BAN_MULTIDEVICE_USER_AUTO = True
@@ -84,6 +88,7 @@ class Config:
WORLD_MAP_FOLDER_PATH = './database/map/'
SONG_FILE_FOLDER_PATH = './database/songs/'
SONGLIST_FILE_PATH = './database/songs/songlist'
CONTENT_BUNDLE_FOLDER_PATH = './database/bundle/'
SQLITE_DATABASE_PATH = './database/arcaea_database.db'
SQLITE_DATABASE_BACKUP_FOLDER_PATH = './database/backup/'
DATABASE_INIT_PATH = './database/init/'

4
latest version/core/constant.py
View File

@@ -43,6 +43,7 @@ class Constant:
WORLD_MAP_FOLDER_PATH = Config.WORLD_MAP_FOLDER_PATH
SONG_FILE_FOLDER_PATH = Config.SONG_FILE_FOLDER_PATH
SONGLIST_FILE_PATH = Config.SONGLIST_FILE_PATH
CONTENT_BUNDLE_FOLDER_PATH = Config.CONTENT_BUNDLE_FOLDER_PATH
SQLITE_DATABASE_PATH = Config.SQLITE_DATABASE_PATH
SQLITE_LOG_DATABASE_PATH = Config.SQLITE_LOG_DATABASE_PATH
SQLITE_DATABASE_DELETED_PATH = Config.SQLITE_DATABASE_DELETED_PATH
@@ -50,6 +51,9 @@ class Constant:
DOWNLOAD_TIMES_LIMIT = Config.DOWNLOAD_TIMES_LIMIT
DOWNLOAD_TIME_GAP_LIMIT = Config.DOWNLOAD_TIME_GAP_LIMIT
DOWNLOAD_LINK_PREFIX = Config.DOWNLOAD_LINK_PREFIX
BUNDLE_DOWNLOAD_TIMES_LIMIT = Config.BUNDLE_DOWNLOAD_TIMES_LIMIT
BUNDLE_DOWNLOAD_TIME_GAP_LIMIT = Config.BUNDLE_DOWNLOAD_TIME_GAP_LIMIT
BUNDLE_DOWNLOAD_LINK_PREFIX = Config.BUNDLE_DOWNLOAD_LINK_PREFIX
LINKPLAY_UNLOCK_LENGTH = 512 # Units: bytes
LINKPLAY_TIMEOUT = 5 # Units: seconds

7
latest version/core/error.py
View File

@@ -101,6 +101,13 @@ class NoAccess(ArcError):
super().__init__(message, error_code, api_error_code, extra_data, status)
class LowVersion(ArcError):
'''版本过低'''
def __init__(self, message=None, error_code=5, api_error_code=-999, extra_data=None, status=403) -> None:
super().__init__(message, error_code, api_error_code, extra_data, status)
class Timeout(ArcError):
'''超时'''
pass

20
latest version/core/init.py
View File

@@ -6,6 +6,7 @@ from shutil import copy, copy2
from time import time
from traceback import format_exc
from core.bundle import BundleParser
from core.config_manager import Config
from core.constant import ARCAEA_LOG_DATBASE_VERSION, ARCAEA_SERVER_VERSION
from core.course import Course
@@ -331,14 +332,27 @@ class FileChecker:
DownloadList.initialize_cache()
if not Config.SONG_FILE_HASH_PRE_CALCULATE:
self.logger.info('Song file hash pre-calculate is disabled.')
self.logger.info('Complete!')
self.logger.info('Song data initialization is complete!')
except Exception as e:
self.logger.error(format_exc())
self.logger.warning('Initialization error!')
self.logger.warning('Song data initialization error!')
f = False
return f
def check_content_bundle(self) -> bool:
'''检查 content bundle 有关文件并初始化缓存'''
f = self.check_folder(Config.CONTENT_BUNDLE_FOLDER_PATH)
self.logger.info("Start to initialize content bundle data...")
try:
BundleParser()
self.logger.info('Content bundle data initialization is complete!')
except Exception as e:
self.logger.error(format_exc())
self.logger.warning('Content bundle data initialization error!')
f = False
return f
def check_before_run(self) -> bool:
'''运行前检查，返回布尔值'''
MemoryDatabase() # 初始化内存数据库
return self.check_song_file() & self.check_update_database()
return self.check_song_file() and self.check_content_bundle() and self.check_update_database()

11
latest version/core/operation.py
View File

@@ -1,3 +1,4 @@
from .bundle import BundleParser
from .constant import Constant
from .download import DownloadList
from .error import NoData
@@ -74,6 +75,16 @@ class RefreshSongFileCache(BaseOperation):
DownloadList.initialize_cache()
class RefreshBundleCache(BaseOperation):
'''
刷新 bundle 缓存
'''
_name = 'refresh_content_bundle_cache'
def run(self):
BundleParser().re_init()
class SaveUpdateScore(BaseOperation):
'''
云存档更新成绩，是覆盖式更新

2
latest version/core/sql.py
View File

@@ -440,6 +440,8 @@ class MemoryDatabase:
self.c.execute('''PRAGMA synchronous = 0''')
self.c.execute('''create table if not exists download_token(user_id int,
song_id text,file_name text,token text,time int,primary key(user_id, song_id, file_name));''')
self.c.execute('''create table if not exists bundle_download_token(token text primary key,
file_path text, time int, device_id text);''')
self.c.execute(
'''create index if not exists download_token_1 on download_token (song_id, file_name);''')
self.conn.commit()