Bennett/Arcaea-server
1
0
Fork 1
mirror of https://github.com/Lost-MSth/Arcaea-server.git synced 2025-12-14 08:06:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Enhance] Content bundle

Browse Source 
- Add support for content bundles
- For Arcaea 5.4.0 play store version and iOS version
This commit is contained in:
Lost-MSth
2024-03-11 16:31:21 +08:00
parent d65cc3bcbe
commit 8f66b90912
12 changed files with 313 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
latest version/core/bundle.py Normal file
View File

@@ -0,0 +1,191 @@
import json
import os
from time import time
from flask import url_for
from .constant import Constant
from .error import NoAccess, RateLimit
from .limiter import ArcLimiter
class ContentBundle:
def __init__(self) -> None:
self.version: str = None
self.prev_version: str = None
self.app_version: str = None
self.uuid: str = None
self.json_size: int = None
self.bundle_size: int = None
self.json_path: str = None # relative path
self.bundle_path: str = None # relative path
self.json_url: str = None
self.bundle_url: str = None
@staticmethod
def parse_version(version: str) -> tuple:
try:
r = tuple(map(int, version.split('.')))
except AttributeError:
r = (0, 0, 0)
return r
@property
def version_tuple(self) -> tuple:
return self.parse_version(self.version)
@classmethod
def from_json(cls, json_data: dict) -> 'ContentBundle':
x = cls()
x.version = json_data['versionNumber']
x.prev_version = json_data['previousVersionNumber']
x.app_version = json_data['applicationVersionNumber']
x.uuid = json_data['uuid']
return x
def to_dict(self) -> dict:
r = {
'contentBundleVersion': self.version,
'appVersion': self.app_version,
'jsonSize': self.json_size,
'bundleSize': self.bundle_size,
}
if self.json_url and self.bundle_url:
r['jsonUrl'] = self.json_url
r['bundleUrl'] = self.bundle_url
return r
def calculate_size(self) -> None:
self.json_size = os.path.getsize(os.path.join(
Constant.CONTENT_BUNDLE_FOLDER_PATH, self.json_path))
self.bundle_size = os.path.getsize(os.path.join(
Constant.CONTENT_BUNDLE_FOLDER_PATH, self.bundle_path))
class BundleParser:
# {app_version: [ List[ContentBundle] ]}
bundles: 'dict[str, list[ContentBundle]]' = {}
max_bundle_version: 'dict[str, str]' = {}
def __init__(self) -> None:
self.parse()
def re_init(self) -> None:
self.bundles.clear()
self.max_bundle_version.clear()
self.parse()
def parse(self) -> None:
for root, dirs, files in os.walk(Constant.CONTENT_BUNDLE_FOLDER_PATH):
for file in files:
if not file.endswith('.json'):
continue
json_path = os.path.join(root, file)
bundle_path = os.path.join(root, f'{file[:-5]}.cb')
with open(json_path, 'rb') as f:
data = json.load(f)
x = ContentBundle.from_json(data)
x.json_path = os.path.relpath(
json_path, Constant.CONTENT_BUNDLE_FOLDER_PATH)
x.bundle_path = os.path.relpath(
bundle_path, Constant.CONTENT_BUNDLE_FOLDER_PATH)
x.json_path = x.json_path.replace('\\', '/')
x.bundle_path = x.bundle_path.replace('\\', '/')
if not os.path.isfile(bundle_path):
raise FileNotFoundError(
f'Bundle file not found: {bundle_path}')
x.calculate_size()
if x.app_version not in self.bundles:
self.bundles[x.app_version] = []
self.bundles[x.app_version].append(x)
# sort by version
for k, v in self.bundles.items():
v.sort(key=lambda x: x.version_tuple)
self.max_bundle_version[k] = v[-1].version
class BundleDownload:
limiter = ArcLimiter(
Constant.BUNDLE_DOWNLOAD_TIMES_LIMIT, 'bundle_download')
def __init__(self, c_m=None):
self.c_m = c_m
self.client_app_version = None
self.client_bundle_version = None
self.device_id = None
def set_client_info(self, app_version: str, bundle_version: str, device_id: str = None) -> None:
self.client_app_version = app_version
self.client_bundle_version = bundle_version
self.device_id = device_id
def get_bundle_list(self) -> list:
bundles: 'list[ContentBundle]' = BundleParser.bundles.get(
self.client_app_version, [])
if not bundles:
return []
now = time()
if Constant.BUNDLE_DOWNLOAD_LINK_PREFIX:
prefix = Constant.BUNDLE_DOWNLOAD_LINK_PREFIX
if prefix[-1] != '/':
prefix += '/'
def url_func(x): return f'{prefix}{x}'
else:
def url_func(x): return url_for(
'bundle_download', token=x, _external=True)
sql_list = []
r = []
for x in bundles:
if x.version_tuple <= ContentBundle.parse_version(self.client_bundle_version):
continue
t1 = os.urandom(64).hex()
t2 = os.urandom(64).hex()
x.json_url = url_func(t1)
x.bundle_url = url_func(t2)
sql_list.append((t1, x.json_path, now, self.device_id))
sql_list.append((t2, x.bundle_path, now, self.device_id))
r.append(x.to_dict())
if not sql_list:
return []
self.c_m.executemany(
'''insert into bundle_download_token values (?, ?, ?, ?)''', sql_list)
return r
def get_path_by_token(self, token: str, ip: str) -> str:
r = self.c_m.execute(
'''select file_path, time, device_id from bundle_download_token where token = ?''', (token,)).fetchone()
if not r:
raise NoAccess('Invalid token.', status=403)
file_path, create_time, device_id = r
if time() - create_time > Constant.BUNDLE_DOWNLOAD_TIME_GAP_LIMIT:
raise NoAccess('Expired token.', status=403)
if file_path.endswith('.cb') and not self.limiter.hit(ip):
raise RateLimit(
f'Too many content bundle downloads, IP: {ip}, DeviceID: {device_id}', status=429)
return file_path

7
latest version/core/config_manager.py
View File

@@ -42,7 +42,8 @@ class Config:
API_TOKEN = ''
DOWNLOAD_LINK_PREFIX = ''
DOWNLOAD_LINK_PREFIX = '' # http(s)://host(:port)/download/
BUNDLE_DOWNLOAD_LINK_PREFIX = '' # http(s)://host(:port)/bundle_download/
DOWNLOAD_USE_NGINX_X_ACCEL_REDIRECT = False
NGINX_X_ACCEL_REDIRECT_PREFIX = '/nginx_download/'
@@ -52,6 +53,9 @@ class Config:
DOWNLOAD_FORBID_WHEN_NO_ITEM = False
BUNDLE_DOWNLOAD_TIMES_LIMIT = '100/60 minutes'
BUNDLE_DOWNLOAD_TIME_GAP_LIMIT = 3000
LOGIN_DEVICE_NUMBER_LIMIT = 1
ALLOW_LOGIN_SAME_DEVICE = False
ALLOW_BAN_MULTIDEVICE_USER_AUTO = True
@@ -84,6 +88,7 @@ class Config:
WORLD_MAP_FOLDER_PATH = './database/map/'
SONG_FILE_FOLDER_PATH = './database/songs/'
SONGLIST_FILE_PATH = './database/songs/songlist'
CONTENT_BUNDLE_FOLDER_PATH = './database/bundle/'
SQLITE_DATABASE_PATH = './database/arcaea_database.db'
SQLITE_DATABASE_BACKUP_FOLDER_PATH = './database/backup/'
DATABASE_INIT_PATH = './database/init/'

4
latest version/core/constant.py
View File

@@ -43,6 +43,7 @@ class Constant:
WORLD_MAP_FOLDER_PATH = Config.WORLD_MAP_FOLDER_PATH
SONG_FILE_FOLDER_PATH = Config.SONG_FILE_FOLDER_PATH
SONGLIST_FILE_PATH = Config.SONGLIST_FILE_PATH
CONTENT_BUNDLE_FOLDER_PATH = Config.CONTENT_BUNDLE_FOLDER_PATH
SQLITE_DATABASE_PATH = Config.SQLITE_DATABASE_PATH
SQLITE_LOG_DATABASE_PATH = Config.SQLITE_LOG_DATABASE_PATH
SQLITE_DATABASE_DELETED_PATH = Config.SQLITE_DATABASE_DELETED_PATH
@@ -50,6 +51,9 @@ class Constant:
DOWNLOAD_TIMES_LIMIT = Config.DOWNLOAD_TIMES_LIMIT
DOWNLOAD_TIME_GAP_LIMIT = Config.DOWNLOAD_TIME_GAP_LIMIT
DOWNLOAD_LINK_PREFIX = Config.DOWNLOAD_LINK_PREFIX
BUNDLE_DOWNLOAD_TIMES_LIMIT = Config.BUNDLE_DOWNLOAD_TIMES_LIMIT
BUNDLE_DOWNLOAD_TIME_GAP_LIMIT = Config.BUNDLE_DOWNLOAD_TIME_GAP_LIMIT
BUNDLE_DOWNLOAD_LINK_PREFIX = Config.BUNDLE_DOWNLOAD_LINK_PREFIX
LINKPLAY_UNLOCK_LENGTH = 512 # Units: bytes
LINKPLAY_TIMEOUT = 5 # Units: seconds

7
latest version/core/error.py
View File

@@ -101,6 +101,13 @@ class NoAccess(ArcError):
super().__init__(message, error_code, api_error_code, extra_data, status)
class LowVersion(ArcError):
'''版本过低'''
def __init__(self, message=None, error_code=5, api_error_code=-999, extra_data=None, status=403) -> None:
super().__init__(message, error_code, api_error_code, extra_data, status)
class Timeout(ArcError):
'''超时'''
pass

20
latest version/core/init.py
View File

@@ -6,6 +6,7 @@ from shutil import copy, copy2
from time import time
from traceback import format_exc
from core.bundle import BundleParser
from core.config_manager import Config
from core.constant import ARCAEA_LOG_DATBASE_VERSION, ARCAEA_SERVER_VERSION
from core.course import Course
@@ -331,14 +332,27 @@ class FileChecker:
DownloadList.initialize_cache()
if not Config.SONG_FILE_HASH_PRE_CALCULATE:
self.logger.info('Song file hash pre-calculate is disabled.')
self.logger.info('Complete!')
self.logger.info('Song data initialization is complete!')
except Exception as e:
self.logger.error(format_exc())
self.logger.warning('Initialization error!')
self.logger.warning('Song data initialization error!')
f = False
return f
def check_content_bundle(self) -> bool:
'''检查 content bundle 有关文件并初始化缓存'''
f = self.check_folder(Config.CONTENT_BUNDLE_FOLDER_PATH)
self.logger.info("Start to initialize content bundle data...")
try:
BundleParser()
self.logger.info('Content bundle data initialization is complete!')
except Exception as e:
self.logger.error(format_exc())
self.logger.warning('Content bundle data initialization error!')
f = False
return f
def check_before_run(self) -> bool:
'''运行前检查，返回布尔值'''
MemoryDatabase() # 初始化内存数据库
return self.check_song_file() & self.check_update_database()
return self.check_song_file() and self.check_content_bundle() and self.check_update_database()

11
latest version/core/operation.py
View File

@@ -1,3 +1,4 @@
from .bundle import BundleParser
from .constant import Constant
from .download import DownloadList
from .error import NoData
@@ -74,6 +75,16 @@ class RefreshSongFileCache(BaseOperation):
DownloadList.initialize_cache()
class RefreshBundleCache(BaseOperation):
'''
刷新 bundle 缓存
'''
_name = 'refresh_content_bundle_cache'
def run(self):
BundleParser().re_init()
class SaveUpdateScore(BaseOperation):
'''
云存档更新成绩，是覆盖式更新

2
latest version/core/sql.py
View File

@@ -440,6 +440,8 @@ class MemoryDatabase:
self.c.execute('''PRAGMA synchronous = 0''')
self.c.execute('''create table if not exists download_token(user_id int,
song_id text,file_name text,token text,time int,primary key(user_id, song_id, file_name));''')
self.c.execute('''create table if not exists bundle_download_token(token text primary key,
file_path text, time int, device_id text);''')
self.c.execute(
'''create index if not exists download_token_1 on download_token (song_id, file_name);''')
self.conn.commit()

26
latest version/main.py
View File

@@ -27,6 +27,7 @@ import server
import web.index
import web.login
# import webapi
from core.bundle import BundleDownload
from core.constant import Constant
from core.download import UserDownload
from core.error import ArcError, NoAccess, RateLimit
@@ -81,7 +82,8 @@ def download(file_path):
x.song_id, x.file_name = file_path.split('/', 1)
x.select_for_check()
if x.is_limited:
raise RateLimit('You have reached the download limit.', 903)
raise RateLimit(
f'User `{x.user.user_id}` has reached the download limit.', 903)
if not x.is_valid:
raise NoAccess('Expired token.')
x.download_hit()
@@ -99,6 +101,26 @@ def download(file_path):
return error_return()
@app.route('/bundle_download/<string:token>', methods=['GET']) # 热更新下载
def bundle_download(token: str):
with Connect(in_memory=True) as c_m:
try:
file_path = BundleDownload(c_m).get_path_by_token(
token, request.remote_addr)
if Config.DOWNLOAD_USE_NGINX_X_ACCEL_REDIRECT:
# nginx X-Accel-Redirect
response = make_response()
response.headers['Content-Type'] = 'application/octet-stream'
response.headers['X-Accel-Redirect'] = Config.NGINX_X_ACCEL_REDIRECT_PREFIX + file_path
return response
return send_from_directory(Constant.CONTENT_BUNDLE_FOLDER_PATH, file_path, as_attachment=True, conditional=True)
except ArcError as e:
if Config.ALLOW_WARNING_LOG:
app.logger.warning(format_exc())
return error_return(e)
return error_return()
if Config.DEPLOY_MODE == 'waitress':
# 给waitress加个日志
@app.after_request
@@ -124,7 +146,7 @@ def tcp_server_run():
elif Config.DEPLOY_MODE == 'waitress':
# waitress WSGI server
import logging
from waitress import serve
from waitress import serve # type: ignore
logger = logging.getLogger('waitress')
logger.setLevel(logging.INFO)
serve(app, host=Config.HOST, port=Config.PORT)

12
latest version/server/func.py
View File

@@ -3,8 +3,9 @@ from traceback import format_exc
from flask import current_app, g, jsonify
from core.bundle import BundleParser
from core.config_manager import Config
from core.error import ArcError, NoAccess
from core.error import ArcError, LowVersion, NoAccess
has_arc_hash = False
try:
@@ -22,7 +23,9 @@ def error_return(e: ArcError = default_error): # 错误返回
# -4 您的账号已在别处登录
# -3 无法连接至服务器
# 2 Arcaea服务器正在维护
# 5 请更新 Arcaea 到最新版本
# 9 新版本请等待几分钟
# 11 有游戏内容需要更新，即将返回主界面
# 100 无法在此ip地址下登录游戏
# 101 用户名占用
# 102 电子邮箱已注册
@@ -65,7 +68,7 @@ def error_return(e: ArcError = default_error): # 错误返回
# 9803 下载已取消
# 9905 没有在云端发现任何数据
# 9907 更新数据时发生了问题
# 9908 服务器只支持最新的版本，请更新Arcaea
# 9908 服务器只支持最新的版本，请更新 Arcaea
# 其它 发生未知错误
r = {"success": False, "error_code": e.error_code}
if e.extra_data:
@@ -106,7 +109,10 @@ def header_check(request) -> ArcError:
headers = request.headers
if Config.ALLOW_APPVERSION: # 版本检查
if 'AppVersion' not in headers or headers['AppVersion'] not in Config.ALLOW_APPVERSION:
return NoAccess('Invalid app version', 5)
return LowVersion('Invalid app version', 5)
if 'ContentBundle' in headers and headers['ContentBundle'] != BundleParser.max_bundle_version[headers['AppVersion']]:
return LowVersion('Invalid content bundle version', 11)
if has_arc_hash and not ArcHashChecker(request).check():
return NoAccess('Invalid request')

34
latest version/server/others.py
View File

@@ -4,6 +4,7 @@ from urllib.parse import parse_qs, urlparse
from flask import Blueprint, jsonify, request
from werkzeug.datastructures import ImmutableMultiDict
from core.bundle import BundleDownload
from core.download import DownloadList
from core.error import RateLimit
from core.sql import Connect
@@ -26,26 +27,19 @@ def game_info():
return success_return(GameInfo().to_dict())
# @bp.route('/game/content_bundle', methods=['GET']) # 热更新
# def game_content_bundle():
# app_version = request.headers.get('AppVersion')
# bundle_version = request.headers.get('ContentBundle')
# import os
# if bundle_version != '5.4.0':
# r = {'orderedResults': [
# {
# 'appVersion': '5.4.0',
# 'contentBundleVersion': '5.4.0',
# 'jsonUrl': 'http://192.168.0.110/bundle_download/bundle.json',
# 'jsonSize': os.path.getsize('./database/bundle/bundle.json'),
# 'bundleUrl': 'http://192.168.0.110/bundle_download/bundle',
# 'bundleSize': os.path.getsize('./database/bundle/bundle')
# },
# ]
# }
# else:
# r = {}
# return success_return(r)
@bp.route('/game/content_bundle', methods=['GET']) # 热更新
@arc_try
def game_content_bundle():
# error code 5, 9 work
app_version = request.headers.get('AppVersion')
bundle_version = request.headers.get('ContentBundle')
device_id = request.headers.get('DeviceId')
with Connect(in_memory=True) as c_m:
x = BundleDownload(c_m)
x.set_client_info(app_version, bundle_version, device_id)
return success_return({
'orderedResults': x.get_bundle_list()
})
@bp.route('/serve/download/me/song', methods=['GET']) # 歌曲下载

15
latest version/templates/web/updatedatabase.html
View File

@@ -13,7 +13,8 @@
这里可以将旧版本的数据库同步到新版本的数据库，并刷新用户拥有的全角色列表。<br />
可上传文件: arcaea_database.db<br />
新数据库不存在的数据会被添加，存在的重复数据也会被改变。<br /><br />
Here you can synchronize the old version of the database to the new version of the database and refresh the list of full
Here you can synchronize the old version of the database to the new version of the database and refresh the list of
full
characters owned by players.<br />
Uploadable files: arcaea_database.db<br />
Data that does not exist in the new database will be added and the existing duplicate data will also be changed.
@@ -40,4 +41,16 @@
<div class="content">Here you can refresh the ratings of all scores in the database. The purpose is to deal
with the updating of songs' chart consts.</div>
</form>
<br />
<hr />
<form action="/web/updatedatabase/refreshsbundle" method="post">
<div class="title">Refresh content bundles</div>
<br />
<input type="submit" value="Refresh">
<div class="content">这里可以刷新储存在内存中的 bundle 文件夹下所有热更新文件的信息。目的是应对热更新文件的修改。</div>
<div class="content">Here you can refresh the information of all the files stored in the bundle folder in memory.
The purpose
is to deal with the updating of content bundles.</div>
</form>
{% endblock %}

14
latest version/web/index.py
View File

@@ -2,7 +2,7 @@ import os
import time
from core.init import FileChecker
from core.operation import RefreshAllScoreRating, RefreshSongFileCache, SaveUpdateScore, UnlockUserItem, DeleteUserScore
from core.operation import RefreshAllScoreRating, RefreshBundleCache, RefreshSongFileCache, SaveUpdateScore, UnlockUserItem, DeleteUserScore
from core.rank import RankList
from core.sql import Connect
from core.user import User
@@ -299,6 +299,18 @@ def update_song_hash():
return render_template('web/updatedatabase.html')
@bp.route('/updatedatabase/refreshsbundle', methods=['POST'])
@login_required
def update_content_bundle():
# 更新 bundle
try:
RefreshBundleCache().run()
flash('数据刷新成功 Success refresh data.')
except:
flash('Something error!')
return render_template('web/updatedatabase.html')
@bp.route('/updatedatabase/refreshsongrating', methods=['POST'])
@login_required
def update_song_rating():