[Refactor] Performance optimization

- Put download token in memory database
- Accelerate multiple querying in `best_score` table
This commit is contained in:
Lost-MSth
2022-11-01 18:08:02 +08:00
parent e04c8ecf2d
commit cf350842ac
11 changed files with 168 additions and 137 deletions

View File

@@ -98,8 +98,8 @@ It is just so interesting. What it can do is under exploration.
## 运行环境与依赖 Running environment and requirements
- Windows/Linux/Mac OS/Android...
- Python 3
- Flask module >= 2.0, Cryptography module >= 35.0.0, limits >= 2.7.0
- Python >= 3.6
- Flask module >= 2.0, Cryptography module >= 3.0.0, limits >= 2.7.0
- Charles, IDA, proxy app... (optional)
<!--

View File

@@ -54,8 +54,8 @@ class UserDownload:
limiter = ArcLimiter(
str(Constant.DOWNLOAD_TIMES_LIMIT) + '/day', 'download')
def __init__(self, c=None, user=None) -> None:
self.c = c
def __init__(self, c_m=None, user=None) -> None:
self.c_m = c_m
self.user = user
self.song_id: str = None
@@ -85,10 +85,10 @@ class UserDownload:
def select_for_check(self) -> None:
'''利用token、song_id、file_name查询其它信息'''
self.c.execute('''select user_id, time from download_token where song_id=? and file_name=? and token = ? limit 1;''',
(self.song_id, self.file_name, self.token))
self.c_m.execute('''select user_id, time from download_token where song_id=? and file_name=? and token = ? limit 1;''',
(self.song_id, self.file_name, self.token))
x = self.c.fetchone()
x = self.c_m.fetchone()
if not x:
raise NoAccess('The token `%s` is not valid.' %
self.token, status=403)
@@ -103,7 +103,7 @@ class UserDownload:
def insert_download_token(self) -> None:
'''将数据插入数据库,让这个下载链接可用'''
self.c.execute('''insert into download_token values(:a,:b,:c,:d,:e)''', {
self.c_m.execute('''insert or replace into download_token values(:a,:b,:c,:d,:e)''', {
'a': self.user.user_id, 'b': self.song_id, 'c': self.file_name, 'd': self.token, 'e': self.token_time})
@property
@@ -131,8 +131,8 @@ class DownloadList(UserDownload):
properties: `user` - `User`类或子类的实例
'''
def __init__(self, c=None, user=None) -> None:
super().__init__(c, user)
def __init__(self, c_m=None, user=None) -> None:
super().__init__(c_m, user)
self.song_ids: list = None
self.url_flag: bool = None
@@ -140,13 +140,12 @@ class DownloadList(UserDownload):
self.downloads: list = []
self.urls: dict = {}
def clear_download_token_from_song(self, song_id: str) -> None:
self.c.execute('''delete from download_token where user_id=:a and song_id=:b''', {
'a': self.user.user_id, 'b': song_id})
def clear_download_token(self) -> None:
'''清除过期下载链接'''
self.c_m.execute('''delete from download_token where time<?''',
(int(time()) - Constant.DOWNLOAD_TIME_GAP_LIMIT,))
def add_one_song(self, song_id: str) -> None:
if self.url_flag:
self.clear_download_token_from_song(song_id)
dir_list = os.listdir(os.path.join(
Constant.SONG_FILE_FOLDER_PATH, song_id))
@@ -155,7 +154,7 @@ class DownloadList(UserDownload):
if os.path.isfile(os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id, i)) and i in ['0.aff', '1.aff', '2.aff', '3.aff', 'base.ogg', '3.ogg', 'video.mp4', 'video_audio.ogg']:
if song_id in get_only_3_song_ids() and i not in ['3.aff', '3.ogg']:
continue
x = UserDownload(self.c, self.user)
x = UserDownload(self.c_m, self.user)
# self.downloads.append(x) # 这实际上没有用
x.song_id = song_id
x.file_name = i

View File

@@ -9,7 +9,7 @@ from core.config_manager import Config
from core.constant import ARCAEA_SERVER_VERSION
from core.course import Course
from core.purchase import Purchase
from core.sql import Connect, DatabaseMigrator
from core.sql import Connect, DatabaseMigrator, MemoryDatabase
from core.user import UserRegister
from core.util import try_rename
@@ -173,7 +173,7 @@ class FileChecker:
self.app.logger.warning('Folder `%s` is missing.' % folder_path)
return f
def chech_update_database(self) -> bool:
def check_update_database(self) -> bool:
if not self.check_file(Config.SQLITE_DATABASE_PATH):
# 新建数据库
try:
@@ -241,4 +241,5 @@ class FileChecker:
def check_before_run(self) -> bool:
'''运行前检查,返回布尔值'''
return self.check_folder(Config.SONG_FILE_FOLDER_PATH) & self.chech_update_database()
MemoryDatabase() # 初始化内存数据库
return self.check_folder(Config.SONG_FILE_FOLDER_PATH) & self.check_update_database()

View File

@@ -1,6 +1,7 @@
from .constant import Constant
from .score import UserScore
from .song import Chart
from .sql import Query, Sql
from .user import UserInfo
@@ -26,24 +27,27 @@ class RankList:
得到top分数表
'''
if self.limit >= 0:
self.c.execute('''select user_id from best_score where song_id = :song_id and difficulty = :difficulty order by score DESC, time_played DESC limit :limit''', {
self.c.execute('''select * from best_score where song_id = :song_id and difficulty = :difficulty order by score DESC, time_played DESC limit :limit''', {
'song_id': self.song.song_id, 'difficulty': self.song.difficulty, 'limit': self.limit})
else:
self.c.execute('''select user_id from best_score where song_id = :song_id and difficulty = :difficulty order by score DESC, time_played DESC''', {
self.c.execute('''select * from best_score where song_id = :song_id and difficulty = :difficulty order by score DESC, time_played DESC''', {
'song_id': self.song.song_id, 'difficulty': self.song.difficulty})
x = self.c.fetchall()
if not x:
return None
user_info_list = Sql(self.c).select('user', ['name', 'character_id', 'is_skill_sealed', 'is_char_uncapped',
'is_char_uncapped_override', 'favorite_character'], Query().from_args({'user_id': [i[0] for i in x]}))
rank = 0
self.list = []
for i in x:
for i, j in zip(x, user_info_list):
rank += 1
y = UserScore(self.c, UserInfo(self.c, i[0]))
y = UserScore(self.c, UserInfo(self.c, i[0])).from_list(i)
y.song = self.song
y.select_score()
y.rank = rank
y.user.from_list_about_character(j)
self.list.append(y)
def select_friend(self, user=None, limit=Constant.MAX_FRIEND_COUNT) -> None:
@@ -54,20 +58,25 @@ class RankList:
if user:
self.user = user
self.c.execute('''select user_id from best_score where user_id in (select :user_id union select user_id_other from friend where user_id_me = :user_id) and song_id = :song_id and difficulty = :difficulty order by score DESC, time_played DESC limit :limit''', {
'user_id': self.user.user_id, 'song_id': self.song.song_id, 'difficulty': self.song.difficulty, 'limit': self.limit})
user_ids = [self.user.user_id] + [x[0] for x in self.user.friend_ids]
self.c.execute(f'''select * from best_score where user_id in ({','.join(['?'] * len(user_ids))}) and song_id = ? and difficulty = ? order by score DESC, time_played DESC limit ?''', user_ids + [
self.song.song_id, self.song.difficulty, self.limit])
x = self.c.fetchall()
if not x:
return None
user_info_list = Sql(self.c).select('user', ['name', 'character_id', 'is_skill_sealed', 'is_char_uncapped',
'is_char_uncapped_override', 'favorite_character'], Query().from_args({'user_id': [i[0] for i in x]}))
rank = 0
self.list = []
for i in x:
for i, j in zip(x, user_info_list):
rank += 1
y = UserScore(self.c, UserInfo(self.c, i[0]))
y = UserScore(self.c, UserInfo(self.c, i[0])).from_list(i)
y.song = self.song
y.select_score()
y.rank = rank
y.user.from_list_about_character(j)
self.list.append(y)
@staticmethod
@@ -118,19 +127,24 @@ class RankList:
sql_limit, sql_offset, need_myself = self.get_my_rank_parameter(
my_rank, int(self.c.fetchone()[0]), self.limit)
self.c.execute('''select user_id from best_score where song_id = :song_id and difficulty = :difficulty order by score DESC, time_played DESC limit :limit offset :offset''', {
self.c.execute('''select * from best_score where song_id = :song_id and difficulty = :difficulty order by score DESC, time_played DESC limit :limit offset :offset''', {
'song_id': self.song.song_id, 'difficulty': self.song.difficulty, 'limit': sql_limit, 'offset': sql_offset})
x = self.c.fetchall()
if x:
user_info_list = Sql(self.c).select('user', ['name', 'character_id', 'is_skill_sealed', 'is_char_uncapped',
'is_char_uncapped_override', 'favorite_character'], Query().from_args({'user_id': [i[0] for i in x]}))
rank = sql_offset if sql_offset > 0 else 0
self.list = []
for i in x:
for i, j in zip(x, user_info_list):
rank += 1
y = UserScore(self.c, UserInfo(self.c, i[0]))
y = UserScore(self.c, UserInfo(self.c, i[0])).from_list(i)
y.song = self.song
y.select_score()
y.rank = rank
y.user.from_list_about_character(j)
self.list.append(y)
if need_myself:
y = UserScore(self.c, UserInfo(self.c, self.user.user_id))
y.song = self.song

View File

@@ -99,9 +99,7 @@ class SaveData:
parameter: `user` - `User`类或子类的实例
'''
self.createdAt = int(time() * 1000)
self.c.execute('''delete from user_save where user_id=:a''', {
'a': user.user_id})
self.c.execute('''insert into user_save values(:a,:b,:c,:d,:e,:f,:g,:h,:i,:j)''', {
self.c.execute('''insert or replace into user_save values(:a,:b,:c,:d,:e,:f,:g,:h,:i,:j)''', {
'a': user.user_id, 'b': json.dumps({'': self.scores_data}), 'c': json.dumps({'': self.clearlamps_data}), 'd': json.dumps({'': self.clearedsongs_data}), 'e': json.dumps({'': self.unlocklist_data}), 'f': json.dumps({'val': self.installid_data}), 'g': json.dumps({'val': self.devicemodelname_data}), 'h': json.dumps({'': self.story_data}), 'i': self.createdAt, 'j': self.finalestate_data})
def set_value(self, key: str, value: str, checksum: str) -> None:

View File

@@ -155,9 +155,10 @@ class UserScore(Score):
super().__init__()
self.c = c
self.user = user
self.rank = None
self.rank = None # 成绩排名给Ranklist用的
def select_score(self) -> None:
'''查询成绩以及用户搭档信息,单次查询可用,不要集体循环查询'''
self.c.execute('''select * from best_score where user_id = :a and song_id = :b and difficulty = :c''',
{'a': self.user.user_id, 'b': self.song.song_id, 'c': self.song.difficulty})
x = self.c.fetchone()

View File

@@ -10,16 +10,21 @@ from .error import ArcError, InputError
class Connect:
# 数据库连接类,上下文管理
def __init__(self, file_path=Constant.SQLITE_DATABASE_PATH):
def __init__(self, file_path: str = Constant.SQLITE_DATABASE_PATH, in_memory: bool = False):
"""
数据库连接默认连接arcaea_database.db\
接受:文件路径\
返回sqlite3连接操作对象
"""
self.file_path = file_path
self.in_memory = in_memory
def __enter__(self):
self.conn = sqlite3.connect(self.file_path, timeout=10)
if self.in_memory:
self.conn = sqlite3.connect(
'file:arc_tmp?mode=memory&cache=shared', uri=True, timeout=10)
else:
self.conn = sqlite3.connect(self.file_path, timeout=10)
self.c = self.conn.cursor()
return self.c
@@ -46,14 +51,17 @@ class Query:
'''查询参数类'''
def __init__(self, query_able: list = None, quzzy_query_able: list = None, sort_able: list = None) -> None:
self.query_able: list = query_able
self.quzzy_query_able: list = quzzy_query_able
self.query_able: list = query_able # None表示不限制
self.quzzy_query_able: list = quzzy_query_able # None表示不限制
self.sort_able: list = sort_able
self.__limit: int = -1
self.__offset: int = 0
self.__query: dict = {} # {'name': 'admin'}
# {'name': 'admin'} or {'name': ['admin', 'user']}
self.__query: dict = {}
self.__fuzzy_query: dict = {} # {'name': 'dmi'}
# [{'column': 'user_id', 'order': 'ASC'}, ...]
self.__sort: list = []
@@ -148,6 +156,10 @@ class Query:
d.get('query', {}), d.get('fuzzy_query', {}), d.get('sort', []))
return self
def from_args(self, query: dict, limit: int = -1, offset: int = 0, sort: list = [], fuzzy_query: dict = {}) -> 'Query':
self.set_value(limit, offset, query, fuzzy_query, sort)
return self
class Sql:
'''
@@ -160,54 +172,35 @@ class Sql:
@staticmethod
def get_select_sql(table_name: str, target_column: list = [], query: 'Query' = None):
'''拼接单表内行查询单句sql语句返回语句和参数列表'''
sql = 'select '
sql_list = []
if len(target_column) >= 2:
sql += target_column[0]
for i in range(1, len(target_column)):
sql += ',' + target_column[i]
sql += ' from ' + table_name
elif len(target_column) == 1:
sql += target_column[0] + ' from ' + table_name
if not target_column:
sql = f'select * from {table_name}'
else:
sql += '* from ' + table_name
sql = f"select {', '.join(target_column)} from {table_name}"
if query is None:
return sql, sql_list
where_key = []
where_like_key = []
for i in query.query:
where_key.append(i)
sql_list.append(query.query[i])
for i in query.fuzzy_query:
where_like_key.append(i)
sql_list.append('%' + query.fuzzy_query[i] + '%')
if where_key or where_like_key:
sql += ' where '
if where_key:
sql += where_key[0] + '=?'
if len(where_key) >= 2:
for i in range(1, len(where_key)):
sql += ' and ' + where_key[i] + '=?'
if where_like_key:
for i in where_like_key:
sql += ' and ' + i + ' like ?'
for k, v in query.query.items():
if isinstance(v, list):
where_key.append(f"{k} in ({','.join(['?'] * len(v))})")
sql_list.extend(v)
else:
sql += where_like_key[0] + ' like ?'
where_key.append(f'{k}=?')
sql_list.append(v)
if len(where_like_key) >= 2:
for i in range(1, len(where_key)):
sql += ' and ' + where_key[i] + ' like ?'
for k, v in query.fuzzy_query.items():
where_key.append(f'{k} like ?')
sql_list.append(f'%{v}%')
if where_key:
sql += ' where '
sql += ' and '.join(where_key)
if query.sort:
sql += ' order by ' + \
query.sort[0]['column'] + ' ' + query.sort[0]['order']
for i in range(1, len(query.sort)):
sql += ', ' + query.sort[i]['column'] + \
' ' + query.sort[i]['order']
', '.join([x['column'] + ' ' + x['order'] for x in query.sort])
if query.limit >= 0:
sql += ' limit ? offset ?'
@@ -225,21 +218,29 @@ class Sql:
@staticmethod
def get_delete_sql(table_name: str, query: 'Query' = None):
'''拼接删除语句query中只有query(where =)会被处理'''
sql = 'delete from ' + table_name
'''拼接删除语句query中只有query和fuzzy_query会被处理'''
sql = f'delete from {table_name}'
if query is None:
return sql, []
sql_list = []
where_key = []
for k, v in query.query.items():
if isinstance(v, list):
where_key.append(f"{k} in ({','.join(['?'] * len(v))})")
sql_list.extend(v)
else:
where_key.append(f'{k}=?')
sql_list.append(v)
if query is not None and query.query:
for k, v in query.fuzzy_query.items():
where_key.append(f'{k} like ?')
sql_list.append(f'%{v}%')
if where_key:
sql += ' where '
where_key = []
for i in query.query:
where_key.append(i)
sql_list.append(query.query[i])
sql += where_key[0] + '=?'
if len(where_key) >= 1:
for i in range(1, len(where_key)):
sql += ' and ' + where_key[i] + '=?'
sql += ' and '.join(where_key)
return sql, sql_list
@@ -268,7 +269,7 @@ class Sql:
table_name, key, len(value_list[0]), insert_type), value_list)
def delete(self, table_name: str, query: 'Query' = None) -> None:
'''删除query中只有query(where =)会被处理'''
'''删除query中只有query和fuzzy_query会被处理'''
sql, sql_list = self.get_delete_sql(table_name, query)
self.c.execute(sql, sql_list)
@@ -351,3 +352,20 @@ class DatabaseMigrator:
self.update_user_char_full(c2) # 更新user_char_full
self.update_user_epilogue(c2) # 更新user的epilogue
class MemoryDatabase:
conn = sqlite3.connect('file:arc_tmp?mode=memory&cache=shared', uri=True)
def __init__(self):
self.c = self.conn.cursor()
self.c.execute('''PRAGMA journal_mode = OFF''')
self.c.execute('''PRAGMA synchronous = 0''')
self.c.execute('''create table if not exists download_token(user_id int,
song_id text,file_name text,token text,time int,primary key(user_id, song_id, file_name));''')
self.c.execute(
'''create index if not exists download_token_1 on download_token (song_id, file_name);''')
self.conn.commit()
def atexit(self):
self.conn.close()

View File

@@ -377,40 +377,42 @@ class UserInfo(User):
self.favorite_character.select_character_uncap_condition(self)
return self.favorite_character
@property
def friend_ids(self) -> list:
self.c.execute('''select user_id_other from friend where user_id_me = :user_id''', {
'user_id': self.user_id})
return self.c.fetchall()
@property
def friends(self) -> list:
# 得到用户的朋友列表
if self.__friends is None:
self.c.execute('''select user_id_other from friend where user_id_me = :user_id''', {
'user_id': self.user_id})
x = self.c.fetchall()
s = []
if x != [] and x[0][0] is not None:
for i in x:
self.c.execute('''select exists(select * from friend where user_id_me = :x and user_id_other = :y)''',
{'x': i[0], 'y': self.user_id})
for i in self.friend_ids:
self.c.execute('''select exists(select * from friend where user_id_me = :x and user_id_other = :y)''',
{'x': i[0], 'y': self.user_id})
is_mutual = True if self.c.fetchone() == (1,) else False
is_mutual = True if self.c.fetchone() == (1,) else False
you = UserOnline(self.c, i[0])
you.select_user()
character = you.character if you.favorite_character is None else you.favorite_character
character.select_character_uncap_condition(you)
you = UserOnline(self.c, i[0])
you.select_user()
character = you.character if you.favorite_character is None else you.favorite_character
character.select_character_uncap_condition(you)
rating = you.rating_ptt if not you.is_hide_rating else -1
rating = you.rating_ptt if not you.is_hide_rating else -1
s.append({
"is_mutual": is_mutual,
"is_char_uncapped_override": character.is_uncapped_override,
"is_char_uncapped": character.is_uncapped,
"is_skill_sealed": you.is_skill_sealed,
"rating": rating,
"join_date": you.join_date,
"character": character.character_id,
"recent_score": you.recent_score_list,
"name": you.name,
"user_id": you.user_id
})
s.append({
"is_mutual": is_mutual,
"is_char_uncapped_override": character.is_uncapped_override,
"is_char_uncapped": character.is_uncapped,
"is_skill_sealed": you.is_skill_sealed,
"rating": rating,
"join_date": you.join_date,
"character": character.character_id,
"recent_score": you.recent_score_list,
"name": you.name,
"user_id": you.user_id
})
s.sort(key=lambda item: item["recent_score"][0]["time_played"] if len(
item["recent_score"]) > 0 else 0, reverse=True)
self.__friends = s
@@ -568,16 +570,8 @@ class UserInfo(User):
self.stamina = UserStamina(self.c, self)
self.stamina.set_value(x[0], x[1])
def select_user_about_character(self) -> None:
'''
查询user表有关角色的信息
'''
self.c.execute('''select name, character_id, is_skill_sealed, is_char_uncapped, is_char_uncapped_override, favorite_character from user where user_id = :a''', {
'a': self.user_id})
x = self.c.fetchone()
if not x:
raise NoData('No user.', 108, -3)
def from_list_about_character(self, x: list) -> None:
'''从数据库user表获取搭档信息'''
self.name = x[0]
self.character = UserCharacter(self.c, x[1], self)
self.is_skill_sealed = x[2] == 1
@@ -586,6 +580,18 @@ class UserInfo(User):
self.favorite_character = None if x[5] == - \
1 else UserCharacter(self.c, x[5], self)
def select_user_about_character(self) -> None:
'''
查询user表有关搭档的信息
'''
self.c.execute('''select name, character_id, is_skill_sealed, is_char_uncapped, is_char_uncapped_override, favorite_character from user where user_id = :a''', {
'a': self.user_id})
x = self.c.fetchone()
if not x:
raise NoData('No user.', 108, -3)
self.from_list_about_character(x)
def select_user_about_world_play(self) -> None:
'''
查询user表有关世界模式打歌的信息

View File

@@ -184,13 +184,6 @@ stamina_multiply int,
fragment_multiply int,
prog_boost_multiply int
);
create table if not exists download_token(user_id int,
song_id text,
file_name text,
token text,
time int,
primary key(user_id, song_id, file_name)
);
create table if not exists item(item_id text,
type text,
is_available int,
@@ -314,7 +307,6 @@ primary key(course_id, item_id, type)
);
create index if not exists best_score_1 on best_score (song_id, difficulty);
create index if not exists download_token_1 on download_token (song_id, file_name);
PRAGMA journal_mode = WAL;
PRAGMA default_cache_size = 8000;

View File

@@ -73,7 +73,7 @@ def favicon():
@app.route('/download/<path:file_path>', methods=['GET']) # 下载
def download(file_path):
with Connect() as c:
with Connect(in_memory=True) as c:
try:
x = UserDownload(c)
x.token = request.args.get('t')

View File

@@ -29,12 +29,14 @@ def game_info():
@auth_required(request)
@arc_try
def download_song(user_id):
with Connect() as c:
x = DownloadList(c, UserOnline(c, user_id))
with Connect(in_memory=True) as c:
x = DownloadList(c, UserOnline(None, user_id))
x.song_ids = request.args.getlist('sid')
x.url_flag = json.loads(request.args.get('url', 'true'))
if x.url_flag and x.is_limited:
raise RateLimit('You have reached the download limit.', 903)
if x.url_flag:
x.clear_download_token()
x.add_songs()
return success_return(x.urls)