Database performance optimization for large quantities

- Add two indices for avoiding possible slow queries

> This is an experimental handle for the database which has some tables over one hundred thousand lines.

latest version/core/download.py

@@ -55,7 +55,6 @@ class UserDownload:
         self.song_id: str = None
         self.file_name: str = None
 
-        self.file_path: str = None
         self.token: str = None
         self.token_time: int = None
 
@@ -66,6 +65,8 @@ class UserDownload:
     @property
     def is_limited(self) -> bool:
         '''是否达到用户最大下载量'''
+        if self.user is None:
+            self.select_for_check()
         self.c.execute(
             '''select count(*) from user_download where user_id = :a''', {'a': self.user.user_id})
         y = self.c.fetchone()
@@ -74,27 +75,26 @@ class UserDownload:
     @property
     def is_valid(self) -> bool:
         '''链接是否有效且未过期'''
-        return int(time()) - self.token_time <= Constant.DOWNLOAD_TIME_GAP_LIMIT and self.song_id+'/'+self.file_name == self.file_path
+        if self.token_time is None:
+            self.select_for_check()
+        return int(time()) - self.token_time <= Constant.DOWNLOAD_TIME_GAP_LIMIT
 
     def insert_user_download(self) -> None:
         '''记录下载信息'''
         self.c.execute('''insert into user_download values(:a,:b,:c)''', {
-                       'a': self.user.user_id, 'b': self.token, 'c': int(time())})
+                       'a': self.user.user_id, 'c': self.token, 'b': int(time())})
 
-    def select_from_token(self, token: str = None) -> None:
+    def select_for_check(self) -> None:
-        if token is not None:
+        '''利用token、song_id、file_name查询其它信息'''
-            self.token = token
+        self.c.execute('''select user_id, time from download_token where song_id=? and file_name=? and token = ? limit 1;''',
-        self.c.execute('''select * from download_token where token = :t limit 1''',
+                       (self.song_id, self.file_name, self.token))
-                       {'t': self.token})
 
         x = self.c.fetchone()
         if not x:
             raise NoAccess('The token `%s` is not valid.' % self.token)
         self.user = User()
         self.user.user_id = x[0]
-        self.song_id = x[1]
+        self.token_time = x[1]
-        self.file_name = x[2]
-        self.token_time = x[4]
 
     def generate_token(self) -> None:
         self.token_time = int(time())

latest version/database/database_initialize.py

@@ -4,7 +4,7 @@ import json
 
 # 数据库初始化文件，删掉arcaea_database.db文件后运行即可，谨慎使用
 
-ARCAEA_SERVER_VERSION = 'v2.9.1.1'
+ARCAEA_SERVER_VERSION = 'v2.9.1.3'
 
 
 def main(path='./'):
@@ -208,9 +208,9 @@ def main(path='./'):
     primary key(user_id, song_id, file_name)
     );''')
     c.execute('''create table if not exists user_download(user_id int,
-    token text,
     time int,
-    primary key(user_id, token, time)
+    token text,
+    primary key(user_id, time, token)
     );''')
     c.execute('''create table if not exists item(item_id text,
     type text,
@@ -337,6 +337,11 @@ def main(path='./'):
     primary key(course_id, item_id, type)
     );''')
 
+    c.execute(
+        '''create index best_score_1 on best_score (song_id, difficulty);''')  # 排名查询优化
+    c.execute(
+        '''create index download_token_1 on download_token (song_id, file_name);''')  # 下载token判断优化
+
     # 搭档初始化
     char = ['hikari', 'tairitsu', 'kou', 'sapphire', 'lethe', 'hikari&tairitsu(reunion)', 'Tairitsu(Axium)', 'Tairitsu(Grievous Lady)', 'stella', 'Hikari & Fisica', 'ilith', 'eto', 'luna', 'shirabe', 'Hikari(Zero)', 'Hikari(Fracture)', 'Hikari(Summer)', 'Tairitsu(Summer)', 'Tairitsu & Trin',
             'ayu', 'Eto & Luna', 'yume', 'Seine & Hikari', 'saya', 'Tairitsu & Chuni Penguin', 'Chuni Penguin', 'haruna', 'nono', 'MTA-XXX', 'MDA-21', 'kanae', 'Hikari(Fantasia)', 'Tairitsu(Sonata)', 'sia', 'DORO*C', 'Tairitsu(Tempest)', 'brillante', 'Ilith(Summer)', 'etude', 'Alice & Tenniel', 'Luna & Mia', 'areus', 'seele', 'isabelle', 'mir', 'lagrange', 'linka', 'nami', 'Saya & Elizabeth', 'lily', 'kanae(midsummer)', 'alice&tenniel(minuet)', 'tairitsu(elegy)', 'marija', 'vita', 'hikari(fatalis)']

latest version/main.py

@@ -52,15 +52,17 @@ def download(file_path):
     with Connect() as c:
         try:
             x = UserDownload(c)
-            x.file_path = file_path
+            x.token = request.args.get('t')
-            x.select_from_token(request.args.get('t'))
+            x.song_id, x.file_name = file_path.split('/', 1)
+            x.select_for_check()
             if x.is_limited:
                 raise ArcError('You have reached the download limit.', 903)
             if x.is_valid:
                 x.insert_user_download()
                 return send_from_directory(Constant.SONG_FILE_FOLDER_PATH, file_path, as_attachment=True, conditional=True)
         except ArcError as e:
-            app.logger.warning(format_exc())
+            if Config.ALLOW_WARNING_LOG:
+                app.logger.warning(format_exc())
             return error_return(e)
     return error_return()