[Bug fix][Refactor] Course scores & Download

- Fix a bug that scores in course mode cannot calculate rating
- Make some optimization for downloading songs

latest version/core/constant.py

@@ -96,6 +96,6 @@ class Constant:
     ]
 
     DATABASE_MIGRATE_TABLES = ['user', 'friend', 'best_score', 'recent30', 'user_world', 'item', 'user_item', 'purchase', 'purchase_item', 'user_save',
-                               'login', 'present', 'user_present', 'present_item', 'redeem', 'user_redeem', 'redeem_item', 'api_login', 'chart', 'user_course', 'user_char']
+                               'login', 'present', 'user_present', 'present_item', 'redeem', 'user_redeem', 'redeem_item', 'api_login', 'chart', 'user_course', 'user_char', 'user_role']
 
     UPDATE_WITH_NEW_CHARACTER_DATA = Config.UPDATE_WITH_NEW_CHARACTER_DATA

latest version/core/download.py

@@ -99,7 +99,7 @@ class UserDownload:
     def generate_token(self) -> None:
         self.token_time = int(time())
         self.token = md5(str(self.user.user_id) + self.song_id +
-                         self.file_name + str(self.token_time))
+                         self.file_name + str(self.token_time) + str(os.urandom(8)))
 
     def insert_download_token(self) -> None:
         '''将数据插入数据库，让这个下载链接可用'''
@@ -111,14 +111,14 @@ class UserDownload:
         '''生成下载链接'''
         if self.token is None:
             self.generate_token()
-            self.insert_download_token()
+            # self.insert_download_token() # 循环插入速度慢，改为executemany
         if Constant.DOWNLOAD_LINK_PREFIX:
             prefix = Constant.DOWNLOAD_LINK_PREFIX
             if prefix[-1] != '/':
                 prefix += '/'
-            return prefix + self.song_id + '/' + self.file_name + '?t=' + self.token
+            return f'{prefix}{self.song_id}/{self.file_name}?t={self.token}'
         else:
-            return url_for('download', file_path=self.song_id + '/' + self.file_name, t=self.token, _external=True)
+            return url_for('download', file_path=f'{self.song_id}/{self.file_name}', t=self.token, _external=True)
 
     @property
     def hash(self) -> str:
@@ -140,67 +140,100 @@ class DownloadList(UserDownload):
         self.downloads: list = []
         self.urls: dict = {}
 
+    @staticmethod
+    def clear_all_cache():
+        '''清除所有歌曲文件有关缓存'''
+        get_song_file_md5.cache_clear()
+        get_only_3_song_ids.cache_clear()
+        DownloadList.get_one_song_file_names.cache_clear()
+        DownloadList.get_all_song_ids.cache_clear()
+
     def clear_download_token(self) -> None:
         '''清除过期下载链接'''
         self.c_m.execute('''delete from download_token where time<?''',
                          (int(time()) - Constant.DOWNLOAD_TIME_GAP_LIMIT,))
 
-    def add_one_song(self, song_id: str) -> None:
-        dir_list = os.listdir(os.path.join(
-            Constant.SONG_FILE_FOLDER_PATH, song_id))
+    def insert_download_tokens(self) -> None:
+        '''插入所有下载链接'''
+        self.c_m.executemany('''insert or replace into download_token values(?,?,?,?,?)''', [(
+            self.user.user_id, x.song_id, x.file_name, x.token, x.token_time) for x in self.downloads])
 
-        re = {}
-        for i in dir_list:
+    @staticmethod
+    @lru_cache(maxsize=2048)
+    def get_one_song_file_names(song_id: str) -> list:
+        '''获取一个歌曲文件夹下的所有合法文件名，有lru缓存'''
+        r = []
+        for i in os.listdir(os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id)):
             if os.path.isfile(os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id, i)) and i in ['0.aff', '1.aff', '2.aff', '3.aff', 'base.ogg', '3.ogg', 'video.mp4', 'video_audio.ogg']:
                 if song_id in get_only_3_song_ids() and i not in ['3.aff', '3.ogg']:
                     continue
-                x = UserDownload(self.c_m, self.user)
-                # self.downloads.append(x) # 这实际上没有用
-                x.song_id = song_id
-                x.file_name = i
-                if i == 'base.ogg':
-                    if 'audio' not in re:
-                        re['audio'] = {}
+                r.append(i)
+        return r
 
-                    re['audio']["checksum"] = x.hash
-                    if self.url_flag:
-                        re['audio']["url"] = x.url
-                elif i == '3.ogg':
-                    if 'audio' not in re:
-                        re['audio'] = {}
+    def add_one_song(self, song_id: str) -> None:
 
-                    if self.url_flag:
-                        re['audio']['3'] = {"checksum": x.hash, "url": x.url}
-                    else:
-                        re['audio']['3'] = {"checksum": x.hash}
-                elif i == 'video.mp4' or i == 'video_audio.ogg':
-                    if 'additional_files' not in re:
-                        re['additional_files'] = []
+        re = {}
+        for i in self.get_one_song_file_names(song_id):
+            x = UserDownload(self.c_m, self.user)
+            self.downloads.append(x)
+            x.song_id = song_id
+            x.file_name = i
+            if i == 'base.ogg':
+                if 'audio' not in re:
+                    re['audio'] = {}
 
-                    if self.url_flag:
-                        re['additional_files'].append(
-                            {"checksum": x.hash, "url": x.url, 'file_name': i})
-                    else:
-                        re['additional_files'].append(
-                            {"checksum": x.hash, 'file_name': i})
+                re['audio']["checksum"] = x.hash
+                if self.url_flag:
+                    re['audio']["url"] = x.url
+            elif i == '3.ogg':
+                if 'audio' not in re:
+                    re['audio'] = {}
+
+                if self.url_flag:
+                    re['audio']['3'] = {"checksum": x.hash, "url": x.url}
                 else:
-                    if 'chart' not in re:
-                        re['chart'] = {}
+                    re['audio']['3'] = {"checksum": x.hash}
+            elif i == 'video.mp4' or i == 'video_audio.ogg':
+                if 'additional_files' not in re:
+                    re['additional_files'] = []
 
-                    if self.url_flag:
-                        re['chart'][i[0]] = {"checksum": x.hash, "url": x.url}
-                    else:
-                        re['chart'][i[0]] = {"checksum": x.hash}
+                if self.url_flag:
+                    re['additional_files'].append(
+                        {"checksum": x.hash, "url": x.url, 'file_name': i})
+                else:
+                    re['additional_files'].append(
+                        {"checksum": x.hash, 'file_name': i})
+            else:
+                if 'chart' not in re:
+                    re['chart'] = {}
+
+                if self.url_flag:
+                    re['chart'][i[0]] = {"checksum": x.hash, "url": x.url}
+                else:
+                    re['chart'][i[0]] = {"checksum": x.hash}
 
         self.urls.update({song_id: re})
 
+    @staticmethod
+    @lru_cache()
+    def get_all_song_ids() -> list:
+        '''获取全歌曲文件夹列表，有lru缓存'''
+        return list(filter(lambda x: os.path.isdir(os.path.join(Constant.SONG_FILE_FOLDER_PATH, x)), os.listdir(Constant.SONG_FILE_FOLDER_PATH)))
+
     def add_songs(self, song_ids: list = None) -> None:
         '''添加一个或多个歌曲到下载列表，若`song_ids`为空，则添加所有歌曲'''
         if song_ids is not None:
             self.song_ids = song_ids
 
-        x = self.song_ids if self.song_ids else os.listdir(
-            Constant.SONG_FILE_FOLDER_PATH)
-        for i in x:
-            if os.path.isdir(os.path.join(Constant.SONG_FILE_FOLDER_PATH, i)):
+        if not self.song_ids:
+            self.song_ids = self.get_all_song_ids()
+            for i in self.song_ids:
                 self.add_one_song(i)
+        else:
+            for i in self.song_ids:
+                if os.path.isdir(os.path.join(Constant.SONG_FILE_FOLDER_PATH, i)):
+                    self.add_one_song(i)
+
+        if self.url_flag:
+            self.clear_download_token()
+            self.insert_download_tokens()

latest version/core/score.py

@@ -273,7 +273,7 @@ class UserPlay(UserScore):
             self.course_play_state = -1
             return None
             # raise NoData('No token data.')
-        self.song.set_chart(x[2], x[3])
+        # self.song.set_chart(x[2], x[3])
         if x[4]:
             self.course_play = CoursePlay(self.c, self.user, self)
             self.course_play.course_id = x[4]

latest version/main.py

@@ -103,7 +103,7 @@ if Config.DEPLOY_MODE == 'waitress':
     @app.after_request
     def after_request(response):
         app.logger.info(
-            f'B {request.remote_addr} - - {request.method} {request.path} {response.status_code}')
+            f'{request.remote_addr} - - {request.method} {request.path} {response.status_code}')
         return response
 
 
@@ -116,8 +116,8 @@ def tcp_server_run():
         WSGIServer(host_port, app, log=app.logger).serve_forever()
     elif Config.DEPLOY_MODE == 'waitress':
         # waitress WSGI server
-        from waitress import serve
         import logging
+        from waitress import serve
         logger = logging.getLogger('waitress')
         logger.setLevel(logging.INFO)
         serve(app, host=Config.HOST, port=Config.PORT)

latest version/server/course.py

@@ -1,6 +1,5 @@
 from core.constant import Constant
 from core.course import UserCourseList
-from core.error import ArcError
 from core.item import ItemCore
 from core.sql import Connect
 from core.user import UserOnline

latest version/server/others.py

@@ -35,8 +35,6 @@ def download_song(user_id):
         x.url_flag = json.loads(request.args.get('url', 'true'))
         if x.url_flag and x.is_limited:
             raise RateLimit('You have reached the download limit.', 903)
-        if x.url_flag:
-            x.clear_download_token()
 
         x.add_songs()
         return success_return(x.urls)

latest version/web/index.py

@@ -2,7 +2,7 @@ import os
 import time
 
 import server.arcscore
-from core.download import get_only_3_song_ids, initialize_songfile
+from core.download import DownloadList, initialize_songfile
 from core.init import FileChecker
 from core.rank import RankList
 from core.sql import Connect
@@ -290,8 +290,7 @@ def update_database():
 def update_song_hash():
     # 更新数据库内谱面文件hash值
     try:
-        get_only_3_song_ids.cache_clear()
-        get_only_3_song_ids()
+        DownloadList.clear_all_cache()
         initialize_songfile()
         flash('数据刷新成功 Success refresh data.')
     except: