[Enhance][Refactor] unranked score & warning log

- Add an option that can be used to forbid some illegal scores
- Add brief warning logs for custom exceptions

latest version/core/config_manager.py

@@ -50,6 +50,8 @@ class Config:
     ALLOW_LOGIN_SAME_DEVICE = False
     ALLOW_BAN_MULTIDEVICE_USER_AUTO = True
 
+    ALLOW_SCORE_WITH_NO_SONG = True
+
     ALLOW_INFO_LOG = False
     ALLOW_WARNING_LOG = False
 

latest version/core/download.py

@@ -20,15 +20,6 @@ def get_song_file_md5(song_id: str, file_name: str) -> str:
     return get_file_md5(path)
 
 
-def initialize_songfile():
-    '''初始化歌曲数据的md5信息'''
-    get_song_file_md5.cache_clear()
-    x = DownloadList()
-    x.url_flag = False
-    x.add_songs()
-    del x
-
-
 class SonglistParser:
     '''songlist文件解析器'''
 
@@ -189,13 +180,22 @@ class DownloadList(UserDownload):
         self.downloads: list = []
         self.urls: dict = {}
 
+    @classmethod
+    def initialize_cache(cls) -> None:
+        '''初始化歌曲数据缓存，包括md5、文件目录遍历、解析songlist'''
+        SonglistParser()
+        x = cls()
+        x.url_flag = False
+        x.add_songs()
+        del x
+
     @staticmethod
-    def clear_all_cache():
+    def clear_all_cache() -> None:
         '''清除所有歌曲文件有关缓存'''
         get_song_file_md5.cache_clear()
         DownloadList.get_one_song_file_names.cache_clear()
         DownloadList.get_all_song_ids.cache_clear()
-        SonglistParser()
+        SonglistParser.songs = {}
 
     def clear_download_token(self) -> None:
         '''清除过期下载链接'''

latest version/core/init.py

@@ -4,11 +4,12 @@ from importlib import import_module
 from json import load
 from shutil import copy, copy2
 from time import time
+from traceback import format_exc
 
 from core.config_manager import Config
 from core.constant import ARCAEA_SERVER_VERSION
 from core.course import Course
-from core.download import SonglistParser
+from core.download import DownloadList
 from core.purchase import Purchase
 from core.sql import Connect, DatabaseMigrator, MemoryDatabase
 from core.user import UserRegister
@@ -181,6 +182,7 @@ class LogDatabaseInit:
 
 
 class FileChecker:
+    '''文件检查及初始化类'''
 
     def __init__(self, logger=None):
         self.logger = logger
@@ -206,7 +208,8 @@ class FileChecker:
                 LogDatabaseInit().init()
                 self.logger.info(
                     f'Success to new the file {Config.SQLITE_LOG_DATABASE_PATH}')
-            except:
+            except Exception as e:
+                self.logger.error(format_exc())
                 self.logger.error(
                     f'Failed to new the file {Config.SQLITE_LOG_DATABASE_PATH}')
                 return False
@@ -218,7 +221,8 @@ class FileChecker:
                 DatabaseInit().init()
                 self.logger.info(
                     'Success to new the file `%s`.' % Config.SQLITE_DATABASE_PATH)
-            except:
+            except Exception as e:
+                self.logger.error(format_exc())
                 self.logger.warning(
                     'Failed to new the file `%s`.' % Config.SQLITE_DATABASE_PATH)
                 return False
@@ -262,7 +266,8 @@ class FileChecker:
                     self.logger.info(
                         'Success to update the file `%s`.' % Config.SQLITE_DATABASE_PATH)
 
-                except ValueError:
+                except Exception as e:
+                    self.logger.error(format_exc())
                     self.logger.warning(
                         'Fail to update the file `%s`.' % Config.SQLITE_DATABASE_PATH)
 
@@ -275,9 +280,20 @@ class FileChecker:
             DatabaseMigrator(old_path, new_path).update_database()
             os.remove(old_path)
 
+    def check_song_file(self) -> bool:
+        '''检查song有关文件并初始化缓存'''
+        f = self.check_folder(Config.SONG_FILE_FOLDER_PATH)
+        self.logger.info("Start to initialize song data...")
+        try:
+            DownloadList.initialize_cache()
+            self.logger.info('Complete!')
+        except Exception as e:
+            self.logger.error(format_exc())
+            self.logger.warning('Initialization error!')
+            f = False
+        return f
+
     def check_before_run(self) -> bool:
         '''运行前检查，返回布尔值'''
-        # TODO: try
         MemoryDatabase()  # 初始化内存数据库
-        SonglistParser()  # 解析songlist
+        return self.check_song_file() & self.check_update_database()
-        return self.check_folder(Config.SONG_FILE_FOLDER_PATH) & self.check_update_database()

latest version/core/operation.py

@@ -1,5 +1,6 @@
 from .sql import Connect, Sql
 from .score import Score
+from .download import DownloadList
 
 
 class BaseOperation:
@@ -53,3 +54,14 @@ class RefreshAllScoreRating(BaseOperation):
                     if values:
                         Sql(c).update_many('best_score', ['rating'], values, [
                             'user_id', 'song_id', 'difficulty'], where_values)
+
+
+class RefreshSongFileCache(BaseOperation):
+    '''
+        刷新歌曲文件缓存，包括文件hash缓存重建、文件目录重遍历、songlist重解析
+    '''
+    name = 'refresh_song_file_cache'
+
+    def run(self):
+        DownloadList.clear_all_cache()
+        DownloadList.initialize_cache()

latest version/core/song.py

@@ -1,4 +1,5 @@
 from .error import NoData
+from .config_manager import Config
 
 
 class Chart:
@@ -33,8 +34,10 @@ class Chart:
             '''select rating_pst, rating_prs, rating_ftr, rating_byn from chart where song_id=:a''', {'a': self.song_id})
         x = self.c.fetchone()
         if x is None:
-            self.defnum = -10
+            if Config.ALLOW_SCORE_WITH_NO_SONG:
-            # raise NoData('The song `%s` does not exist.' % self.song_id)
+                self.defnum = -10
+            else:
+                raise NoData(f'The song `{self.song_id}` does not exist.', 120)
         else:
             self.defnum = x[self.difficulty]
 

latest version/main.py

@@ -27,7 +27,7 @@ import server
 import web.index
 import web.login
 from core.constant import Constant
-from core.download import (UserDownload, initialize_songfile)
+from core.download import UserDownload
 from core.error import ArcError, NoAccess, RateLimit
 from core.init import FileChecker
 from core.sql import Connect
@@ -128,6 +128,18 @@ def tcp_server_run():
             app.run(Config.HOST, Config.PORT)
 
 
+def generate_log_file_dict(level: str, filename: str) -> dict:
+    return {
+        "class": "logging.handlers.RotatingFileHandler",
+        "maxBytes": 1024 * 1024,
+        "backupCount": 1,
+        "encoding": "utf-8",
+        "level": level,
+        "formatter": "default",
+        "filename": filename
+    }
+
+
 def main():
     log_dict = {
         'version': 1,
@@ -141,15 +153,7 @@ def main():
                 'stream': 'ext://flask.logging.wsgi_errors_stream',
                 'formatter': 'default'
             },
-            "error_file": {
+            "error_file": generate_log_file_dict('ERROR', './log/error.log')
-                "class": "logging.handlers.RotatingFileHandler",
-                "maxBytes": 1024 * 1024,
-                "backupCount": 1,
-                "encoding": "utf-8",
-                "level": "ERROR",
-                "formatter": "default",
-                "filename": "./log/error.log"
-            }
         },
         'formatters': {
             'default': {
@@ -159,51 +163,30 @@ def main():
     }
     if Config.ALLOW_INFO_LOG:
         log_dict['root']['handlers'].append('info_file')
-        log_dict['handlers']['info_file'] = {
+        log_dict['handlers']['info_file'] = generate_log_file_dict(
-            "class": "logging.handlers.RotatingFileHandler",
+            'INFO', './log/info.log')
-            "maxBytes": 1024 * 1024,
-            "backupCount": 1,
-            "encoding": "utf-8",
-            "level": "INFO",
-            "formatter": "default",
-            "filename": "./log/info.log"
-        }
     if Config.ALLOW_WARNING_LOG:
         log_dict['root']['handlers'].append('warning_file')
-        log_dict['handlers']['warning_file'] = {
+        log_dict['handlers']['warning_file'] = generate_log_file_dict(
-            "class": "logging.handlers.RotatingFileHandler",
+            'WARNING', './log/warning.log')
-            "maxBytes": 1024 * 1024,
-            "backupCount": 1,
-            "encoding": "utf-8",
-            "level": "WARNING",
-            "formatter": "default",
-            "filename": "./log/warning.log"
-        }
 
     dictConfig(log_dict)
 
     Connect.logger = app.logger
     if not FileChecker(app.logger).check_before_run():
-        app.logger.error('Something wrong. The server will not run.')
+        app.logger.error('Some errors occurred. The server will not run.')
         input('Press ENTER key to exit.')
         sys.exit()
 
-    app.logger.info("Start to initialize song data...")
-    try:
-        initialize_songfile()
-        app.logger.info('Complete!')
-    except:
-        app.logger.warning('Initialization error!')
-
     if Config.LINKPLAY_HOST and Config.SET_LINKPLAY_SERVER_AS_SUB_PROCESS:
         from linkplay_server import link_play
         process = [Process(target=link_play, args=(
             Config.LINKPLAY_HOST, int(Config.LINKPLAY_UDP_PORT), int(Config.LINKPLAY_TCP_PORT)))]
         [p.start() for p in process]
-        app.logger.info("Link Play UDP server is running on " +
+        app.logger.info(
-                        Config.LINKPLAY_HOST + ':' + str(Config.LINKPLAY_UDP_PORT) + " ...")
+            f"Link Play UDP server is running on {Config.LINKPLAY_HOST}:{Config.LINKPLAY_UDP_PORT} ...")
-        app.logger.info("Link Play TCP server is running on " +
+        app.logger.info(
-                        Config.LINKPLAY_HOST + ':' + str(Config.LINKPLAY_TCP_PORT) + " ...")
+            f"Link Play TCP server is running on {Config.LINKPLAY_HOST}:{Config.LINKPLAY_TCP_PORT} ...")
         tcp_server_run()
         [p.join() for p in process]
     else:

latest version/server/auth.py

@@ -5,7 +5,7 @@ from core.config_manager import Config
 from core.error import ArcError, NoAccess
 from core.sql import Connect
 from core.user import UserAuth, UserLogin
-from flask import Blueprint, jsonify, request
+from flask import Blueprint, g, jsonify, request
 
 from .func import arc_try, error_return
 
@@ -55,6 +55,7 @@ def auth_required(request):
                     user = UserAuth(c)
                     user.token = headers['Authorization'][7:]
                     user_id = user.token_get_id()
+                    g.user = user
                 except ArcError as e:
                     return error_return(e)
             return view(user_id, *args, **kwargs)

latest version/server/func.py

@@ -3,7 +3,7 @@ from traceback import format_exc
 
 from core.config_manager import Config
 from core.error import ArcError
-from flask import current_app, jsonify
+from flask import current_app, g, jsonify
 
 default_error = ArcError('Unknown Error', status=500)
 
@@ -83,6 +83,9 @@ def arc_try(view):
         except ArcError as e:
             if Config.ALLOW_WARNING_LOG:
                 current_app.logger.warning(format_exc())
+            user = g.get("user", None)
+            current_app.logger.warning(
+                f'{user.user_id if user is not None else ""} - {e.error_code}|{e.api_error_code}: {e}')
             return error_return(e)
 
     return wrapped_view

latest version/web/index.py

@@ -1,9 +1,8 @@
 import os
 import time
 
-from core.download import DownloadList, initialize_songfile
 from core.init import FileChecker
-from core.operation import RefreshAllScoreRating
+from core.operation import RefreshAllScoreRating, RefreshSongFileCache
 from core.rank import RankList
 from core.sql import Connect
 from flask import Blueprint, flash, redirect, render_template, request, url_for
@@ -290,8 +289,7 @@ def update_database():
 def update_song_hash():
     # 更新数据库内谱面文件hash值
     try:
-        DownloadList.clear_all_cache()
+        RefreshSongFileCache().run()
-        initialize_songfile()
         flash('数据刷新成功 Success refresh data.')
     except:
         flash('Something error!')