diff --git a/latest version/core/bgtask.py b/latest version/core/bgtask.py index 109330b..cfb3a32 100644 --- a/latest version/core/bgtask.py +++ b/latest version/core/bgtask.py @@ -35,6 +35,16 @@ def logdb_execute_func(sql, *args, **kwargs): c.execute(sql, *args, **kwargs) +def logdb_execute_many_func(sql, *args, **kwargs): + with Connect(Constant.SQLITE_LOG_DATABASE_PATH) as c: + c.executemany(sql, *args, **kwargs) + + def logdb_execute(sql: str, *args, **kwargs): '''异步执行SQL，日志库写入，注意不会直接返回结果''' return BGTask(logdb_execute_func, sql, *args, **kwargs) + + +def logdb_execute_many(sql: str, *args, **kwargs): + '''异步批量执行SQL，日志库写入，注意不会直接返回结果''' + return BGTask(logdb_execute_many_func, sql, *args, **kwargs) diff --git a/latest version/core/constant.py b/latest version/core/constant.py index a9f8130..4dbd378 100644 --- a/latest version/core/constant.py +++ b/latest version/core/constant.py @@ -1,6 +1,7 @@ from .config_manager import Config ARCAEA_SERVER_VERSION = 'v2.11.1.3' +ARCAEA_LOG_DATBASE_VERSION = 'v1.1' class Constant: @@ -101,4 +102,6 @@ class Constant: DATABASE_MIGRATE_TABLES = ['user', 'friend', 'best_score', 'recent30', 'user_world', 'item', 'user_item', 'purchase', 'purchase_item', 'user_save', 'login', 'present', 'user_present', 'present_item', 'redeem', 'user_redeem', 'redeem_item', 'api_login', 'chart', 'user_course', 'user_char', 'user_role'] + LOG_DATABASE_MIGRATE_TABLES = ['cache', 'user_score', 'user_rating'] + UPDATE_WITH_NEW_CHARACTER_DATA = Config.UPDATE_WITH_NEW_CHARACTER_DATA diff --git a/latest version/core/init.py b/latest version/core/init.py index fa8d172..9510803 100644 --- a/latest version/core/init.py +++ b/latest version/core/init.py @@ -7,11 +7,12 @@ from time import time from traceback import format_exc from core.config_manager import Config -from core.constant import ARCAEA_SERVER_VERSION +from core.constant import ARCAEA_LOG_DATBASE_VERSION, ARCAEA_SERVER_VERSION from core.course import Course from core.download import DownloadList from core.purchase import Purchase -from core.sql import Connect, DatabaseMigrator, MemoryDatabase +from core.sql import (Connect, DatabaseMigrator, LogDatabaseMigrator, + MemoryDatabase) from core.user import UserRegister from core.util import try_rename @@ -208,6 +209,29 @@ class FileChecker: self.logger.error( f'Failed to new the file {Config.SQLITE_LOG_DATABASE_PATH}') return False + else: + # 检查更新 + with Connect(Config.SQLITE_LOG_DATABASE_PATH) as c: + try: + x = c.execute( + '''select value from cache where key="version"''').fetchone() + except: + x = None + if not x or x[0] != ARCAEA_LOG_DATBASE_VERSION: + self.logger.warning( + f'Maybe the file `{Config.SQLITE_LOG_DATABASE_PATH}` is an old version.') + try: + self.logger.info( + f'Try to update the file `{Config.SQLITE_LOG_DATABASE_PATH}`') + self.update_log_database() + self.logger.info( + f'Success to update the file `{Config.SQLITE_LOG_DATABASE_PATH}`') + except Exception as e: + self.logger.error(format_exc()) + self.logger.error( + f'Failed to update the file `{Config.SQLITE_LOG_DATABASE_PATH}`') + return False + if not self.check_file(Config.SQLITE_DATABASE_PATH): # 新建数据库 try: @@ -275,6 +299,12 @@ class FileChecker: DatabaseMigrator(old_path, new_path).update_database() os.remove(old_path) + @staticmethod + def update_log_database(old_path: str = Config.SQLITE_LOG_DATABASE_PATH) -> None: + '''直接更新日志数据库''' + if os.path.isfile(old_path): + LogDatabaseMigrator(old_path).update_database() + def check_song_file(self) -> bool: '''检查song有关文件并初始化缓存''' f = self.check_folder(Config.SONG_FILE_FOLDER_PATH) diff --git a/latest version/core/score.py b/latest version/core/score.py index ddabb54..f3d9155 100644 --- a/latest version/core/score.py +++ b/latest version/core/score.py @@ -2,14 +2,15 @@ from base64 import b64encode from os import urandom from time import time -from .bgtask import logdb_execute +from .bgtask import BGTask, logdb_execute +from .config_manager import Config from .constant import Constant from .course import CoursePlay from .error import NoData, StaminaNotEnough from .item import ItemCore from .song import Chart -from .sql import Query, Sql -from .util import md5 +from .sql import Connect, Query, Sql +from .util import get_today_timestamp, md5 from .world import WorldPlay @@ -431,6 +432,18 @@ class UserPlay(UserScore): logdb_execute('''insert into user_score values(?,?,?,?,?,?,?,?,?,?,?,?,?)''', (self.user.user_id, self.song.song_id, self.song.difficulty, self.time_played, self.score, self.shiny_perfect_count, self.perfect_count, self.near_count, self.miss_count, self.health, self.modifier, self.clear_type, self.rating)) + def record_rating_ptt(self, user_rating_ptt: float) -> None: + '''向log数据库记录用户ptt变化''' + today_timestamp = get_today_timestamp() + with Connect(Config.SQLITE_LOG_DATABASE_PATH) as c2: + old_ptt = c2.execute('''select rating_ptt from user_rating where user_id=? and time=?''', ( + self.user.user_id, today_timestamp)).fetchone() + + old_ptt = 0 if old_ptt is None else old_ptt[0] + if old_ptt != user_rating_ptt: + c2.execute('''insert or replace into user_rating values(?,?,?)''', + (self.user.user_id, today_timestamp, user_rating_ptt)) + def upload_score(self) -> None: '''上传分数，包括user的recent更新，best更新，recent30更新，世界模式计算''' self.get_play_state() @@ -474,10 +487,11 @@ class UserPlay(UserScore): self.update_recent30() # 总PTT更新 - self.user.rating_ptt = int(self.ptt.value * 100) + user_rating_ptt = self.ptt.value + self.user.rating_ptt = int(user_rating_ptt * 100) + BGTask(self.record_rating_ptt, user_rating_ptt) # 记录总PTT变换 self.c.execute('''update user set rating_ptt = :a where user_id = :b''', { 'a': self.user.rating_ptt, 'b': self.user.user_id}) - # TODO: PTT log # 世界模式判断 if self.is_world_mode: diff --git a/latest version/core/sql.py b/latest version/core/sql.py index 66f31b0..f39ed4f 100644 --- a/latest version/core/sql.py +++ b/latest version/core/sql.py @@ -1,8 +1,10 @@ +import os import sqlite3 import traceback from atexit import register -from .constant import Constant +from .config_manager import Config +from .constant import ARCAEA_LOG_DATBASE_VERSION, Constant from .error import ArcError, InputError @@ -404,6 +406,31 @@ class DatabaseMigrator: self.update_user_char_full(c2) # 更新user_char_full +class LogDatabaseMigrator: + + def __init__(self, c1_path: str = Config.SQLITE_LOG_DATABASE_PATH) -> None: + self.c1_path = c1_path + # self.c2_path = c2_path + self.init_folder_path = Config.DATABASE_INIT_PATH + self.c = None + + @property + def sql_path(self) -> str: + return os.path.join(self.init_folder_path, 'log_tables.sql') + + def table_update(self) -> None: + '''直接更新数据库结构''' + with open(self.sql_path, 'r') as f: + self.c.executescript(f.read()) + self.c.execute( + '''insert or replace into cache values("version", :a, -1);''', {'a': ARCAEA_LOG_DATBASE_VERSION}) + + def update_database(self) -> None: + with Connect(self.c1_path) as c: + self.c = c + self.table_update() + + class MemoryDatabase: conn = sqlite3.connect('file:arc_tmp?mode=memory&cache=shared', uri=True) diff --git a/latest version/core/user.py b/latest version/core/user.py index 2718369..c35179d 100644 --- a/latest version/core/user.py +++ b/latest version/core/user.py @@ -763,7 +763,7 @@ class UserChanger(UserInfo, UserRegister): if columns is not None: d = {} for column in columns: - if column == 'password': + if column == 'password' and self.password != '': d[column] = self.hash_pwd else: d[column] = self.__dict__[column] diff --git a/latest version/core/util.py b/latest version/core/util.py index b0ca0b8..0236906 100644 --- a/latest version/core/util.py +++ b/latest version/core/util.py @@ -1,5 +1,7 @@ import hashlib import os +from datetime import date +from time import mktime def md5(code: str) -> str: @@ -37,3 +39,8 @@ def try_rename(path: str, new_path: str) -> str: os.rename(path, final_path) return final_path + + +def get_today_timestamp(): + '''相对于本机本地时间的今天0点的时间戳''' + return int(mktime(date.today().timetuple())) diff --git a/latest version/database/init/log_tables.sql b/latest version/database/init/log_tables.sql index aa6cd06..9801a5b 100644 --- a/latest version/database/init/log_tables.sql +++ b/latest version/database/init/log_tables.sql @@ -17,6 +17,11 @@ clear_type int, rating real, primary key(user_id, song_id, difficulty, time_played) ); +create table if not exists user_rating(user_id int, +time int, +rating_ptt real, +primary key(user_id, time) +); create index if not exists user_score_1 on user_score (song_id, difficulty); create index if not exists user_score_2 on user_score (time_played); diff --git a/tools/clean_logdb.py b/tools/clean_logdb.py new file mode 100644 index 0000000..268c84e --- /dev/null +++ b/tools/clean_logdb.py @@ -0,0 +1,121 @@ +import os +import sqlite3 +from datetime import datetime +from time import mktime + +LOG_DATABASE_PATH = './arcaea_log.db' + + +class Connect(): + # 数据库连接类，上下文管理 + + def __init__(self, file_path=LOG_DATABASE_PATH): + """ + 数据库连接 + 接受：文件路径 + 返回：sqlite3连接操作对象 + """ + self.file_path = file_path + + def __enter__(self): + self.conn = sqlite3.connect(self.file_path) + self.c = self.conn.cursor() + return self.c + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + if exc_type is not None: + print(f'exc_type: {exc_type}') + print(f'exc_val: {exc_val}') + print(f'exc_tb: {exc_tb}') + if self.conn: + self.conn.rollback() + + if self.conn: + self.conn.commit() + self.conn.close() + + return True + + +def clean_by_time(table_name: str, time_colume_name: str, colume_count: int): + with Connect() as c: + today = datetime.now() + print(f'The time now is {today}.') + day = input( + 'Please input the number of days the data before which you want to delete: ') + try: + day = int(day) + except ValueError: + print('Invalid input!') + return + time = mktime(today.timetuple()) - day * 24 * 60 * 60 + delete_count = c.execute( + f'select count(*) from {table_name} where {time_colume_name} < ?', (time,)).fetchone()[0] + all_count = c.execute( + f'select count(*) from {table_name}').fetchone()[0] + print( + f'Before {day} days, there are {delete_count} records to be deleted, {all_count} records in total.') + flag = input('Are you sure to delete these records? (y/n) ') + if flag == 'y' or flag == 'Y': + if delete_count >= 1000000: + print( + 'It will cost a long time to delete these records, please wait patiently...') + print('Deleting...') + c.execute('PRAGMA cache_size = 32768') + c.execute('PRAGMA synchronous = OFF') + c.execute('PRAGMA temp_store = MEMORY') + if delete_count / all_count >= 0.8: + data = c.execute( + f'select * from {table_name} where {time_colume_name} > ?', (time,)).fetchall() + c.execute(f'delete from {table_name}') + c.executemany( + f'insert into {table_name} values ({",".join(["?"]*colume_count)})', data) + else: + c.execute( + f'delete from {table_name} where {time_colume_name} < ?', (time,)) + c.execute('PRAGMA temp_store = DEFAULT') + print('Delete successfully!') + else: + print('Delete canceled!') + + +def vacuum(): + print('This operation will release unused space in the database file.') + print('It will cost a long time to release unused space if the database file is so large.') + flag = input('Are you sure to release unused space? (y/n) ') + if flag == 'y' or flag == 'Y': + with Connect() as c: + print('Releasing unused space...') + c.execute('vacuum') + print('Release unused space successfully!') + else: + print('Release unused space canceled!') + + +def main(): + if not os.path.exists(LOG_DATABASE_PATH): + print('The database file `arcaea_log.db` does not exist!') + print('-- Arcaea Server Log Database Cleaner --') + print('Note: It is more recommended to delete the database file directly.') + while True: + print('-' * 40) + print('1. clean `user_score` table') + print('2. clean `user_rating` table') + print('3. release unused space (`vacuum` command)') + print('0. exit') + choice = input('Please input your choice: ') + if choice == '1': + clean_by_time('user_score', 'time_played', 13) + elif choice == '2': + clean_by_time('user_rating', 'time', 3) + elif choice == '3': + vacuum() + elif choice == '0': + break + else: + print('Invalid choice!') + + +if __name__ == '__main__': + main() + input('Press `Enter` key to exit.')