diff --git a/latest version/core/download.py b/latest version/core/download.py index 7caaf6b..5858d2d 100644 --- a/latest version/core/download.py +++ b/latest version/core/download.py @@ -1,4 +1,6 @@ import os +import sqlite3 +import threading from functools import lru_cache from json import loads from time import time @@ -13,12 +15,171 @@ from .user import User from .util import get_file_md5, md5 +class SongFileCache: + _local = threading.local() + _schema_lock = threading.Lock() + + @classmethod + def db_path(cls) -> str: + return os.path.join(os.path.dirname(Config.SQLITE_DATABASE_PATH) or '.', 'song_cache.db') + + @classmethod + def _conn(cls) -> sqlite3.Connection: + conn = getattr(cls._local, 'conn', None) + if conn is not None: + return conn + os.makedirs(os.path.dirname(cls.db_path()) or '.', exist_ok=True) + conn = sqlite3.connect(cls.db_path(), isolation_level=None, check_same_thread=False) + conn.row_factory = sqlite3.Row + cls._local.conn = conn + return conn + + @classmethod + def ensure_schema(cls) -> None: + with cls._schema_lock: + c = cls._conn() + c.execute('PRAGMA journal_mode=WAL;') + c.execute('PRAGMA synchronous=NORMAL;') + c.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);') + c.execute('CREATE TABLE IF NOT EXISTS songs (song_id TEXT PRIMARY KEY, dir_mtime_ns INTEGER NOT NULL, last_scan INTEGER NOT NULL);') + c.execute('CREATE TABLE IF NOT EXISTS files (song_id TEXT NOT NULL, file_name TEXT NOT NULL, size INTEGER NOT NULL, mtime_ns INTEGER NOT NULL, md5 TEXT, last_seen INTEGER NOT NULL, PRIMARY KEY (song_id, file_name));') + c.execute('CREATE INDEX IF NOT EXISTS idx_files_song_id ON files(song_id);') + + @classmethod + def _delete_song(cls, song_id: str) -> None: + cls.ensure_schema() + c = cls._conn() + c.execute('DELETE FROM files WHERE song_id=?;', (song_id,)) + c.execute('DELETE FROM songs WHERE song_id=?;', (song_id,)) + + @classmethod + def _delete_file(cls, song_id: str, file_name: str) -> None: + cls.ensure_schema() + c = cls._conn() + c.execute('DELETE FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name)) + + @classmethod + def sync_song(cls, song_id: str, dir_mtime_ns: int = None) -> None: + cls.ensure_schema() + song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id) + if not os.path.isdir(song_dir): + cls._delete_song(song_id) + return + if dir_mtime_ns is None: + dir_mtime_ns = os.stat(song_dir).st_mtime_ns + c = cls._conn() + row = c.execute('SELECT dir_mtime_ns FROM songs WHERE song_id=?;', (song_id,)).fetchone() + if row is not None and int(row['dir_mtime_ns']) == int(dir_mtime_ns): + return + now = int(time()) + c.execute('INSERT OR REPLACE INTO songs(song_id, dir_mtime_ns, last_scan) VALUES(?,?,?);', (song_id, int(dir_mtime_ns), now)) + file_names = [] + try: + for entry in os.scandir(song_dir): + if entry.is_file() and SonglistParser.is_available_file(song_id, entry.name): + file_names.append(entry.name) + except FileNotFoundError: + cls._delete_song(song_id) + return + if not file_names: + c.execute('DELETE FROM files WHERE song_id=?;', (song_id,)) + return + placeholders = ','.join(['?'] * len(file_names)) + c.execute(f'DELETE FROM files WHERE song_id=? AND file_name NOT IN ({placeholders});', (song_id, *file_names)) + for file_name in file_names: + path = os.path.join(song_dir, file_name) + if not os.path.isfile(path): + cls._delete_file(song_id, file_name) + continue + st = os.stat(path) + existing = c.execute('SELECT size, mtime_ns, md5 FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name)).fetchone() + if existing is not None and int(existing['size']) == int(st.st_size) and int(existing['mtime_ns']) == int(st.st_mtime_ns): + if Config.SONG_FILE_HASH_PRE_CALCULATE and not existing['md5']: + c.execute('UPDATE files SET md5=?, last_seen=? WHERE song_id=? AND file_name=?;', (get_file_md5(path), now, song_id, file_name)) + else: + c.execute('UPDATE files SET last_seen=? WHERE song_id=? AND file_name=?;', (now, song_id, file_name)) + continue + md5_value = get_file_md5(path) if Config.SONG_FILE_HASH_PRE_CALCULATE else None + c.execute('INSERT OR REPLACE INTO files(song_id, file_name, size, mtime_ns, md5, last_seen) VALUES(?,?,?,?,?,?);', (song_id, file_name, int(st.st_size), int(st.st_mtime_ns), md5_value, now)) + + @classmethod + def sync_all(cls) -> None: + cls.ensure_schema() + root_dir = Constant.SONG_FILE_FOLDER_PATH + if not os.path.isdir(root_dir): + return + song_ids = [entry.name for entry in os.scandir(root_dir) if entry.is_dir() and entry.name != '.' and entry.name != '..'] + for song_id in song_ids: + song_dir = os.path.join(root_dir, song_id) + try: + dir_mtime_ns = os.stat(song_dir).st_mtime_ns + except FileNotFoundError: + continue + cls.sync_song(song_id, dir_mtime_ns) + c = cls._conn() + if song_ids: + placeholders = ','.join(['?'] * len(song_ids)) + c.execute(f'DELETE FROM songs WHERE song_id NOT IN ({placeholders});', song_ids) + c.execute(f'DELETE FROM files WHERE song_id NOT IN ({placeholders});', song_ids) + else: + c.execute('DELETE FROM songs;') + c.execute('DELETE FROM files;') + + @classmethod + def get_all_song_ids(cls, root_mtime_ns: int) -> list: + cls.ensure_schema() + c = cls._conn() + meta = c.execute('SELECT value FROM meta WHERE key="root_mtime_ns";').fetchone() + if meta is None or int(meta['value']) != int(root_mtime_ns): + cls.sync_all() + c.execute('INSERT OR REPLACE INTO meta(key, value) VALUES(?,?);', ('root_mtime_ns', str(int(root_mtime_ns)))) + return [row['song_id'] for row in c.execute('SELECT song_id FROM songs ORDER BY song_id;').fetchall()] + + @classmethod + def get_song_file_names(cls, song_id: str, dir_mtime_ns: int) -> list: + cls.ensure_schema() + c = cls._conn() + row = c.execute('SELECT dir_mtime_ns FROM songs WHERE song_id=?;', (song_id,)).fetchone() + if row is None or int(row['dir_mtime_ns']) != int(dir_mtime_ns): + cls.sync_song(song_id, dir_mtime_ns) + return [r['file_name'] for r in c.execute('SELECT file_name FROM files WHERE song_id=? ORDER BY file_name;', (song_id,)).fetchall()] + + @classmethod + def get_song_file_md5(cls, song_id: str, file_name: str, file_mtime_ns: int, file_size: int) -> str: + cls.ensure_schema() + path = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id, file_name) + if not os.path.isfile(path): + cls._delete_file(song_id, file_name) + return None + c = cls._conn() + row = c.execute('SELECT size, mtime_ns, md5 FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name)).fetchone() + now = int(time()) + if row is not None and int(row['size']) == int(file_size) and int(row['mtime_ns']) == int(file_mtime_ns) and row['md5']: + c.execute('UPDATE files SET last_seen=? WHERE song_id=? AND file_name=?;', (now, song_id, file_name)) + return row['md5'] + md5_value = get_file_md5(path) + c.execute('INSERT OR REPLACE INTO files(song_id, file_name, size, mtime_ns, md5, last_seen) VALUES(?,?,?,?,?,?);', (song_id, file_name, int(file_size), int(file_mtime_ns), md5_value, now)) + song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id) + if os.path.isdir(song_dir): + try: + c.execute('INSERT OR IGNORE INTO songs(song_id, dir_mtime_ns, last_scan) VALUES(?,?,?);', (song_id, int(os.stat(song_dir).st_mtime_ns), now)) + except FileNotFoundError: + pass + return md5_value + + @lru_cache(maxsize=8192) +def _get_song_file_md5_cached(song_id: str, file_name: str, file_mtime_ns: int, file_size: int) -> str: + return SongFileCache.get_song_file_md5(song_id, file_name, file_mtime_ns, file_size) + + def get_song_file_md5(song_id: str, file_name: str) -> str: path = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id, file_name) if not os.path.isfile(path): + SongFileCache._delete_file(song_id, file_name) return None - return get_file_md5(path) + st = os.stat(path) + return _get_song_file_md5_cached(song_id, file_name, int(st.st_mtime_ns), int(st.st_size)) class SonglistParser: @@ -235,18 +396,15 @@ class DownloadList(UserDownload): def initialize_cache(cls) -> None: '''初始化歌曲数据缓存,包括md5、文件目录遍历、解析songlist''' SonglistParser() - if Config.SONG_FILE_HASH_PRE_CALCULATE: - x = cls() - x.url_flag = False - x.add_songs() - del x + SongFileCache.ensure_schema() + SongFileCache.sync_all() @staticmethod def clear_all_cache() -> None: '''清除所有歌曲文件有关缓存''' - get_song_file_md5.cache_clear() - DownloadList.get_one_song_file_names.cache_clear() - DownloadList.get_all_song_ids.cache_clear() + _get_song_file_md5_cached.cache_clear() + DownloadList._get_cached_song_file_names.cache_clear() + DownloadList._get_all_cached_song_ids.cache_clear() SonglistParser.songs = {} SonglistParser.pack_info = {} SonglistParser.free_songs = set() @@ -265,14 +423,22 @@ class DownloadList(UserDownload): @staticmethod @lru_cache(maxsize=2048) - def get_one_song_file_names(song_id: str) -> list: + def _get_cached_song_file_names(song_id: str, dir_mtime_ns: int) -> list: '''获取一个歌曲文件夹下的所有合法文件名,有lru缓存''' - r = [] - for i in os.scandir(os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id)): - file_name = i.name - if i.is_file() and SonglistParser.is_available_file(song_id, file_name): - r.append(file_name) - return r + return SongFileCache.get_song_file_names(song_id, dir_mtime_ns) + + @staticmethod + def get_one_song_file_names(song_id: str) -> list: + song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id) + if not os.path.isdir(song_dir): + SongFileCache._delete_song(song_id) + return [] + try: + dir_mtime_ns = os.stat(song_dir).st_mtime_ns + except FileNotFoundError: + SongFileCache._delete_song(song_id) + return [] + return DownloadList._get_cached_song_file_names(song_id, int(dir_mtime_ns)) def add_one_song(self, song_id: str) -> None: @@ -321,9 +487,20 @@ class DownloadList(UserDownload): @staticmethod @lru_cache() - def get_all_song_ids() -> list: + def _get_all_cached_song_ids(root_mtime_ns: int) -> list: '''获取全歌曲文件夹列表,有lru缓存''' - return [i.name for i in os.scandir(Constant.SONG_FILE_FOLDER_PATH) if i.is_dir()] + return SongFileCache.get_all_song_ids(root_mtime_ns) + + @staticmethod + def get_all_song_ids() -> list: + root_dir = Constant.SONG_FILE_FOLDER_PATH + if not os.path.isdir(root_dir): + return [] + try: + root_mtime_ns = os.stat(root_dir).st_mtime_ns + except FileNotFoundError: + return [] + return DownloadList._get_all_cached_song_ids(int(root_mtime_ns)) def add_songs(self, song_ids: list = None) -> None: '''添加一个或多个歌曲到下载列表,若`song_ids`为空,则添加所有歌曲''' diff --git a/latest version/core/init.py b/latest version/core/init.py index affa6f1..86c3835 100644 --- a/latest version/core/init.py +++ b/latest version/core/init.py @@ -327,7 +327,8 @@ class FileChecker: def check_song_file(self) -> bool: '''检查song有关文件并初始化缓存''' f = self.check_folder(Config.SONG_FILE_FOLDER_PATH) - self.logger.info("Initialize song data...") + song_cache_path = os.path.join(os.path.dirname(Config.SQLITE_DATABASE_PATH) or '.', 'song_cache.db') + self.logger.info("Initialize song data..." if not os.path.isfile(song_cache_path) else "Reusing song data cache...") try: DownloadList.initialize_cache() if not Config.SONG_FILE_HASH_PRE_CALCULATE: