diff --git a/latest version/core/download.py b/latest version/core/download.py index 5858d2d..7caaf6b 100644 --- a/latest version/core/download.py +++ b/latest version/core/download.py @@ -1,6 +1,4 @@ import os -import sqlite3 -import threading from functools import lru_cache from json import loads from time import time @@ -15,171 +13,12 @@ from .user import User from .util import get_file_md5, md5 -class SongFileCache: - _local = threading.local() - _schema_lock = threading.Lock() - - @classmethod - def db_path(cls) -> str: - return os.path.join(os.path.dirname(Config.SQLITE_DATABASE_PATH) or '.', 'song_cache.db') - - @classmethod - def _conn(cls) -> sqlite3.Connection: - conn = getattr(cls._local, 'conn', None) - if conn is not None: - return conn - os.makedirs(os.path.dirname(cls.db_path()) or '.', exist_ok=True) - conn = sqlite3.connect(cls.db_path(), isolation_level=None, check_same_thread=False) - conn.row_factory = sqlite3.Row - cls._local.conn = conn - return conn - - @classmethod - def ensure_schema(cls) -> None: - with cls._schema_lock: - c = cls._conn() - c.execute('PRAGMA journal_mode=WAL;') - c.execute('PRAGMA synchronous=NORMAL;') - c.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);') - c.execute('CREATE TABLE IF NOT EXISTS songs (song_id TEXT PRIMARY KEY, dir_mtime_ns INTEGER NOT NULL, last_scan INTEGER NOT NULL);') - c.execute('CREATE TABLE IF NOT EXISTS files (song_id TEXT NOT NULL, file_name TEXT NOT NULL, size INTEGER NOT NULL, mtime_ns INTEGER NOT NULL, md5 TEXT, last_seen INTEGER NOT NULL, PRIMARY KEY (song_id, file_name));') - c.execute('CREATE INDEX IF NOT EXISTS idx_files_song_id ON files(song_id);') - - @classmethod - def _delete_song(cls, song_id: str) -> None: - cls.ensure_schema() - c = cls._conn() - c.execute('DELETE FROM files WHERE song_id=?;', (song_id,)) - c.execute('DELETE FROM songs WHERE song_id=?;', (song_id,)) - - @classmethod - def _delete_file(cls, song_id: str, file_name: str) -> None: - cls.ensure_schema() - c = cls._conn() - c.execute('DELETE FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name)) - - @classmethod - def sync_song(cls, song_id: str, dir_mtime_ns: int = None) -> None: - cls.ensure_schema() - song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id) - if not os.path.isdir(song_dir): - cls._delete_song(song_id) - return - if dir_mtime_ns is None: - dir_mtime_ns = os.stat(song_dir).st_mtime_ns - c = cls._conn() - row = c.execute('SELECT dir_mtime_ns FROM songs WHERE song_id=?;', (song_id,)).fetchone() - if row is not None and int(row['dir_mtime_ns']) == int(dir_mtime_ns): - return - now = int(time()) - c.execute('INSERT OR REPLACE INTO songs(song_id, dir_mtime_ns, last_scan) VALUES(?,?,?);', (song_id, int(dir_mtime_ns), now)) - file_names = [] - try: - for entry in os.scandir(song_dir): - if entry.is_file() and SonglistParser.is_available_file(song_id, entry.name): - file_names.append(entry.name) - except FileNotFoundError: - cls._delete_song(song_id) - return - if not file_names: - c.execute('DELETE FROM files WHERE song_id=?;', (song_id,)) - return - placeholders = ','.join(['?'] * len(file_names)) - c.execute(f'DELETE FROM files WHERE song_id=? AND file_name NOT IN ({placeholders});', (song_id, *file_names)) - for file_name in file_names: - path = os.path.join(song_dir, file_name) - if not os.path.isfile(path): - cls._delete_file(song_id, file_name) - continue - st = os.stat(path) - existing = c.execute('SELECT size, mtime_ns, md5 FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name)).fetchone() - if existing is not None and int(existing['size']) == int(st.st_size) and int(existing['mtime_ns']) == int(st.st_mtime_ns): - if Config.SONG_FILE_HASH_PRE_CALCULATE and not existing['md5']: - c.execute('UPDATE files SET md5=?, last_seen=? WHERE song_id=? AND file_name=?;', (get_file_md5(path), now, song_id, file_name)) - else: - c.execute('UPDATE files SET last_seen=? WHERE song_id=? AND file_name=?;', (now, song_id, file_name)) - continue - md5_value = get_file_md5(path) if Config.SONG_FILE_HASH_PRE_CALCULATE else None - c.execute('INSERT OR REPLACE INTO files(song_id, file_name, size, mtime_ns, md5, last_seen) VALUES(?,?,?,?,?,?);', (song_id, file_name, int(st.st_size), int(st.st_mtime_ns), md5_value, now)) - - @classmethod - def sync_all(cls) -> None: - cls.ensure_schema() - root_dir = Constant.SONG_FILE_FOLDER_PATH - if not os.path.isdir(root_dir): - return - song_ids = [entry.name for entry in os.scandir(root_dir) if entry.is_dir() and entry.name != '.' and entry.name != '..'] - for song_id in song_ids: - song_dir = os.path.join(root_dir, song_id) - try: - dir_mtime_ns = os.stat(song_dir).st_mtime_ns - except FileNotFoundError: - continue - cls.sync_song(song_id, dir_mtime_ns) - c = cls._conn() - if song_ids: - placeholders = ','.join(['?'] * len(song_ids)) - c.execute(f'DELETE FROM songs WHERE song_id NOT IN ({placeholders});', song_ids) - c.execute(f'DELETE FROM files WHERE song_id NOT IN ({placeholders});', song_ids) - else: - c.execute('DELETE FROM songs;') - c.execute('DELETE FROM files;') - - @classmethod - def get_all_song_ids(cls, root_mtime_ns: int) -> list: - cls.ensure_schema() - c = cls._conn() - meta = c.execute('SELECT value FROM meta WHERE key="root_mtime_ns";').fetchone() - if meta is None or int(meta['value']) != int(root_mtime_ns): - cls.sync_all() - c.execute('INSERT OR REPLACE INTO meta(key, value) VALUES(?,?);', ('root_mtime_ns', str(int(root_mtime_ns)))) - return [row['song_id'] for row in c.execute('SELECT song_id FROM songs ORDER BY song_id;').fetchall()] - - @classmethod - def get_song_file_names(cls, song_id: str, dir_mtime_ns: int) -> list: - cls.ensure_schema() - c = cls._conn() - row = c.execute('SELECT dir_mtime_ns FROM songs WHERE song_id=?;', (song_id,)).fetchone() - if row is None or int(row['dir_mtime_ns']) != int(dir_mtime_ns): - cls.sync_song(song_id, dir_mtime_ns) - return [r['file_name'] for r in c.execute('SELECT file_name FROM files WHERE song_id=? ORDER BY file_name;', (song_id,)).fetchall()] - - @classmethod - def get_song_file_md5(cls, song_id: str, file_name: str, file_mtime_ns: int, file_size: int) -> str: - cls.ensure_schema() - path = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id, file_name) - if not os.path.isfile(path): - cls._delete_file(song_id, file_name) - return None - c = cls._conn() - row = c.execute('SELECT size, mtime_ns, md5 FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name)).fetchone() - now = int(time()) - if row is not None and int(row['size']) == int(file_size) and int(row['mtime_ns']) == int(file_mtime_ns) and row['md5']: - c.execute('UPDATE files SET last_seen=? WHERE song_id=? AND file_name=?;', (now, song_id, file_name)) - return row['md5'] - md5_value = get_file_md5(path) - c.execute('INSERT OR REPLACE INTO files(song_id, file_name, size, mtime_ns, md5, last_seen) VALUES(?,?,?,?,?,?);', (song_id, file_name, int(file_size), int(file_mtime_ns), md5_value, now)) - song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id) - if os.path.isdir(song_dir): - try: - c.execute('INSERT OR IGNORE INTO songs(song_id, dir_mtime_ns, last_scan) VALUES(?,?,?);', (song_id, int(os.stat(song_dir).st_mtime_ns), now)) - except FileNotFoundError: - pass - return md5_value - - @lru_cache(maxsize=8192) -def _get_song_file_md5_cached(song_id: str, file_name: str, file_mtime_ns: int, file_size: int) -> str: - return SongFileCache.get_song_file_md5(song_id, file_name, file_mtime_ns, file_size) - - def get_song_file_md5(song_id: str, file_name: str) -> str: path = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id, file_name) if not os.path.isfile(path): - SongFileCache._delete_file(song_id, file_name) return None - st = os.stat(path) - return _get_song_file_md5_cached(song_id, file_name, int(st.st_mtime_ns), int(st.st_size)) + return get_file_md5(path) class SonglistParser: @@ -396,15 +235,18 @@ class DownloadList(UserDownload): def initialize_cache(cls) -> None: '''初始化歌曲数据缓存，包括md5、文件目录遍历、解析songlist''' SonglistParser() - SongFileCache.ensure_schema() - SongFileCache.sync_all() + if Config.SONG_FILE_HASH_PRE_CALCULATE: + x = cls() + x.url_flag = False + x.add_songs() + del x @staticmethod def clear_all_cache() -> None: '''清除所有歌曲文件有关缓存''' - _get_song_file_md5_cached.cache_clear() - DownloadList._get_cached_song_file_names.cache_clear() - DownloadList._get_all_cached_song_ids.cache_clear() + get_song_file_md5.cache_clear() + DownloadList.get_one_song_file_names.cache_clear() + DownloadList.get_all_song_ids.cache_clear() SonglistParser.songs = {} SonglistParser.pack_info = {} SonglistParser.free_songs = set() @@ -423,22 +265,14 @@ class DownloadList(UserDownload): @staticmethod @lru_cache(maxsize=2048) - def _get_cached_song_file_names(song_id: str, dir_mtime_ns: int) -> list: - '''获取一个歌曲文件夹下的所有合法文件名，有lru缓存''' - return SongFileCache.get_song_file_names(song_id, dir_mtime_ns) - - @staticmethod def get_one_song_file_names(song_id: str) -> list: - song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id) - if not os.path.isdir(song_dir): - SongFileCache._delete_song(song_id) - return [] - try: - dir_mtime_ns = os.stat(song_dir).st_mtime_ns - except FileNotFoundError: - SongFileCache._delete_song(song_id) - return [] - return DownloadList._get_cached_song_file_names(song_id, int(dir_mtime_ns)) + '''获取一个歌曲文件夹下的所有合法文件名，有lru缓存''' + r = [] + for i in os.scandir(os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id)): + file_name = i.name + if i.is_file() and SonglistParser.is_available_file(song_id, file_name): + r.append(file_name) + return r def add_one_song(self, song_id: str) -> None: @@ -487,20 +321,9 @@ class DownloadList(UserDownload): @staticmethod @lru_cache() - def _get_all_cached_song_ids(root_mtime_ns: int) -> list: - '''获取全歌曲文件夹列表，有lru缓存''' - return SongFileCache.get_all_song_ids(root_mtime_ns) - - @staticmethod def get_all_song_ids() -> list: - root_dir = Constant.SONG_FILE_FOLDER_PATH - if not os.path.isdir(root_dir): - return [] - try: - root_mtime_ns = os.stat(root_dir).st_mtime_ns - except FileNotFoundError: - return [] - return DownloadList._get_all_cached_song_ids(int(root_mtime_ns)) + '''获取全歌曲文件夹列表，有lru缓存''' + return [i.name for i in os.scandir(Constant.SONG_FILE_FOLDER_PATH) if i.is_dir()] def add_songs(self, song_ids: list = None) -> None: '''添加一个或多个歌曲到下载列表，若`song_ids`为空，则添加所有歌曲''' diff --git a/latest version/core/init.py b/latest version/core/init.py index 86c3835..affa6f1 100644 --- a/latest version/core/init.py +++ b/latest version/core/init.py @@ -327,8 +327,7 @@ class FileChecker: def check_song_file(self) -> bool: '''检查song有关文件并初始化缓存''' f = self.check_folder(Config.SONG_FILE_FOLDER_PATH) - song_cache_path = os.path.join(os.path.dirname(Config.SQLITE_DATABASE_PATH) or '.', 'song_cache.db') - self.logger.info("Initialize song data..." if not os.path.isfile(song_cache_path) else "Reusing song data cache...") + self.logger.info("Initialize song data...") try: DownloadList.initialize_cache() if not Config.SONG_FILE_HASH_PRE_CALCULATE: