#234

latest version/core/download.py

@@ -1,6 +1,4 @@
 import os
-import sqlite3
-import threading
 from functools import lru_cache
 from json import loads
 from time import time
@@ -15,171 +13,12 @@ from .user import User
 from .util import get_file_md5, md5
 
 
-class SongFileCache:
-    _local = threading.local()
-    _schema_lock = threading.Lock()
-
-    @classmethod
-    def db_path(cls) -> str:
-        return os.path.join(os.path.dirname(Config.SQLITE_DATABASE_PATH) or '.', 'song_cache.db')
-
-    @classmethod
-    def _conn(cls) -> sqlite3.Connection:
-        conn = getattr(cls._local, 'conn', None)
-        if conn is not None:
-            return conn
-        os.makedirs(os.path.dirname(cls.db_path()) or '.', exist_ok=True)
-        conn = sqlite3.connect(cls.db_path(), isolation_level=None, check_same_thread=False)
-        conn.row_factory = sqlite3.Row
-        cls._local.conn = conn
-        return conn
-
-    @classmethod
-    def ensure_schema(cls) -> None:
-        with cls._schema_lock:
-            c = cls._conn()
-            c.execute('PRAGMA journal_mode=WAL;')
-            c.execute('PRAGMA synchronous=NORMAL;')
-            c.execute('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);')
-            c.execute('CREATE TABLE IF NOT EXISTS songs (song_id TEXT PRIMARY KEY, dir_mtime_ns INTEGER NOT NULL, last_scan INTEGER NOT NULL);')
-            c.execute('CREATE TABLE IF NOT EXISTS files (song_id TEXT NOT NULL, file_name TEXT NOT NULL, size INTEGER NOT NULL, mtime_ns INTEGER NOT NULL, md5 TEXT, last_seen INTEGER NOT NULL, PRIMARY KEY (song_id, file_name));')
-            c.execute('CREATE INDEX IF NOT EXISTS idx_files_song_id ON files(song_id);')
-
-    @classmethod
-    def _delete_song(cls, song_id: str) -> None:
-        cls.ensure_schema()
-        c = cls._conn()
-        c.execute('DELETE FROM files WHERE song_id=?;', (song_id,))
-        c.execute('DELETE FROM songs WHERE song_id=?;', (song_id,))
-
-    @classmethod
-    def _delete_file(cls, song_id: str, file_name: str) -> None:
-        cls.ensure_schema()
-        c = cls._conn()
-        c.execute('DELETE FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name))
-
-    @classmethod
-    def sync_song(cls, song_id: str, dir_mtime_ns: int = None) -> None:
-        cls.ensure_schema()
-        song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id)
-        if not os.path.isdir(song_dir):
-            cls._delete_song(song_id)
-            return
-        if dir_mtime_ns is None:
-            dir_mtime_ns = os.stat(song_dir).st_mtime_ns
-        c = cls._conn()
-        row = c.execute('SELECT dir_mtime_ns FROM songs WHERE song_id=?;', (song_id,)).fetchone()
-        if row is not None and int(row['dir_mtime_ns']) == int(dir_mtime_ns):
-            return
-        now = int(time())
-        c.execute('INSERT OR REPLACE INTO songs(song_id, dir_mtime_ns, last_scan) VALUES(?,?,?);', (song_id, int(dir_mtime_ns), now))
-        file_names = []
-        try:
-            for entry in os.scandir(song_dir):
-                if entry.is_file() and SonglistParser.is_available_file(song_id, entry.name):
-                    file_names.append(entry.name)
-        except FileNotFoundError:
-            cls._delete_song(song_id)
-            return
-        if not file_names:
-            c.execute('DELETE FROM files WHERE song_id=?;', (song_id,))
-            return
-        placeholders = ','.join(['?'] * len(file_names))
-        c.execute(f'DELETE FROM files WHERE song_id=? AND file_name NOT IN ({placeholders});', (song_id, *file_names))
-        for file_name in file_names:
-            path = os.path.join(song_dir, file_name)
-            if not os.path.isfile(path):
-                cls._delete_file(song_id, file_name)
-                continue
-            st = os.stat(path)
-            existing = c.execute('SELECT size, mtime_ns, md5 FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name)).fetchone()
-            if existing is not None and int(existing['size']) == int(st.st_size) and int(existing['mtime_ns']) == int(st.st_mtime_ns):
-                if Config.SONG_FILE_HASH_PRE_CALCULATE and not existing['md5']:
-                    c.execute('UPDATE files SET md5=?, last_seen=? WHERE song_id=? AND file_name=?;', (get_file_md5(path), now, song_id, file_name))
-                else:
-                    c.execute('UPDATE files SET last_seen=? WHERE song_id=? AND file_name=?;', (now, song_id, file_name))
-                continue
-            md5_value = get_file_md5(path) if Config.SONG_FILE_HASH_PRE_CALCULATE else None
-            c.execute('INSERT OR REPLACE INTO files(song_id, file_name, size, mtime_ns, md5, last_seen) VALUES(?,?,?,?,?,?);', (song_id, file_name, int(st.st_size), int(st.st_mtime_ns), md5_value, now))
-
-    @classmethod
-    def sync_all(cls) -> None:
-        cls.ensure_schema()
-        root_dir = Constant.SONG_FILE_FOLDER_PATH
-        if not os.path.isdir(root_dir):
-            return
-        song_ids = [entry.name for entry in os.scandir(root_dir) if entry.is_dir() and entry.name != '.' and entry.name != '..']
-        for song_id in song_ids:
-            song_dir = os.path.join(root_dir, song_id)
-            try:
-                dir_mtime_ns = os.stat(song_dir).st_mtime_ns
-            except FileNotFoundError:
-                continue
-            cls.sync_song(song_id, dir_mtime_ns)
-        c = cls._conn()
-        if song_ids:
-            placeholders = ','.join(['?'] * len(song_ids))
-            c.execute(f'DELETE FROM songs WHERE song_id NOT IN ({placeholders});', song_ids)
-            c.execute(f'DELETE FROM files WHERE song_id NOT IN ({placeholders});', song_ids)
-        else:
-            c.execute('DELETE FROM songs;')
-            c.execute('DELETE FROM files;')
-
-    @classmethod
-    def get_all_song_ids(cls, root_mtime_ns: int) -> list:
-        cls.ensure_schema()
-        c = cls._conn()
-        meta = c.execute('SELECT value FROM meta WHERE key="root_mtime_ns";').fetchone()
-        if meta is None or int(meta['value']) != int(root_mtime_ns):
-            cls.sync_all()
-            c.execute('INSERT OR REPLACE INTO meta(key, value) VALUES(?,?);', ('root_mtime_ns', str(int(root_mtime_ns))))
-        return [row['song_id'] for row in c.execute('SELECT song_id FROM songs ORDER BY song_id;').fetchall()]
-
-    @classmethod
-    def get_song_file_names(cls, song_id: str, dir_mtime_ns: int) -> list:
-        cls.ensure_schema()
-        c = cls._conn()
-        row = c.execute('SELECT dir_mtime_ns FROM songs WHERE song_id=?;', (song_id,)).fetchone()
-        if row is None or int(row['dir_mtime_ns']) != int(dir_mtime_ns):
-            cls.sync_song(song_id, dir_mtime_ns)
-        return [r['file_name'] for r in c.execute('SELECT file_name FROM files WHERE song_id=? ORDER BY file_name;', (song_id,)).fetchall()]
-
-    @classmethod
-    def get_song_file_md5(cls, song_id: str, file_name: str, file_mtime_ns: int, file_size: int) -> str:
-        cls.ensure_schema()
-        path = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id, file_name)
-        if not os.path.isfile(path):
-            cls._delete_file(song_id, file_name)
-            return None
-        c = cls._conn()
-        row = c.execute('SELECT size, mtime_ns, md5 FROM files WHERE song_id=? AND file_name=?;', (song_id, file_name)).fetchone()
-        now = int(time())
-        if row is not None and int(row['size']) == int(file_size) and int(row['mtime_ns']) == int(file_mtime_ns) and row['md5']:
-            c.execute('UPDATE files SET last_seen=? WHERE song_id=? AND file_name=?;', (now, song_id, file_name))
-            return row['md5']
-        md5_value = get_file_md5(path)
-        c.execute('INSERT OR REPLACE INTO files(song_id, file_name, size, mtime_ns, md5, last_seen) VALUES(?,?,?,?,?,?);', (song_id, file_name, int(file_size), int(file_mtime_ns), md5_value, now))
-        song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id)
-        if os.path.isdir(song_dir):
-            try:
-                c.execute('INSERT OR IGNORE INTO songs(song_id, dir_mtime_ns, last_scan) VALUES(?,?,?);', (song_id, int(os.stat(song_dir).st_mtime_ns), now))
-            except FileNotFoundError:
-                pass
-        return md5_value
-
-
 @lru_cache(maxsize=8192)
-def _get_song_file_md5_cached(song_id: str, file_name: str, file_mtime_ns: int, file_size: int) -> str:
-    return SongFileCache.get_song_file_md5(song_id, file_name, file_mtime_ns, file_size)
-
-
 def get_song_file_md5(song_id: str, file_name: str) -> str:
     path = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id, file_name)
     if not os.path.isfile(path):
-        SongFileCache._delete_file(song_id, file_name)
         return None
-    st = os.stat(path)
+    return get_file_md5(path)
-    return _get_song_file_md5_cached(song_id, file_name, int(st.st_mtime_ns), int(st.st_size))
 
 
 class SonglistParser:
@@ -396,15 +235,18 @@ class DownloadList(UserDownload):
     def initialize_cache(cls) -> None:
         '''初始化歌曲数据缓存，包括md5、文件目录遍历、解析songlist'''
         SonglistParser()
-        SongFileCache.ensure_schema()
+        if Config.SONG_FILE_HASH_PRE_CALCULATE:
-        SongFileCache.sync_all()
+            x = cls()
+            x.url_flag = False
+            x.add_songs()
+            del x
 
     @staticmethod
     def clear_all_cache() -> None:
         '''清除所有歌曲文件有关缓存'''
-        _get_song_file_md5_cached.cache_clear()
+        get_song_file_md5.cache_clear()
-        DownloadList._get_cached_song_file_names.cache_clear()
+        DownloadList.get_one_song_file_names.cache_clear()
-        DownloadList._get_all_cached_song_ids.cache_clear()
+        DownloadList.get_all_song_ids.cache_clear()
         SonglistParser.songs = {}
         SonglistParser.pack_info = {}
         SonglistParser.free_songs = set()
@@ -423,22 +265,14 @@ class DownloadList(UserDownload):
 
     @staticmethod
     @lru_cache(maxsize=2048)
-    def _get_cached_song_file_names(song_id: str, dir_mtime_ns: int) -> list:
-        '''获取一个歌曲文件夹下的所有合法文件名，有lru缓存'''
-        return SongFileCache.get_song_file_names(song_id, dir_mtime_ns)
-
-    @staticmethod
     def get_one_song_file_names(song_id: str) -> list:
-        song_dir = os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id)
+        '''获取一个歌曲文件夹下的所有合法文件名，有lru缓存'''
-        if not os.path.isdir(song_dir):
+        r = []
-            SongFileCache._delete_song(song_id)
+        for i in os.scandir(os.path.join(Constant.SONG_FILE_FOLDER_PATH, song_id)):
-            return []
+            file_name = i.name
-        try:
+            if i.is_file() and SonglistParser.is_available_file(song_id, file_name):
-            dir_mtime_ns = os.stat(song_dir).st_mtime_ns
+                r.append(file_name)
-        except FileNotFoundError:
+        return r
-            SongFileCache._delete_song(song_id)
-            return []
-        return DownloadList._get_cached_song_file_names(song_id, int(dir_mtime_ns))
 
     def add_one_song(self, song_id: str) -> None:
 
@@ -487,20 +321,9 @@ class DownloadList(UserDownload):
 
     @staticmethod
     @lru_cache()
-    def _get_all_cached_song_ids(root_mtime_ns: int) -> list:
-        '''获取全歌曲文件夹列表，有lru缓存'''
-        return SongFileCache.get_all_song_ids(root_mtime_ns)
-
-    @staticmethod
     def get_all_song_ids() -> list:
-        root_dir = Constant.SONG_FILE_FOLDER_PATH
+        '''获取全歌曲文件夹列表，有lru缓存'''
-        if not os.path.isdir(root_dir):
+        return [i.name for i in os.scandir(Constant.SONG_FILE_FOLDER_PATH) if i.is_dir()]
-            return []
-        try:
-            root_mtime_ns = os.stat(root_dir).st_mtime_ns
-        except FileNotFoundError:
-            return []
-        return DownloadList._get_all_cached_song_ids(int(root_mtime_ns))
 
     def add_songs(self, song_ids: list = None) -> None:
         '''添加一个或多个歌曲到下载列表，若`song_ids`为空，则添加所有歌曲'''

latest version/core/init.py

@@ -327,8 +327,7 @@ class FileChecker:
     def check_song_file(self) -> bool:
         '''检查song有关文件并初始化缓存'''
         f = self.check_folder(Config.SONG_FILE_FOLDER_PATH)
-        song_cache_path = os.path.join(os.path.dirname(Config.SQLITE_DATABASE_PATH) or '.', 'song_cache.db')
+        self.logger.info("Initialize song data...")
-        self.logger.info("Initialize song data..." if not os.path.isfile(song_cache_path) else "Reusing song data cache...")
         try:
             DownloadList.initialize_cache()
             if not Config.SONG_FILE_HASH_PRE_CALCULATE: