From edbf01daa1d42ff3babce3d02664a0b33650ea21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=92=95=E8=B0=B7=E9=85=B1?= Date: Tue, 23 Sep 2025 01:34:43 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=B0=B1=E9=9D=A2=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/dependencies/beatmapset_cache.py | 16 ++ app/router/v2/beatmapset.py | 67 ++++++-- app/service/beatmapset_cache_service.py | 205 ++++++++++++++++++++++++ 3 files changed, 279 insertions(+), 9 deletions(-) create mode 100644 app/dependencies/beatmapset_cache.py create mode 100644 app/service/beatmapset_cache_service.py diff --git a/app/dependencies/beatmapset_cache.py b/app/dependencies/beatmapset_cache.py new file mode 100644 index 0000000..f5ac96b --- /dev/null +++ b/app/dependencies/beatmapset_cache.py @@ -0,0 +1,16 @@ +""" +Beatmapset缓存服务依赖注入 +""" + +from __future__ import annotations + +from app.dependencies.database import get_redis +from app.service.beatmapset_cache_service import BeatmapsetCacheService, get_beatmapset_cache_service + +from fastapi import Depends +from redis.asyncio import Redis + + +def get_beatmapset_cache_dependency(redis: Redis = Depends(get_redis)) -> BeatmapsetCacheService: + """获取beatmapset缓存服务依赖""" + return get_beatmapset_cache_service(redis) diff --git a/app/router/v2/beatmapset.py b/app/router/v2/beatmapset.py index 2ddb1b1..f3c6166 100644 --- a/app/router/v2/beatmapset.py +++ b/app/router/v2/beatmapset.py @@ -7,6 +7,7 @@ from urllib.parse import parse_qs from app.database import Beatmap, Beatmapset, BeatmapsetResp, FavouriteBeatmapset, User from app.database.beatmapset import SearchBeatmapsetsResp from app.dependencies.beatmap_download import get_beatmap_download_service +from app.dependencies.beatmapset_cache import get_beatmapset_cache_dependency from app.dependencies.database import Database, get_redis, with_db from app.dependencies.fetcher import get_fetcher from app.dependencies.geoip import get_client_ip, get_geoip_helper @@ -15,6 +16,7 @@ from app.fetcher import Fetcher from app.models.beatmap import SearchQueryModel from app.service.asset_proxy_helper import process_response_assets from app.service.beatmap_download_service import BeatmapDownloadService +from app.service.beatmapset_cache_service import BeatmapsetCacheService, generate_hash from .router import router @@ -54,6 +56,7 @@ async def search_beatmapset( current_user: User = Security(get_current_user, scopes=["public"]), fetcher: Fetcher = Depends(get_fetcher), redis=Depends(get_redis), + cache_service: BeatmapsetCacheService = Depends(get_beatmapset_cache_dependency), ): params = parse_qs(qs=request.url.query, keep_blank_values=True) cursor = {} @@ -94,15 +97,31 @@ async def search_beatmapset( ): # TODO: search locally return SearchBeatmapsetsResp(total=0, beatmapsets=[]) + + # 生成查询和游标的哈希用于缓存 + query_hash = generate_hash(query.model_dump()) + cursor_hash = generate_hash(cursor) + + # 尝试从缓存获取搜索结果 + cached_result = await cache_service.get_search_from_cache(query_hash, cursor_hash) + if cached_result: + sets = SearchBeatmapsetsResp(**cached_result) + # 处理资源代理 + processed_sets = await process_response_assets(sets, request) + return processed_sets + try: sets = await fetcher.search_beatmapset(query, cursor, redis) background_tasks.add_task(_save_to_db, sets) + # 缓存搜索结果 + await cache_service.cache_search_result(query_hash, cursor_hash, sets.model_dump()) + # 处理资源代理 processed_sets = await process_response_assets(sets, request) return processed_sets except HTTPError as e: - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e @router.get( @@ -118,13 +137,27 @@ async def lookup_beatmapset( beatmap_id: int = Query(description="谱面 ID"), current_user: User = Security(get_current_user, scopes=["public"]), fetcher: Fetcher = Depends(get_fetcher), + cache_service: BeatmapsetCacheService = Depends(get_beatmapset_cache_dependency), ): - beatmap = await Beatmap.get_or_fetch(db, fetcher, bid=beatmap_id) - resp = await BeatmapsetResp.from_db(beatmap.beatmapset, session=db, user=current_user) + # 先尝试从缓存获取 + cached_resp = await cache_service.get_beatmap_lookup_from_cache(beatmap_id) + if cached_resp: + # 处理资源代理 + processed_resp = await process_response_assets(cached_resp, request) + return processed_resp - # 处理资源代理 - processed_resp = await process_response_assets(resp, request) - return processed_resp + try: + beatmap = await Beatmap.get_or_fetch(db, fetcher, bid=beatmap_id) + resp = await BeatmapsetResp.from_db(beatmap.beatmapset, session=db, user=current_user) + + # 缓存结果 + await cache_service.cache_beatmap_lookup(beatmap_id, resp) + + # 处理资源代理 + processed_resp = await process_response_assets(resp, request) + return processed_resp + except HTTPError as exc: + raise HTTPException(status_code=404, detail="Beatmap not found") from exc @router.get( @@ -136,15 +169,31 @@ async def lookup_beatmapset( ) async def get_beatmapset( db: Database, + request: Request, beatmapset_id: int = Path(..., description="谱面集 ID"), current_user: User = Security(get_current_user, scopes=["public"]), fetcher: Fetcher = Depends(get_fetcher), + cache_service: BeatmapsetCacheService = Depends(get_beatmapset_cache_dependency), ): + # 先尝试从缓存获取 + cached_resp = await cache_service.get_beatmapset_from_cache(beatmapset_id) + if cached_resp: + # 处理资源代理 + processed_resp = await process_response_assets(cached_resp, request) + return processed_resp + try: beatmapset = await Beatmapset.get_or_fetch(db, fetcher, beatmapset_id) - return await BeatmapsetResp.from_db(beatmapset, session=db, include=["recent_favourites"], user=current_user) - except HTTPError: - raise HTTPException(status_code=404, detail="Beatmapset not found") + resp = await BeatmapsetResp.from_db(beatmapset, session=db, include=["recent_favourites"], user=current_user) + + # 缓存结果 + await cache_service.cache_beatmapset(resp) + + # 处理资源代理 + processed_resp = await process_response_assets(resp, request) + return processed_resp + except HTTPError as exc: + raise HTTPException(status_code=404, detail="Beatmapset not found") from exc @router.get( diff --git a/app/service/beatmapset_cache_service.py b/app/service/beatmapset_cache_service.py new file mode 100644 index 0000000..4bcd01b --- /dev/null +++ b/app/service/beatmapset_cache_service.py @@ -0,0 +1,205 @@ +""" +Beatmapset缓存服务 +用于缓存beatmapset数据，减少数据库查询频率 +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime +from typing import TYPE_CHECKING + +from app.config import settings +from app.database.beatmapset import BeatmapsetResp +from app.log import logger + +from redis.asyncio import Redis + +if TYPE_CHECKING: + from app.fetcher import Fetcher + + +class DateTimeEncoder(json.JSONEncoder): + """处理datetime序列化的JSON编码器""" + + def default(self, obj): + if isinstance(obj, datetime): + return obj.isoformat() + return super().default(obj) + + +def safe_json_dumps(data) -> str: + """安全的JSON序列化，处理datetime对象""" + return json.dumps(data, cls=DateTimeEncoder, ensure_ascii=False) + + +def generate_hash(data) -> str: + """生成数据的MD5哈希值""" + if isinstance(data, str): + content = data + else: + content = safe_json_dumps(data) + return hashlib.md5(content.encode()).hexdigest() + + +class BeatmapsetCacheService: + """Beatmapset缓存服务""" + + def __init__(self, redis: Redis): + self.redis = redis + self._default_ttl = getattr(settings, "beatmapset_cache_expire_seconds", 3600) # 1小时默认TTL + + def _get_beatmapset_cache_key(self, beatmapset_id: int) -> str: + """生成beatmapset缓存键""" + return f"beatmapset:{beatmapset_id}" + + def _get_beatmap_lookup_cache_key(self, beatmap_id: int) -> str: + """生成beatmap lookup缓存键""" + return f"beatmap_lookup:{beatmap_id}:beatmapset" + + def _get_search_cache_key(self, query_hash: str, cursor_hash: str) -> str: + """生成搜索结果缓存键""" + return f"beatmapset_search:{query_hash}:{cursor_hash}" + + async def get_beatmapset_from_cache(self, beatmapset_id: int) -> BeatmapsetResp | None: + """从缓存获取beatmapset信息""" + try: + cache_key = self._get_beatmapset_cache_key(beatmapset_id) + cached_data = await self.redis.get(cache_key) + if cached_data: + logger.debug(f"Beatmapset cache hit for {beatmapset_id}") + data = json.loads(cached_data) + return BeatmapsetResp(**data) + return None + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error getting beatmapset from cache: {e}") + return None + + async def cache_beatmapset( + self, + beatmapset_resp: BeatmapsetResp, + expire_seconds: int | None = None, + ): + """缓存beatmapset信息""" + try: + if expire_seconds is None: + expire_seconds = self._default_ttl + if beatmapset_resp.id is None: + logger.warning("Cannot cache beatmapset with None id") + return + cache_key = self._get_beatmapset_cache_key(beatmapset_resp.id) + cached_data = beatmapset_resp.model_dump_json() + await self.redis.setex(cache_key, expire_seconds, cached_data) # type: ignore + logger.debug(f"Cached beatmapset {beatmapset_resp.id} for {expire_seconds}s") + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error caching beatmapset: {e}") + + async def get_beatmap_lookup_from_cache(self, beatmap_id: int) -> BeatmapsetResp | None: + """从缓存获取通过beatmap ID查找的beatmapset信息""" + try: + cache_key = self._get_beatmap_lookup_cache_key(beatmap_id) + cached_data = await self.redis.get(cache_key) + if cached_data: + logger.debug(f"Beatmap lookup cache hit for {beatmap_id}") + data = json.loads(cached_data) + return BeatmapsetResp(**data) + return None + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error getting beatmap lookup from cache: {e}") + return None + + async def cache_beatmap_lookup( + self, + beatmap_id: int, + beatmapset_resp: BeatmapsetResp, + expire_seconds: int | None = None, + ): + """缓存通过beatmap ID查找的beatmapset信息""" + try: + if expire_seconds is None: + expire_seconds = self._default_ttl + cache_key = self._get_beatmap_lookup_cache_key(beatmap_id) + cached_data = beatmapset_resp.model_dump_json() + await self.redis.setex(cache_key, expire_seconds, cached_data) # type: ignore + logger.debug(f"Cached beatmap lookup {beatmap_id} for {expire_seconds}s") + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error caching beatmap lookup: {e}") + + async def get_search_from_cache(self, query_hash: str, cursor_hash: str) -> dict | None: + """从缓存获取搜索结果""" + try: + cache_key = self._get_search_cache_key(query_hash, cursor_hash) + cached_data = await self.redis.get(cache_key) + if cached_data: + logger.debug(f"Search cache hit for {query_hash[:8]}...{cursor_hash[:8]}") + return json.loads(cached_data) + return None + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error getting search from cache: {e}") + return None + + async def cache_search_result( + self, + query_hash: str, + cursor_hash: str, + search_result: dict, + expire_seconds: int | None = None, + ): + """缓存搜索结果""" + try: + if expire_seconds is None: + expire_seconds = min(self._default_ttl, 300) # 搜索结果缓存时间较短，最多5分钟 + cache_key = self._get_search_cache_key(query_hash, cursor_hash) + cached_data = safe_json_dumps(search_result) + await self.redis.setex(cache_key, expire_seconds, cached_data) # type: ignore + logger.debug(f"Cached search result for {expire_seconds}s") + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error caching search result: {e}") + + async def invalidate_beatmapset_cache(self, beatmapset_id: int): + """使beatmapset缓存失效""" + try: + cache_key = self._get_beatmapset_cache_key(beatmapset_id) + await self.redis.delete(cache_key) + logger.debug(f"Invalidated beatmapset cache for {beatmapset_id}") + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error invalidating beatmapset cache: {e}") + + async def invalidate_beatmap_lookup_cache(self, beatmap_id: int): + """使beatmap lookup缓存失效""" + try: + cache_key = self._get_beatmap_lookup_cache_key(beatmap_id) + await self.redis.delete(cache_key) + logger.debug(f"Invalidated beatmap lookup cache for {beatmap_id}") + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error invalidating beatmap lookup cache: {e}") + + async def get_cache_stats(self) -> dict: + """获取缓存统计信息""" + try: + beatmapset_keys = await self.redis.keys("beatmapset:*") + lookup_keys = await self.redis.keys("beatmap_lookup:*") + search_keys = await self.redis.keys("beatmapset_search:*") + + return { + "cached_beatmapsets": len(beatmapset_keys), + "cached_lookups": len(lookup_keys), + "cached_searches": len(search_keys), + "total_keys": len(beatmapset_keys) + len(lookup_keys) + len(search_keys), + } + except (ValueError, TypeError, AttributeError) as e: + logger.error(f"Error getting cache stats: {e}") + return {"error": str(e)} + + +# 全局缓存服务实例 +_cache_service: BeatmapsetCacheService | None = None + + +def get_beatmapset_cache_service(redis: Redis) -> BeatmapsetCacheService: + """获取beatmapset缓存服务实例""" + global _cache_service # noqa: PLW0603 + if _cache_service is None: + _cache_service = BeatmapsetCacheService(redis) + return _cache_service