This commit is contained in:
咕谷酱
2025-08-22 00:07:19 +08:00
parent bade8658ed
commit 80d4237c5d
22 changed files with 423 additions and 356 deletions

View File

@@ -1,7 +1,6 @@
from __future__ import annotations
import time
from typing import Optional
from app.dependencies.database import get_redis
from app.log import logger
@@ -11,6 +10,7 @@ from httpx import AsyncClient
class TokenAuthError(Exception):
"""Token 授权失败异常"""
pass
@@ -55,7 +55,7 @@ class BaseFetcher:
return await self._request_with_retry(url, method, **kwargs)
async def _request_with_retry(
self, url: str, method: str = "GET", max_retries: Optional[int] = None, **kwargs
self, url: str, method: str = "GET", max_retries: int | None = None, **kwargs
) -> dict:
"""
带重试机制的请求方法
@@ -64,7 +64,7 @@ class BaseFetcher:
max_retries = self.max_retries
last_error = None
for attempt in range(max_retries + 1):
try:
# 检查 token 是否过期
@@ -126,7 +126,9 @@ class BaseFetcher:
)
continue
else:
logger.error(f"Request failed after {max_retries + 1} attempts: {e}")
logger.error(
f"Request failed after {max_retries + 1} attempts: {e}"
)
break
# 如果所有重试都失败了
@@ -194,9 +196,13 @@ class BaseFetcher:
f"fetcher:refresh_token:{self.client_id}",
self.refresh_token,
)
logger.info(f"Successfully refreshed access token for client {self.client_id}")
logger.info(
f"Successfully refreshed access token for client {self.client_id}"
)
except Exception as e:
logger.error(f"Failed to refresh access token for client {self.client_id}: {e}")
logger.error(
f"Failed to refresh access token for client {self.client_id}: {e}"
)
# 清除无效的 token要求重新授权
self.access_token = ""
self.refresh_token = ""
@@ -204,7 +210,9 @@ class BaseFetcher:
redis = get_redis()
await redis.delete(f"fetcher:access_token:{self.client_id}")
await redis.delete(f"fetcher:refresh_token:{self.client_id}")
logger.warning(f"Cleared invalid tokens. Please re-authorize: {self.authorize_url}")
logger.warning(
f"Cleared invalid tokens. Please re-authorize: {self.authorize_url}"
)
raise
async def _trigger_reauthorization(self) -> None:
@@ -216,18 +224,18 @@ class BaseFetcher:
f"Authentication failed after {self._auth_retry_count} attempts. "
f"Triggering reauthorization for client {self.client_id}"
)
# 清除内存中的 token
self.access_token = ""
self.refresh_token = ""
self.token_expiry = 0
self._auth_retry_count = 0 # 重置重试计数器
# 清除 Redis 中的 token
redis = get_redis()
await redis.delete(f"fetcher:access_token:{self.client_id}")
await redis.delete(f"fetcher:refresh_token:{self.client_id}")
logger.warning(
f"All tokens cleared for client {self.client_id}. "
f"Please re-authorize using: {self.authorize_url}"

View File

@@ -101,6 +101,7 @@ class BeatmapsetFetcher(BaseFetcher):
return json.loads(cursor_json)
except Exception:
return {}
async def get_beatmapset(self, beatmap_set_id: int) -> BeatmapsetResp:
logger.opt(colors=True).debug(
f"<blue>[BeatmapsetFetcher]</blue> get_beatmapset: <y>{beatmap_set_id}</y>"
@@ -164,9 +165,7 @@ class BeatmapsetFetcher(BaseFetcher):
# 将结果缓存 15 分钟
cache_ttl = 15 * 60 # 15 分钟
await redis_client.set(
cache_key,
json.dumps(api_response, separators=(",", ":")),
ex=cache_ttl
cache_key, json.dumps(api_response, separators=(",", ":")), ex=cache_ttl
)
logger.opt(colors=True).debug(
@@ -178,10 +177,12 @@ class BeatmapsetFetcher(BaseFetcher):
# 智能预取只在用户明确搜索时才预取避免过多API请求
# 且只在有搜索词或特定条件时预取,避免首页浏览时的过度预取
if (api_response.get("cursor") and
(query.q or query.s != "leaderboard" or cursor)):
if api_response.get("cursor") and (
query.q or query.s != "leaderboard" or cursor
):
# 在后台预取下1页减少预取量
import asyncio
# 不立即创建任务,而是延迟一段时间再预取
async def delayed_prefetch():
await asyncio.sleep(3.0) # 延迟3秒
@@ -200,8 +201,11 @@ class BeatmapsetFetcher(BaseFetcher):
return resp
async def prefetch_next_pages(
self, query: SearchQueryModel, current_cursor: Cursor,
redis_client: redis.Redis, pages: int = 3
self,
query: SearchQueryModel,
current_cursor: Cursor,
redis_client: redis.Redis,
pages: int = 3,
) -> None:
"""预取下几页内容"""
if not current_cursor:
@@ -269,7 +273,7 @@ class BeatmapsetFetcher(BaseFetcher):
await redis_client.set(
next_cache_key,
json.dumps(api_response, separators=(",", ":")),
ex=prefetch_ttl
ex=prefetch_ttl,
)
logger.opt(colors=True).debug(
@@ -317,7 +321,6 @@ class BeatmapsetFetcher(BaseFetcher):
params=params,
)
if api_response.get("cursor"):
cursor_dict = api_response["cursor"]
api_response["cursor_string"] = self._encode_cursor(cursor_dict)
@@ -327,7 +330,7 @@ class BeatmapsetFetcher(BaseFetcher):
await redis_client.set(
cache_key,
json.dumps(api_response, separators=(",", ":")),
ex=cache_ttl
ex=cache_ttl,
)
logger.opt(colors=True).info(
@@ -335,7 +338,6 @@ class BeatmapsetFetcher(BaseFetcher):
f"Warmed up cache for {query.sort} (TTL: {cache_ttl}s)"
)
if api_response.get("cursor"):
await self.prefetch_next_pages(
query, api_response["cursor"], redis_client, pages=2