refactor(fetcher): implement passive rate limiter for API requests

app/fetcher/_base.py

@@ -1,10 +1,11 @@
 import asyncio
+from datetime import datetime
 import time
 
 from app.dependencies.database import get_redis
 from app.log import fetcher_logger
 
-from httpx import AsyncClient
+from httpx import AsyncClient, HTTPStatusError
 
 
 class TokenAuthError(Exception):
@@ -13,10 +14,63 @@ class TokenAuthError(Exception):
     pass
 
 
+class PassiveRateLimiter:
+    """
+    被动速率限制器
+    当收到 429 响应时，读取 Retry-After 头并暂停所有请求
+    """
+
+    def __init__(self):
+        self._lock = asyncio.Lock()
+        self._retry_after_time: float | None = None
+        self._waiting_tasks: set[asyncio.Task] = set()
+
+    async def wait_if_limited(self) -> None:
+        """如果正在限流中，等待限流解除"""
+        async with self._lock:
+            if self._retry_after_time is not None:
+                current_time = time.time()
+                if current_time < self._retry_after_time:
+                    wait_seconds = self._retry_after_time - current_time
+                    logger.warning(f"Rate limited, waiting {wait_seconds:.2f} seconds")
+                    await asyncio.sleep(wait_seconds)
+                    self._retry_after_time = None
+
+    async def handle_rate_limit(self, retry_after: str | int | None) -> None:
+        """
+        处理 429 响应，设置限流时间
+
+        Args:
+            retry_after: Retry-After 头的值，可以是秒数或 HTTP 日期
+        """
+        async with self._lock:
+            if retry_after is None:
+                # 如果没有 Retry-After 头，默认等待 60 秒
+                wait_seconds = 60
+            elif isinstance(retry_after, int):
+                wait_seconds = retry_after
+            elif retry_after.isdigit():
+                wait_seconds = int(retry_after)
+            else:
+                # 尝试解析 HTTP 日期格式
+                try:
+                    retry_time = datetime.strptime(retry_after, "%a, %d %b %Y %H:%M:%S %Z")
+                    wait_seconds = max(0, (retry_time - datetime.utcnow()).total_seconds())
+                except ValueError:
+                    # 解析失败，默认等待 60 秒
+                    wait_seconds = 60
+
+            self._retry_after_time = time.time() + wait_seconds
+            logger.warning(f"Rate limit triggered, will retry after {wait_seconds} seconds")
+
+
 logger = fetcher_logger("Fetcher")
 
 
 class BaseFetcher:
+    # 类级别的 rate limiter，所有实例共享
+    _rate_limiter = PassiveRateLimiter()
+
     def __init__(
         self,
         client_id: str,
@@ -51,7 +105,7 @@ class BaseFetcher:
 
     async def request_api(self, url: str, method: str = "GET", **kwargs) -> dict:
         """
-        发送 API 请求
+        发送 API 请求，支持被动速率限制
         """
         await self.ensure_valid_access_token()
 
@@ -59,24 +113,41 @@ class BaseFetcher:
         attempt = 0
 
         while attempt < 2:
+            # 在发送请求前等待速率限制
+            await self._rate_limiter.wait_if_limited()
+
             request_headers = {**headers, **self.header}
             request_kwargs = kwargs.copy()
 
             async with AsyncClient() as client:
-                response = await client.request(
-                    method,
-                    url,
-                    headers=request_headers,
-                    **request_kwargs,
-                )
+                try:
+                    response = await client.request(
+                        method,
+                        url,
+                        headers=request_headers,
+                        **request_kwargs,
+                    )
+                    response.raise_for_status()
+                    return response.json()
 
-            if response.status_code != 401:
-                response.raise_for_status()
-                return response.json()
+                except HTTPStatusError as e:
+                    # 处理 429 速率限制响应
+                    if e.response.status_code == 429:
+                        retry_after = e.response.headers.get("Retry-After")
+                        logger.warning(f"Rate limited for {url}, Retry-After: {retry_after}")
+                        await self._rate_limiter.handle_rate_limit(retry_after)
+                        # 速率限制后重试当前请求（不增加 attempt）
+                        continue
 
-            attempt += 1
-            logger.warning(f"Received 401 error for {url}, attempt {attempt}")
-            await self._handle_unauthorized()
+                    # 处理 401 未授权响应
+                    if e.response.status_code == 401:
+                        attempt += 1
+                        logger.warning(f"Received 401 error for {url}, attempt {attempt}")
+                        await self._handle_unauthorized()
+                        continue
+
+                    # 其他 HTTP 错误直接抛出
+                    raise
 
         await self._clear_access_token()
         logger.warning(f"Failed to authorize after retries for {url}, cleaned up tokens")