Bennett/g0v0-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

整理代码

Browse Source
This commit is contained in:
咕谷酱
2025-08-22 05:57:28 +08:00
parent ad131c0158
commit ce465aa049
20 changed files with 1078 additions and 799 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/database/score.py
View File

@@ -45,7 +45,7 @@ from .relationship import (
) )
from .score_token import ScoreToken from .score_token import ScoreToken
from pydantic import field_validator, field_serializer from pydantic import field_serializer, field_validator
from redis.asyncio import Redis from redis.asyncio import Redis
from sqlalchemy import Boolean, Column, ColumnExpressionArgument, DateTime from sqlalchemy import Boolean, Column, ColumnExpressionArgument, DateTime
from sqlalchemy.ext.asyncio import AsyncAttrs from sqlalchemy.ext.asyncio import AsyncAttrs
@@ -126,7 +126,7 @@ class ScoreBase(AsyncAttrs, SQLModel, UTCBaseModel):
if isinstance(v, dict): if isinstance(v, dict):
serialized = {} serialized = {}
for key, value in v.items(): for key, value in v.items():
if hasattr(key, 'value'): if hasattr(key, "value"):
# 如果是枚举，使用其值 # 如果是枚举，使用其值
serialized[key.value] = value serialized[key.value] = value
else: else:
@@ -138,7 +138,7 @@ class ScoreBase(AsyncAttrs, SQLModel, UTCBaseModel):
@field_serializer("rank", when_used="json") @field_serializer("rank", when_used="json")
def serialize_rank(self, v): def serialize_rank(self, v):
"""序列化等级，确保枚举值正确转换为字符串""" """序列化等级，确保枚举值正确转换为字符串"""
if hasattr(v, 'value'): if hasattr(v, "value"):
return v.value return v.value
return str(v) return str(v)
@@ -188,7 +188,7 @@ class Score(ScoreBase, table=True):
@field_serializer("gamemode", when_used="json") @field_serializer("gamemode", when_used="json")
def serialize_gamemode(self, v): def serialize_gamemode(self, v):
"""序列化游戏模式，确保枚举值正确转换为字符串""" """序列化游戏模式，确保枚举值正确转换为字符串"""
if hasattr(v, 'value'): if hasattr(v, "value"):
return v.value return v.value
return str(v) return str(v)
@@ -281,7 +281,7 @@ class ScoreResp(ScoreBase):
if isinstance(v, dict): if isinstance(v, dict):
serialized = {} serialized = {}
for key, value in v.items(): for key, value in v.items():
if hasattr(key, 'value'): if hasattr(key, "value"):
# 如果是枚举，使用其值 # 如果是枚举，使用其值
serialized[key.value] = value serialized[key.value] = value
else: else:

6
app/dependencies/database.py
View File

@@ -10,8 +10,8 @@ from app.config import settings
from fastapi import Depends from fastapi import Depends
from pydantic import BaseModel from pydantic import BaseModel
import redis.asyncio as redis
import redis as sync_redis import redis as sync_redis
import redis.asyncio as redis
from sqlalchemy.ext.asyncio import create_async_engine from sqlalchemy.ext.asyncio import create_async_engine
from sqlmodel import SQLModel from sqlmodel import SQLModel
from sqlmodel.ext.asyncio.session import AsyncSession from sqlmodel.ext.asyncio.session import AsyncSession
@@ -40,7 +40,9 @@ engine = create_async_engine(
redis_client = redis.from_url(settings.redis_url, decode_responses=True) redis_client = redis.from_url(settings.redis_url, decode_responses=True)
# Redis 消息缓存连接 (db1) - 使用同步客户端在线程池中执行 # Redis 消息缓存连接 (db1) - 使用同步客户端在线程池中执行
redis_message_client = sync_redis.from_url(settings.redis_url, decode_responses=True, db=1) redis_message_client = sync_redis.from_url(
settings.redis_url, decode_responses=True, db=1
)
# 数据库依赖 # 数据库依赖

6
app/models/score.py
View File

@@ -7,7 +7,7 @@ from app.config import settings
from .mods import API_MODS, APIMod from .mods import API_MODS, APIMod
from pydantic import BaseModel, Field, ValidationInfo, field_validator, field_serializer from pydantic import BaseModel, Field, ValidationInfo, field_serializer, field_validator
if TYPE_CHECKING: if TYPE_CHECKING:
import rosu_pp_py as rosu import rosu_pp_py as rosu
@@ -212,7 +212,7 @@ class SoloScoreSubmissionInfo(BaseModel):
if isinstance(v, dict): if isinstance(v, dict):
serialized = {} serialized = {}
for key, value in v.items(): for key, value in v.items():
if hasattr(key, 'value'): if hasattr(key, "value"):
# 如果是枚举，使用其值 # 如果是枚举，使用其值
serialized[key.value] = value serialized[key.value] = value
else: else:
@@ -224,7 +224,7 @@ class SoloScoreSubmissionInfo(BaseModel):
@field_serializer("rank", when_used="json") @field_serializer("rank", when_used="json")
def serialize_rank(self, v): def serialize_rank(self, v):
"""序列化等级，确保枚举值正确转换为字符串""" """序列化等级，确保枚举值正确转换为字符串"""
if hasattr(v, 'value'): if hasattr(v, "value"):
return v.value return v.value
return str(v) return str(v)

5
app/models/stats.py
View File

@@ -1,13 +1,13 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime from datetime import datetime
from typing import Any
from pydantic import BaseModel from pydantic import BaseModel
class OnlineStats(BaseModel): class OnlineStats(BaseModel):
"""在线统计信息""" """在线统计信息"""
registered_users: int registered_users: int
online_users: int online_users: int
playing_users: int playing_users: int
@@ -16,6 +16,7 @@ class OnlineStats(BaseModel):
class OnlineHistoryPoint(BaseModel): class OnlineHistoryPoint(BaseModel):
"""在线历史数据点""" """在线历史数据点"""
timestamp: datetime timestamp: datetime
online_count: int online_count: int
playing_count: int playing_count: int
@@ -23,12 +24,14 @@ class OnlineHistoryPoint(BaseModel):
class OnlineHistoryStats(BaseModel): class OnlineHistoryStats(BaseModel):
"""24小时在线历史统计""" """24小时在线历史统计"""
history: list[OnlineHistoryPoint] history: list[OnlineHistoryPoint]
current_stats: OnlineStats current_stats: OnlineStats
class ServerStatistics(BaseModel): class ServerStatistics(BaseModel):
"""服务器统计信息""" """服务器统计信息"""
total_users: int total_users: int
online_users: int online_users: int
playing_users: int playing_users: int

16
app/router/notification/channel.py
View File

@@ -122,9 +122,7 @@ async def join_channel(
).first() ).first()
else: else:
db_channel = ( db_channel = (
await session.exec( await session.exec(select(ChatChannel).where(ChatChannel.name == channel))
select(ChatChannel).where(ChatChannel.name == channel)
)
).first() ).first()
if db_channel is None: if db_channel is None:
@@ -154,9 +152,7 @@ async def leave_channel(
).first() ).first()
else: else:
db_channel = ( db_channel = (
await session.exec( await session.exec(select(ChatChannel).where(ChatChannel.name == channel))
select(ChatChannel).where(ChatChannel.name == channel)
)
).first() ).first()
if db_channel is None: if db_channel is None:
@@ -230,9 +226,7 @@ async def get_channel(
).first() ).first()
else: else:
db_channel = ( db_channel = (
await session.exec( await session.exec(select(ChatChannel).where(ChatChannel.name == channel))
select(ChatChannel).where(ChatChannel.name == channel)
)
).first() ).first()
if db_channel is None: if db_channel is None:
@@ -325,7 +319,9 @@ async def create_channel(
channel_name = f"pm_{current_user.id}_{req.target_id}" channel_name = f"pm_{current_user.id}_{req.target_id}"
else: else:
channel_name = req.channel.name if req.channel else "Unnamed Channel" channel_name = req.channel.name if req.channel else "Unnamed Channel"
result = await session.exec(select(ChatChannel).where(ChatChannel.name == channel_name)) result = await session.exec(
select(ChatChannel).where(ChatChannel.name == channel_name)
)
channel = result.first() channel = result.first()
if channel is None: if channel is None:

24
app/router/notification/message.py
View File

@@ -1,10 +1,5 @@
from __future__ import annotations from __future__ import annotations
import json
import uuid
from datetime import datetime
from typing import Optional
from app.database import ChatMessageResp from app.database import ChatMessageResp
from app.database.chat import ( from app.database.chat import (
ChannelType, ChannelType,
@@ -16,14 +11,13 @@ from app.database.chat import (
UserSilenceResp, UserSilenceResp,
) )
from app.database.lazer_user import User from app.database.lazer_user import User
from app.dependencies.database import Database, get_redis, get_redis_message from app.dependencies.database import Database, get_redis
from app.dependencies.param import BodyOrForm from app.dependencies.param import BodyOrForm
from app.dependencies.user import get_current_user from app.dependencies.user import get_current_user
from app.log import logger
from app.models.notification import ChannelMessage, ChannelMessageTeam from app.models.notification import ChannelMessage, ChannelMessageTeam
from app.router.v2 import api_v2_router as router from app.router.v2 import api_v2_router as router
from app.service.optimized_message import optimized_message_service
from app.service.redis_message_system import redis_message_system from app.service.redis_message_system import redis_message_system
from app.log import logger
from .banchobot import bot from .banchobot import bot
from .server import server from .server import server
@@ -106,9 +100,7 @@ async def send_message(
).first() ).first()
else: else:
db_channel = ( db_channel = (
await session.exec( await session.exec(select(ChatChannel).where(ChatChannel.name == channel))
select(ChatChannel).where(ChatChannel.name == channel)
)
).first() ).first()
if db_channel is None: if db_channel is None:
@@ -128,7 +120,7 @@ async def send_message(
user=current_user, user=current_user,
content=req.message, content=req.message,
is_action=req.is_action, is_action=req.is_action,
user_uuid=req.uuid user_uuid=req.uuid,
) )
# 立即广播消息给所有客户端 # 立即广播消息给所有客户端
@@ -191,9 +183,7 @@ async def get_message(
).first() ).first()
else: else:
db_channel = ( db_channel = (
await session.exec( await session.exec(select(ChatChannel).where(ChatChannel.name == channel))
select(ChatChannel).where(ChatChannel.name == channel)
)
).first() ).first()
if db_channel is None: if db_channel is None:
@@ -247,9 +237,7 @@ async def mark_as_read(
).first() ).first()
else: else:
db_channel = ( db_channel = (
await session.exec( await session.exec(select(ChatChannel).where(ChatChannel.name == channel))
select(ChatChannel).where(ChatChannel.name == channel)
)
).first() ).first()
if db_channel is None: if db_channel is None:

24
app/router/notification/server.py
View File

@@ -96,7 +96,9 @@ class ChatServer:
async def send_message_to_channel( async def send_message_to_channel(
self, message: ChatMessageResp, is_bot_command: bool = False self, message: ChatMessageResp, is_bot_command: bool = False
): ):
logger.info(f"Sending message to channel {message.channel_id}, message_id: {message.message_id}, is_bot_command: {is_bot_command}") logger.info(
f"Sending message to channel {message.channel_id}, message_id: {message.message_id}, is_bot_command: {is_bot_command}"
)
event = ChatEvent( event = ChatEvent(
event="chat.message.new", event="chat.message.new",
@@ -107,7 +109,9 @@ class ChatServer:
self._add_task(self.send_event(message.sender_id, event)) self._add_task(self.send_event(message.sender_id, event))
else: else:
# 总是广播消息，无论是临时ID还是真实ID # 总是广播消息，无论是临时ID还是真实ID
logger.info(f"Broadcasting message to all users in channel {message.channel_id}") logger.info(
f"Broadcasting message to all users in channel {message.channel_id}"
)
self._add_task( self._add_task(
self.broadcast( self.broadcast(
message.channel_id, message.channel_id,
@@ -121,10 +125,16 @@ class ChatServer:
await self.mark_as_read( await self.mark_as_read(
message.channel_id, message.sender_id, message.message_id message.channel_id, message.sender_id, message.message_id
) )
await self.redis.set(f"chat:{message.channel_id}:last_msg", message.message_id) await self.redis.set(
logger.info(f"Updated last message ID for channel {message.channel_id} to {message.message_id}") f"chat:{message.channel_id}:last_msg", message.message_id
)
logger.info(
f"Updated last message ID for channel {message.channel_id} to {message.message_id}"
)
else: else:
logger.debug(f"Skipping last message update for message ID: {message.message_id}") logger.debug(
f"Skipping last message update for message ID: {message.message_id}"
)
async def batch_join_channel( async def batch_join_channel(
self, users: list[User], channel: ChatChannel, session: AsyncSession self, users: list[User], channel: ChatChannel, session: AsyncSession
@@ -340,9 +350,7 @@ async def chat_websocket(
server.connect(user_id, websocket) server.connect(user_id, websocket)
# 使用明确的查询避免延迟加载 # 使用明确的查询避免延迟加载
db_channel = ( db_channel = (
await session.exec( await session.exec(select(ChatChannel).where(ChatChannel.channel_id == 1))
select(ChatChannel).where(ChatChannel.channel_id == 1)
)
).first() ).first()
if db_channel is not None: if db_channel is not None:
await server.join_channel(user, db_channel, session) await server.join_channel(user, db_channel, session)

1
app/router/v2/router.py
View File

@@ -5,4 +5,3 @@ from fastapi import APIRouter
router = APIRouter(prefix="/api/v2") router = APIRouter(prefix="/api/v2")
# 导入所有子路由模块来注册路由 # 导入所有子路由模块来注册路由
from . import stats # 统计路由

23
app/router/v2/score.py
View File

@@ -75,9 +75,10 @@ READ_SCORE_TIMEOUT = 10
async def process_user_achievement(score_id: int): async def process_user_achievement(score_id: int):
from sqlmodel.ext.asyncio.session import AsyncSession
from app.dependencies.database import engine from app.dependencies.database import engine
from sqlmodel.ext.asyncio.session import AsyncSession
session = AsyncSession(engine) session = AsyncSession(engine)
try: try:
await process_achievements(session, get_redis(), score_id) await process_achievements(session, get_redis(), score_id)
@@ -166,11 +167,13 @@ async def submit_score(
has_pp, has_pp,
has_leaderboard, has_leaderboard,
) )
score = (await db.exec( score = (
select(Score) await db.exec(
.options(joinedload(Score.user)) # pyright: ignore[reportArgumentType] select(Score)
.where(Score.id == score_id) .options(joinedload(Score.user)) # pyright: ignore[reportArgumentType]
)).first() .where(Score.id == score_id)
)
).first()
assert score is not None assert score is not None
resp = await ScoreResp.from_db(db, score) resp = await ScoreResp.from_db(db, score)
@@ -205,10 +208,7 @@ async def submit_score(
if user_id is not None: if user_id is not None:
background_task.add_task( background_task.add_task(
_refresh_user_cache_background, _refresh_user_cache_background, redis, user_id, score_gamemode
redis,
user_id,
score_gamemode
) )
background_task.add_task(process_user_achievement, resp.id) background_task.add_task(process_user_achievement, resp.id)
return resp return resp
@@ -217,9 +217,10 @@ async def submit_score(
async def _refresh_user_cache_background(redis: Redis, user_id: int, mode: GameMode): async def _refresh_user_cache_background(redis: Redis, user_id: int, mode: GameMode):
"""后台任务：刷新用户缓存""" """后台任务：刷新用户缓存"""
try: try:
from sqlmodel.ext.asyncio.session import AsyncSession
from app.dependencies.database import engine from app.dependencies.database import engine
from sqlmodel.ext.asyncio.session import AsyncSession
user_cache_service = get_user_cache_service(redis) user_cache_service = get_user_cache_service(redis)
# 创建独立的数据库会话 # 创建独立的数据库会话
session = AsyncSession(engine) session = AsyncSession(engine)

108
app/router/v2/stats.py
View File

@@ -1,17 +1,15 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
from datetime import datetime, timedelta
import json
from typing import Any
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
import json
from app.dependencies.database import get_redis, get_redis_message from app.dependencies.database import get_redis, get_redis_message
from app.log import logger from app.log import logger
from .router import router from .router import router
from fastapi import APIRouter
from pydantic import BaseModel from pydantic import BaseModel
# Redis key constants # Redis key constants
@@ -23,20 +21,25 @@ REDIS_ONLINE_HISTORY_KEY = "server:online_history"
# 线程池用于同步Redis操作 # 线程池用于同步Redis操作
_executor = ThreadPoolExecutor(max_workers=2) _executor = ThreadPoolExecutor(max_workers=2)
async def _redis_exec(func, *args, **kwargs): async def _redis_exec(func, *args, **kwargs):
"""在线程池中执行同步Redis操作""" """在线程池中执行同步Redis操作"""
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
return await loop.run_in_executor(_executor, func, *args, **kwargs) return await loop.run_in_executor(_executor, func, *args, **kwargs)
class ServerStats(BaseModel): class ServerStats(BaseModel):
"""服务器统计信息响应模型""" """服务器统计信息响应模型"""
registered_users: int registered_users: int
online_users: int online_users: int
playing_users: int playing_users: int
timestamp: datetime timestamp: datetime
class OnlineHistoryPoint(BaseModel): class OnlineHistoryPoint(BaseModel):
"""在线历史数据点""" """在线历史数据点"""
timestamp: datetime timestamp: datetime
online_count: int online_count: int
playing_count: int playing_count: int
@@ -44,11 +47,14 @@ class OnlineHistoryPoint(BaseModel):
peak_playing: int | None = None # 峰值游玩数（增强数据） peak_playing: int | None = None # 峰值游玩数（增强数据）
total_samples: int | None = None # 采样次数（增强数据） total_samples: int | None = None # 采样次数（增强数据）
class OnlineHistoryResponse(BaseModel): class OnlineHistoryResponse(BaseModel):
"""24小时在线历史响应模型""" """24小时在线历史响应模型"""
history: list[OnlineHistoryPoint] history: list[OnlineHistoryPoint]
current_stats: ServerStats current_stats: ServerStats
@router.get("/stats", response_model=ServerStats, tags=["统计"]) @router.get("/stats", response_model=ServerStats, tags=["统计"])
async def get_server_stats() -> ServerStats: async def get_server_stats() -> ServerStats:
""" """
@@ -63,14 +69,14 @@ async def get_server_stats() -> ServerStats:
registered_count, online_count, playing_count = await asyncio.gather( registered_count, online_count, playing_count = await asyncio.gather(
_get_registered_users_count(redis), _get_registered_users_count(redis),
_get_online_users_count(redis), _get_online_users_count(redis),
_get_playing_users_count(redis) _get_playing_users_count(redis),
) )
return ServerStats( return ServerStats(
registered_users=registered_count, registered_users=registered_count,
online_users=online_count, online_users=online_count,
playing_users=playing_count, playing_users=playing_count,
timestamp=datetime.utcnow() timestamp=datetime.utcnow(),
) )
except Exception as e: except Exception as e:
logger.error(f"Error getting server stats: {e}") logger.error(f"Error getting server stats: {e}")
@@ -79,9 +85,10 @@ async def get_server_stats() -> ServerStats:
registered_users=0, registered_users=0,
online_users=0, online_users=0,
playing_users=0, playing_users=0,
timestamp=datetime.utcnow() timestamp=datetime.utcnow(),
) )
@router.get("/stats/history", response_model=OnlineHistoryResponse, tags=["统计"]) @router.get("/stats/history", response_model=OnlineHistoryResponse, tags=["统计"])
async def get_online_history() -> OnlineHistoryResponse: async def get_online_history() -> OnlineHistoryResponse:
""" """
@@ -93,7 +100,9 @@ async def get_online_history() -> OnlineHistoryResponse:
try: try:
# 获取历史数据 - 使用同步Redis客户端 # 获取历史数据 - 使用同步Redis客户端
redis_sync = get_redis_message() redis_sync = get_redis_message()
history_data = await _redis_exec(redis_sync.lrange, REDIS_ONLINE_HISTORY_KEY, 0, -1) history_data = await _redis_exec(
redis_sync.lrange, REDIS_ONLINE_HISTORY_KEY, 0, -1
)
history_points = [] history_points = []
# 处理历史数据 # 处理历史数据
@@ -101,14 +110,20 @@ async def get_online_history() -> OnlineHistoryResponse:
try: try:
point_data = json.loads(data) point_data = json.loads(data)
# 支持新旧格式的历史数据 # 支持新旧格式的历史数据
history_points.append(OnlineHistoryPoint( history_points.append(
timestamp=datetime.fromisoformat(point_data["timestamp"]), OnlineHistoryPoint(
online_count=point_data["online_count"], timestamp=datetime.fromisoformat(point_data["timestamp"]),
playing_count=point_data["playing_count"], online_count=point_data["online_count"],
peak_online=point_data.get("peak_online"), # 新字段，可能不存在 playing_count=point_data["playing_count"],
peak_playing=point_data.get("peak_playing"), # 新字段，可能不存在 peak_online=point_data.get("peak_online"), # 新字段，可能不存在
total_samples=point_data.get("total_samples") # 新字段，可能不存在 peak_playing=point_data.get(
)) "peak_playing"
), # 新字段，可能不存在
total_samples=point_data.get(
"total_samples"
), # 新字段，可能不存在
)
)
except (json.JSONDecodeError, KeyError, ValueError) as e: except (json.JSONDecodeError, KeyError, ValueError) as e:
logger.warning(f"Invalid history data point: {data}, error: {e}") logger.warning(f"Invalid history data point: {data}, error: {e}")
continue continue
@@ -118,17 +133,23 @@ async def get_online_history() -> OnlineHistoryResponse:
# 如果历史数据为空或者最新数据超过15分钟，添加当前数据点 # 如果历史数据为空或者最新数据超过15分钟，添加当前数据点
if not history_points or ( if not history_points or (
history_points and history_points
(current_stats.timestamp - max(history_points, key=lambda x: x.timestamp).timestamp).total_seconds() > 15 * 60 and (
current_stats.timestamp
- max(history_points, key=lambda x: x.timestamp).timestamp
).total_seconds()
> 15 * 60
): ):
history_points.append(OnlineHistoryPoint( history_points.append(
timestamp=current_stats.timestamp, OnlineHistoryPoint(
online_count=current_stats.online_users, timestamp=current_stats.timestamp,
playing_count=current_stats.playing_users, online_count=current_stats.online_users,
peak_online=current_stats.online_users, # 当前实时数据作为峰值 playing_count=current_stats.playing_users,
peak_playing=current_stats.playing_users, peak_online=current_stats.online_users, # 当前实时数据作为峰值
total_samples=1 peak_playing=current_stats.playing_users,
)) total_samples=1,
)
)
# 按时间排序（最新的在前） # 按时间排序（最新的在前）
history_points.sort(key=lambda x: x.timestamp, reverse=True) history_points.sort(key=lambda x: x.timestamp, reverse=True)
@@ -137,17 +158,13 @@ async def get_online_history() -> OnlineHistoryResponse:
history_points = history_points[:48] history_points = history_points[:48]
return OnlineHistoryResponse( return OnlineHistoryResponse(
history=history_points, history=history_points, current_stats=current_stats
current_stats=current_stats
) )
except Exception as e: except Exception as e:
logger.error(f"Error getting online history: {e}") logger.error(f"Error getting online history: {e}")
# 返回空历史和当前状态 # 返回空历史和当前状态
current_stats = await get_server_stats() current_stats = await get_server_stats()
return OnlineHistoryResponse( return OnlineHistoryResponse(history=[], current_stats=current_stats)
history=[],
current_stats=current_stats
)
async def _get_registered_users_count(redis) -> int: async def _get_registered_users_count(redis) -> int:
@@ -159,6 +176,7 @@ async def _get_registered_users_count(redis) -> int:
logger.error(f"Error getting registered users count: {e}") logger.error(f"Error getting registered users count: {e}")
return 0 return 0
async def _get_online_users_count(redis) -> int: async def _get_online_users_count(redis) -> int:
"""获取当前在线用户数""" """获取当前在线用户数"""
try: try:
@@ -168,6 +186,7 @@ async def _get_online_users_count(redis) -> int:
logger.error(f"Error getting online users count: {e}") logger.error(f"Error getting online users count: {e}")
return 0 return 0
async def _get_playing_users_count(redis) -> int: async def _get_playing_users_count(redis) -> int:
"""获取当前游玩用户数""" """获取当前游玩用户数"""
try: try:
@@ -177,13 +196,15 @@ async def _get_playing_users_count(redis) -> int:
logger.error(f"Error getting playing users count: {e}") logger.error(f"Error getting playing users count: {e}")
return 0 return 0
# 统计更新功能 # 统计更新功能
async def update_registered_users_count() -> None: async def update_registered_users_count() -> None:
"""更新注册用户数缓存""" """更新注册用户数缓存"""
from app.dependencies.database import with_db
from app.database import User
from app.const import BANCHOBOT_ID from app.const import BANCHOBOT_ID
from sqlmodel import select, func from app.database import User
from app.dependencies.database import with_db
from sqlmodel import func, select
redis = get_redis() redis = get_redis()
try: try:
@@ -198,6 +219,7 @@ async def update_registered_users_count() -> None:
except Exception as e: except Exception as e:
logger.error(f"Error updating registered users count: {e}") logger.error(f"Error updating registered users count: {e}")
async def add_online_user(user_id: int) -> None: async def add_online_user(user_id: int) -> None:
"""添加在线用户""" """添加在线用户"""
redis_sync = get_redis_message() redis_sync = get_redis_message()
@@ -212,11 +234,13 @@ async def add_online_user(user_id: int) -> None:
# 立即更新当前区间统计 # 立即更新当前区间统计
from app.service.enhanced_interval_stats import update_user_activity_in_interval from app.service.enhanced_interval_stats import update_user_activity_in_interval
asyncio.create_task(update_user_activity_in_interval(user_id, is_playing=False)) asyncio.create_task(update_user_activity_in_interval(user_id, is_playing=False))
except Exception as e: except Exception as e:
logger.error(f"Error adding online user {user_id}: {e}") logger.error(f"Error adding online user {user_id}: {e}")
async def remove_online_user(user_id: int) -> None: async def remove_online_user(user_id: int) -> None:
"""移除在线用户""" """移除在线用户"""
redis_sync = get_redis_message() redis_sync = get_redis_message()
@@ -226,6 +250,7 @@ async def remove_online_user(user_id: int) -> None:
except Exception as e: except Exception as e:
logger.error(f"Error removing online user {user_id}: {e}") logger.error(f"Error removing online user {user_id}: {e}")
async def add_playing_user(user_id: int) -> None: async def add_playing_user(user_id: int) -> None:
"""添加游玩用户""" """添加游玩用户"""
redis_sync = get_redis_message() redis_sync = get_redis_message()
@@ -240,11 +265,13 @@ async def add_playing_user(user_id: int) -> None:
# 立即更新当前区间统计 # 立即更新当前区间统计
from app.service.enhanced_interval_stats import update_user_activity_in_interval from app.service.enhanced_interval_stats import update_user_activity_in_interval
asyncio.create_task(update_user_activity_in_interval(user_id, is_playing=True)) asyncio.create_task(update_user_activity_in_interval(user_id, is_playing=True))
except Exception as e: except Exception as e:
logger.error(f"Error adding playing user {user_id}: {e}") logger.error(f"Error adding playing user {user_id}: {e}")
async def remove_playing_user(user_id: int) -> None: async def remove_playing_user(user_id: int) -> None:
"""移除游玩用户""" """移除游玩用户"""
redis_sync = get_redis_message() redis_sync = get_redis_message()
@@ -253,6 +280,7 @@ async def remove_playing_user(user_id: int) -> None:
except Exception as e: except Exception as e:
logger.error(f"Error removing playing user {user_id}: {e}") logger.error(f"Error removing playing user {user_id}: {e}")
async def record_hourly_stats() -> None: async def record_hourly_stats() -> None:
"""记录统计数据 - 简化版本，主要作为fallback使用""" """记录统计数据 - 简化版本，主要作为fallback使用"""
redis_sync = get_redis_message() redis_sync = get_redis_message()
@@ -271,16 +299,20 @@ async def record_hourly_stats() -> None:
"playing_count": playing_count, "playing_count": playing_count,
"peak_online": online_count, "peak_online": online_count,
"peak_playing": playing_count, "peak_playing": playing_count,
"total_samples": 1 "total_samples": 1,
} }
# 添加到历史记录 # 添加到历史记录
await _redis_exec(redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, json.dumps(history_point)) await _redis_exec(
redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, json.dumps(history_point)
)
# 只保留48个数据点（24小时，每30分钟一个点） # 只保留48个数据点（24小时，每30分钟一个点）
await _redis_exec(redis_sync.ltrim, REDIS_ONLINE_HISTORY_KEY, 0, 47) await _redis_exec(redis_sync.ltrim, REDIS_ONLINE_HISTORY_KEY, 0, 47)
# 设置过期时间为26小时，确保有足够缓冲 # 设置过期时间为26小时，确保有足够缓冲
await redis_async.expire(REDIS_ONLINE_HISTORY_KEY, 26 * 3600) await redis_async.expire(REDIS_ONLINE_HISTORY_KEY, 26 * 3600)
logger.info(f"Recorded fallback stats: online={online_count}, playing={playing_count} at {current_time.strftime('%H:%M:%S')}") logger.info(
f"Recorded fallback stats: online={online_count}, playing={playing_count} at {current_time.strftime('%H:%M:%S')}"
)
except Exception as e: except Exception as e:
logger.error(f"Error recording fallback stats: {e}") logger.error(f"Error recording fallback stats: {e}")

204
app/service/enhanced_interval_stats.py
View File

@@ -4,11 +4,9 @@
from __future__ import annotations from __future__ import annotations
import json from dataclasses import dataclass
import asyncio
from datetime import datetime, timedelta from datetime import datetime, timedelta
from typing import Dict, Set, Optional, List import json
from dataclasses import dataclass, asdict
from app.dependencies.database import get_redis, get_redis_message from app.dependencies.database import get_redis, get_redis_message
from app.log import logger from app.log import logger
@@ -16,7 +14,7 @@ from app.router.v2.stats import (
REDIS_ONLINE_HISTORY_KEY, REDIS_ONLINE_HISTORY_KEY,
_get_online_users_count, _get_online_users_count,
_get_playing_users_count, _get_playing_users_count,
_redis_exec _redis_exec,
) )
# Redis keys for interval statistics # Redis keys for interval statistics
@@ -29,6 +27,7 @@ CURRENT_INTERVAL_INFO_KEY = "server:current_interval_info" # 当前区间信息
@dataclass @dataclass
class IntervalInfo: class IntervalInfo:
"""区间信息""" """区间信息"""
start_time: datetime start_time: datetime
end_time: datetime end_time: datetime
interval_key: str interval_key: str
@@ -38,25 +37,26 @@ class IntervalInfo:
now = datetime.utcnow() now = datetime.utcnow()
return self.start_time <= now < self.end_time return self.start_time <= now < self.end_time
def to_dict(self) -> Dict: def to_dict(self) -> dict:
return { return {
'start_time': self.start_time.isoformat(), "start_time": self.start_time.isoformat(),
'end_time': self.end_time.isoformat(), "end_time": self.end_time.isoformat(),
'interval_key': self.interval_key "interval_key": self.interval_key,
} }
@classmethod @classmethod
def from_dict(cls, data: Dict) -> 'IntervalInfo': def from_dict(cls, data: dict) -> "IntervalInfo":
return cls( return cls(
start_time=datetime.fromisoformat(data['start_time']), start_time=datetime.fromisoformat(data["start_time"]),
end_time=datetime.fromisoformat(data['end_time']), end_time=datetime.fromisoformat(data["end_time"]),
interval_key=data['interval_key'] interval_key=data["interval_key"],
) )
@dataclass @dataclass
class IntervalStats: class IntervalStats:
"""区间统计数据""" """区间统计数据"""
interval_key: str interval_key: str
start_time: datetime start_time: datetime
end_time: datetime end_time: datetime
@@ -67,31 +67,31 @@ class IntervalStats:
total_samples: int # 采样次数 total_samples: int # 采样次数
created_at: datetime created_at: datetime
def to_dict(self) -> Dict: def to_dict(self) -> dict:
return { return {
'interval_key': self.interval_key, "interval_key": self.interval_key,
'start_time': self.start_time.isoformat(), "start_time": self.start_time.isoformat(),
'end_time': self.end_time.isoformat(), "end_time": self.end_time.isoformat(),
'unique_online_users': self.unique_online_users, "unique_online_users": self.unique_online_users,
'unique_playing_users': self.unique_playing_users, "unique_playing_users": self.unique_playing_users,
'peak_online_count': self.peak_online_count, "peak_online_count": self.peak_online_count,
'peak_playing_count': self.peak_playing_count, "peak_playing_count": self.peak_playing_count,
'total_samples': self.total_samples, "total_samples": self.total_samples,
'created_at': self.created_at.isoformat() "created_at": self.created_at.isoformat(),
} }
@classmethod @classmethod
def from_dict(cls, data: Dict) -> 'IntervalStats': def from_dict(cls, data: dict) -> "IntervalStats":
return cls( return cls(
interval_key=data['interval_key'], interval_key=data["interval_key"],
start_time=datetime.fromisoformat(data['start_time']), start_time=datetime.fromisoformat(data["start_time"]),
end_time=datetime.fromisoformat(data['end_time']), end_time=datetime.fromisoformat(data["end_time"]),
unique_online_users=data['unique_online_users'], unique_online_users=data["unique_online_users"],
unique_playing_users=data['unique_playing_users'], unique_playing_users=data["unique_playing_users"],
peak_online_count=data['peak_online_count'], peak_online_count=data["peak_online_count"],
peak_playing_count=data['peak_playing_count'], peak_playing_count=data["peak_playing_count"],
total_samples=data['total_samples'], total_samples=data["total_samples"],
created_at=datetime.fromisoformat(data['created_at']) created_at=datetime.fromisoformat(data["created_at"]),
) )
@@ -117,13 +117,13 @@ class EnhancedIntervalStatsManager:
@staticmethod @staticmethod
async def get_current_interval_info() -> IntervalInfo: async def get_current_interval_info() -> IntervalInfo:
"""获取当前区间信息""" """获取当前区间信息"""
start_time, end_time = EnhancedIntervalStatsManager.get_current_interval_boundaries() start_time, end_time = (
EnhancedIntervalStatsManager.get_current_interval_boundaries()
)
interval_key = EnhancedIntervalStatsManager.generate_interval_key(start_time) interval_key = EnhancedIntervalStatsManager.generate_interval_key(start_time)
return IntervalInfo( return IntervalInfo(
start_time=start_time, start_time=start_time, end_time=end_time, interval_key=interval_key
end_time=end_time,
interval_key=interval_key
) )
@staticmethod @staticmethod
@@ -133,19 +133,23 @@ class EnhancedIntervalStatsManager:
redis_async = get_redis() redis_async = get_redis()
try: try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info() current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
# 存储当前区间信息 # 存储当前区间信息
await _redis_exec( await _redis_exec(
redis_sync.set, redis_sync.set,
CURRENT_INTERVAL_INFO_KEY, CURRENT_INTERVAL_INFO_KEY,
json.dumps(current_interval.to_dict()) json.dumps(current_interval.to_dict()),
) )
await redis_async.expire(CURRENT_INTERVAL_INFO_KEY, 35 * 60) # 35分钟过期 await redis_async.expire(CURRENT_INTERVAL_INFO_KEY, 35 * 60) # 35分钟过期
# 初始化区间用户集合（如果不存在） # 初始化区间用户集合（如果不存在）
online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}" online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}"
playing_key = f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}" playing_key = (
f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
)
# 设置过期时间为35分钟 # 设置过期时间为35分钟
await redis_async.expire(online_key, 35 * 60) await redis_async.expire(online_key, 35 * 60)
@@ -161,20 +165,22 @@ class EnhancedIntervalStatsManager:
peak_online_count=0, peak_online_count=0,
peak_playing_count=0, peak_playing_count=0,
total_samples=0, total_samples=0,
created_at=datetime.utcnow() created_at=datetime.utcnow(),
) )
await _redis_exec( await _redis_exec(
redis_sync.set, redis_sync.set,
current_interval.interval_key, current_interval.interval_key,
json.dumps(stats.to_dict()) json.dumps(stats.to_dict()),
) )
await redis_async.expire(current_interval.interval_key, 35 * 60) await redis_async.expire(current_interval.interval_key, 35 * 60)
# 如果历史记录为空，自动填充前24小时数据为0 # 如果历史记录为空，自动填充前24小时数据为0
await EnhancedIntervalStatsManager._ensure_24h_history_exists() await EnhancedIntervalStatsManager._ensure_24h_history_exists()
logger.info(f"Initialized interval stats for {current_interval.start_time.strftime('%H:%M')} - {current_interval.end_time.strftime('%H:%M')}") logger.info(
f"Initialized interval stats for {current_interval.start_time.strftime('%H:%M')} - {current_interval.end_time.strftime('%H:%M')}"
)
except Exception as e: except Exception as e:
logger.error(f"Error initializing current interval: {e}") logger.error(f"Error initializing current interval: {e}")
@@ -187,27 +193,37 @@ class EnhancedIntervalStatsManager:
try: try:
# 检查现有历史数据数量 # 检查现有历史数据数量
history_length = await _redis_exec(redis_sync.llen, REDIS_ONLINE_HISTORY_KEY) history_length = await _redis_exec(
redis_sync.llen, REDIS_ONLINE_HISTORY_KEY
)
if history_length < 48: # 少于48个数据点（24小时*2） if history_length < 48: # 少于48个数据点（24小时*2）
logger.info(f"History has only {history_length} points, filling with zeros for 24h") logger.info(
f"History has only {history_length} points, filling with zeros for 24h"
)
# 计算需要填充的数据点数量 # 计算需要填充的数据点数量
needed_points = 48 - history_length needed_points = 48 - history_length
# 从当前时间往前推，创建缺失的时间点（都填充为0） # 从当前时间往前推，创建缺失的时间点（都填充为0）
current_time = datetime.utcnow() current_time = datetime.utcnow()
current_interval_start, _ = EnhancedIntervalStatsManager.get_current_interval_boundaries() current_interval_start, _ = (
EnhancedIntervalStatsManager.get_current_interval_boundaries()
)
# 从当前区间开始往前推，创建历史数据点（确保时间对齐到30分钟边界） # 从当前区间开始往前推，创建历史数据点（确保时间对齐到30分钟边界）
fill_points = [] fill_points = []
for i in range(needed_points): for i in range(needed_points):
# 每次往前推30分钟，确保时间对齐 # 每次往前推30分钟，确保时间对齐
point_time = current_interval_start - timedelta(minutes=30 * (i + 1)) point_time = current_interval_start - timedelta(
minutes=30 * (i + 1)
)
# 确保时间对齐到30分钟边界 # 确保时间对齐到30分钟边界
aligned_minute = (point_time.minute // 30) * 30 aligned_minute = (point_time.minute // 30) * 30
point_time = point_time.replace(minute=aligned_minute, second=0, microsecond=0) point_time = point_time.replace(
minute=aligned_minute, second=0, microsecond=0
)
history_point = { history_point = {
"timestamp": point_time.isoformat(), "timestamp": point_time.isoformat(),
@@ -215,7 +231,7 @@ class EnhancedIntervalStatsManager:
"playing_count": 0, "playing_count": 0,
"peak_online": 0, "peak_online": 0,
"peak_playing": 0, "peak_playing": 0,
"total_samples": 0 "total_samples": 0,
} }
fill_points.append(json.dumps(history_point)) fill_points.append(json.dumps(history_point))
@@ -225,7 +241,9 @@ class EnhancedIntervalStatsManager:
temp_key = f"{REDIS_ONLINE_HISTORY_KEY}_temp" temp_key = f"{REDIS_ONLINE_HISTORY_KEY}_temp"
if history_length > 0: if history_length > 0:
# 复制现有数据到临时key # 复制现有数据到临时key
existing_data = await _redis_exec(redis_sync.lrange, REDIS_ONLINE_HISTORY_KEY, 0, -1) existing_data = await _redis_exec(
redis_sync.lrange, REDIS_ONLINE_HISTORY_KEY, 0, -1
)
if existing_data: if existing_data:
for data in existing_data: for data in existing_data:
await _redis_exec(redis_sync.rpush, temp_key, data) await _redis_exec(redis_sync.rpush, temp_key, data)
@@ -235,13 +253,19 @@ class EnhancedIntervalStatsManager:
# 先添加填充数据（最旧的） # 先添加填充数据（最旧的）
for point in reversed(fill_points): # 反向添加，最旧的在最后 for point in reversed(fill_points): # 反向添加，最旧的在最后
await _redis_exec(redis_sync.rpush, REDIS_ONLINE_HISTORY_KEY, point) await _redis_exec(
redis_sync.rpush, REDIS_ONLINE_HISTORY_KEY, point
)
# 再添加原有数据（较新的） # 再添加原有数据（较新的）
if history_length > 0: if history_length > 0:
existing_data = await _redis_exec(redis_sync.lrange, temp_key, 0, -1) existing_data = await _redis_exec(
redis_sync.lrange, temp_key, 0, -1
)
for data in existing_data: for data in existing_data:
await _redis_exec(redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, data) await _redis_exec(
redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, data
)
# 清理临时key # 清理临时key
await redis_async.delete(temp_key) await redis_async.delete(temp_key)
@@ -252,7 +276,9 @@ class EnhancedIntervalStatsManager:
# 设置过期时间 # 设置过期时间
await redis_async.expire(REDIS_ONLINE_HISTORY_KEY, 26 * 3600) await redis_async.expire(REDIS_ONLINE_HISTORY_KEY, 26 * 3600)
logger.info(f"Filled {len(fill_points)} historical data points with zeros") logger.info(
f"Filled {len(fill_points)} historical data points with zeros"
)
except Exception as e: except Exception as e:
logger.error(f"Error ensuring 24h history exists: {e}") logger.error(f"Error ensuring 24h history exists: {e}")
@@ -264,7 +290,9 @@ class EnhancedIntervalStatsManager:
redis_async = get_redis() redis_async = get_redis()
try: try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info() current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
# 添加到区间在线用户集合 # 添加到区间在线用户集合
online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}" online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}"
@@ -273,14 +301,18 @@ class EnhancedIntervalStatsManager:
# 如果用户在游玩，也添加到游玩用户集合 # 如果用户在游玩，也添加到游玩用户集合
if is_playing: if is_playing:
playing_key = f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}" playing_key = (
f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
)
await _redis_exec(redis_sync.sadd, playing_key, str(user_id)) await _redis_exec(redis_sync.sadd, playing_key, str(user_id))
await redis_async.expire(playing_key, 35 * 60) await redis_async.expire(playing_key, 35 * 60)
# 立即更新区间统计（同步更新，确保数据实时性） # 立即更新区间统计（同步更新，确保数据实时性）
await EnhancedIntervalStatsManager._update_interval_stats() await EnhancedIntervalStatsManager._update_interval_stats()
logger.debug(f"Added user {user_id} to current interval {current_interval.start_time.strftime('%H:%M')}-{current_interval.end_time.strftime('%H:%M')}") logger.debug(
f"Added user {user_id} to current interval {current_interval.start_time.strftime('%H:%M')}-{current_interval.end_time.strftime('%H:%M')}"
)
except Exception as e: except Exception as e:
logger.error(f"Error adding user {user_id} to interval: {e}") logger.error(f"Error adding user {user_id} to interval: {e}")
@@ -292,11 +324,15 @@ class EnhancedIntervalStatsManager:
redis_async = get_redis() redis_async = get_redis()
try: try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info() current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
# 获取区间内独特用户数 # 获取区间内独特用户数
online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}" online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}"
playing_key = f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}" playing_key = (
f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
)
unique_online = await _redis_exec(redis_sync.scard, online_key) unique_online = await _redis_exec(redis_sync.scard, online_key)
unique_playing = await _redis_exec(redis_sync.scard, playing_key) unique_playing = await _redis_exec(redis_sync.scard, playing_key)
@@ -306,12 +342,16 @@ class EnhancedIntervalStatsManager:
current_playing = await _get_playing_users_count(redis_async) current_playing = await _get_playing_users_count(redis_async)
# 获取现有统计数据 # 获取现有统计数据
existing_data = await _redis_exec(redis_sync.get, current_interval.interval_key) existing_data = await _redis_exec(
redis_sync.get, current_interval.interval_key
)
if existing_data: if existing_data:
stats = IntervalStats.from_dict(json.loads(existing_data)) stats = IntervalStats.from_dict(json.loads(existing_data))
# 更新峰值 # 更新峰值
stats.peak_online_count = max(stats.peak_online_count, current_online) stats.peak_online_count = max(stats.peak_online_count, current_online)
stats.peak_playing_count = max(stats.peak_playing_count, current_playing) stats.peak_playing_count = max(
stats.peak_playing_count, current_playing
)
stats.total_samples += 1 stats.total_samples += 1
else: else:
# 创建新的统计记录 # 创建新的统计记录
@@ -324,7 +364,7 @@ class EnhancedIntervalStatsManager:
peak_online_count=current_online, peak_online_count=current_online,
peak_playing_count=current_playing, peak_playing_count=current_playing,
total_samples=1, total_samples=1,
created_at=datetime.utcnow() created_at=datetime.utcnow(),
) )
# 更新独特用户数 # 更新独特用户数
@@ -335,29 +375,35 @@ class EnhancedIntervalStatsManager:
await _redis_exec( await _redis_exec(
redis_sync.set, redis_sync.set,
current_interval.interval_key, current_interval.interval_key,
json.dumps(stats.to_dict()) json.dumps(stats.to_dict()),
) )
await redis_async.expire(current_interval.interval_key, 35 * 60) await redis_async.expire(current_interval.interval_key, 35 * 60)
logger.debug(f"Updated interval stats: online={unique_online}, playing={unique_playing}, peak_online={stats.peak_online_count}, peak_playing={stats.peak_playing_count}") logger.debug(
f"Updated interval stats: online={unique_online}, playing={unique_playing}, peak_online={stats.peak_online_count}, peak_playing={stats.peak_playing_count}"
)
except Exception as e: except Exception as e:
logger.error(f"Error updating interval stats: {e}") logger.error(f"Error updating interval stats: {e}")
@staticmethod @staticmethod
async def finalize_interval() -> Optional[IntervalStats]: async def finalize_interval() -> IntervalStats | None:
"""完成当前区间统计并保存到历史""" """完成当前区间统计并保存到历史"""
redis_sync = get_redis_message() redis_sync = get_redis_message()
redis_async = get_redis() redis_async = get_redis()
try: try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info() current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
# 最后一次更新统计 # 最后一次更新统计
await EnhancedIntervalStatsManager._update_interval_stats() await EnhancedIntervalStatsManager._update_interval_stats()
# 获取最终统计数据 # 获取最终统计数据
stats_data = await _redis_exec(redis_sync.get, current_interval.interval_key) stats_data = await _redis_exec(
redis_sync.get, current_interval.interval_key
)
if not stats_data: if not stats_data:
logger.warning("No interval stats found to finalize") logger.warning("No interval stats found to finalize")
return None return None
@@ -371,11 +417,13 @@ class EnhancedIntervalStatsManager:
"playing_count": stats.unique_playing_users, "playing_count": stats.unique_playing_users,
"peak_online": stats.peak_online_count, "peak_online": stats.peak_online_count,
"peak_playing": stats.peak_playing_count, "peak_playing": stats.peak_playing_count,
"total_samples": stats.total_samples "total_samples": stats.total_samples,
} }
# 添加到历史记录 # 添加到历史记录
await _redis_exec(redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, json.dumps(history_point)) await _redis_exec(
redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, json.dumps(history_point)
)
# 只保留48个数据点（24小时，每30分钟一个点） # 只保留48个数据点（24小时，每30分钟一个点）
await _redis_exec(redis_sync.ltrim, REDIS_ONLINE_HISTORY_KEY, 0, 47) await _redis_exec(redis_sync.ltrim, REDIS_ONLINE_HISTORY_KEY, 0, 47)
# 设置过期时间为26小时，确保有足够缓冲 # 设置过期时间为26小时，确保有足够缓冲
@@ -398,13 +446,17 @@ class EnhancedIntervalStatsManager:
return None return None
@staticmethod @staticmethod
async def get_current_interval_stats() -> Optional[IntervalStats]: async def get_current_interval_stats() -> IntervalStats | None:
"""获取当前区间统计""" """获取当前区间统计"""
redis_sync = get_redis_message() redis_sync = get_redis_message()
try: try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info() current_interval = (
stats_data = await _redis_exec(redis_sync.get, current_interval.interval_key) await EnhancedIntervalStatsManager.get_current_interval_info()
)
stats_data = await _redis_exec(
redis_sync.get, current_interval.interval_key
)
if stats_data: if stats_data:
return IntervalStats.from_dict(json.loads(stats_data)) return IntervalStats.from_dict(json.loads(stats_data))
@@ -428,8 +480,8 @@ class EnhancedIntervalStatsManager:
for key in keys: for key in keys:
try: try:
# 从key中提取时间 # 从key中提取时间
time_part = key.decode().split(':')[-1] # YYYYMMDD_HHMM格式 time_part = key.decode().split(":")[-1] # YYYYMMDD_HHMM格式
key_time = datetime.strptime(time_part, '%Y%m%d_%H%M') key_time = datetime.strptime(time_part, "%Y%m%d_%H%M")
if key_time < cutoff_time: if key_time < cutoff_time:
await redis_async.delete(key) await redis_async.delete(key)
@@ -448,6 +500,8 @@ class EnhancedIntervalStatsManager:
# 便捷函数，用于替换现有的统计更新函数 # 便捷函数，用于替换现有的统计更新函数
async def update_user_activity_in_interval(user_id: int, is_playing: bool = False) -> None: async def update_user_activity_in_interval(
user_id: int, is_playing: bool = False
) -> None:
"""用户活动时更新区间统计（在登录、开始游玩等时调用）""" """用户活动时更新区间统计（在登录、开始游玩等时调用）"""
await EnhancedIntervalStatsManager.add_user_to_interval(user_id, is_playing) await EnhancedIntervalStatsManager.add_user_to_interval(user_id, is_playing)

86
app/service/message_queue.py
View File

@@ -3,15 +3,14 @@ Redis 消息队列服务
用于实现实时消息推送和异步数据库持久化 用于实现实时消息推送和异步数据库持久化
""" """
import asyncio from __future__ import annotations
import json
import uuid
from datetime import datetime
from functools import partial
from typing import Optional, Union
import concurrent.futures
from app.database.chat import ChatMessage, ChatChannel, MessageType, ChannelType import asyncio
import concurrent.futures
from datetime import datetime
import uuid
from app.database.chat import ChatMessage, MessageType
from app.dependencies.database import get_redis, with_db from app.dependencies.database import get_redis, with_db
from app.log import logger from app.log import logger
@@ -63,7 +62,9 @@ class MessageQueue:
await self._run_in_executor( await self._run_in_executor(
lambda: self.redis.hset(f"msg:{temp_uuid}", mapping=message_data) lambda: self.redis.hset(f"msg:{temp_uuid}", mapping=message_data)
) )
await self._run_in_executor(self.redis.expire, f"msg:{temp_uuid}", 3600) # 1小时过期 await self._run_in_executor(
self.redis.expire, f"msg:{temp_uuid}", 3600
) # 1小时过期
# 加入处理队列 # 加入处理队列
await self._run_in_executor(self.redis.lpush, "message_queue", temp_uuid) await self._run_in_executor(self.redis.lpush, "message_queue", temp_uuid)
@@ -71,15 +72,19 @@ class MessageQueue:
logger.info(f"Message enqueued with temp_uuid: {temp_uuid}") logger.info(f"Message enqueued with temp_uuid: {temp_uuid}")
return temp_uuid return temp_uuid
async def get_message_status(self, temp_uuid: str) -> Optional[dict]: async def get_message_status(self, temp_uuid: str) -> dict | None:
"""获取消息状态""" """获取消息状态"""
message_data = await self._run_in_executor(self.redis.hgetall, f"msg:{temp_uuid}") message_data = await self._run_in_executor(
self.redis.hgetall, f"msg:{temp_uuid}"
)
if not message_data: if not message_data:
return None return None
return message_data return message_data
async def get_cached_messages(self, channel_id: int, limit: int = 50, since: int = 0) -> list[dict]: async def get_cached_messages(
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[dict]:
""" """
从 Redis 获取缓存的消息 从 Redis 获取缓存的消息
@@ -98,7 +103,9 @@ class MessageQueue:
messages = [] messages = []
for uuid_str in message_uuids: for uuid_str in message_uuids:
message_data = await self._run_in_executor(self.redis.hgetall, f"msg:{uuid_str}") message_data = await self._run_in_executor(
self.redis.hgetall, f"msg:{uuid_str}"
)
if message_data: if message_data:
# 检查是否满足 since 条件 # 检查是否满足 since 条件
if since > 0 and "message_id" in message_data: if since > 0 and "message_id" in message_data:
@@ -109,14 +116,22 @@ class MessageQueue:
return messages[::-1] # 返回时间顺序 return messages[::-1] # 返回时间顺序
async def cache_channel_message(self, channel_id: int, temp_uuid: str, max_cache: int = 100): async def cache_channel_message(
self, channel_id: int, temp_uuid: str, max_cache: int = 100
):
"""将消息 UUID 缓存到频道消息列表""" """将消息 UUID 缓存到频道消息列表"""
# 添加到频道消息列表开头 # 添加到频道消息列表开头
await self._run_in_executor(self.redis.lpush, f"channel:{channel_id}:messages", temp_uuid) await self._run_in_executor(
self.redis.lpush, f"channel:{channel_id}:messages", temp_uuid
)
# 限制缓存大小 # 限制缓存大小
await self._run_in_executor(self.redis.ltrim, f"channel:{channel_id}:messages", 0, max_cache - 1) await self._run_in_executor(
self.redis.ltrim, f"channel:{channel_id}:messages", 0, max_cache - 1
)
# 设置过期时间（24小时） # 设置过期时间（24小时）
await self._run_in_executor(self.redis.expire, f"channel:{channel_id}:messages", 86400) await self._run_in_executor(
self.redis.expire, f"channel:{channel_id}:messages", 86400
)
async def _process_message_queue(self): async def _process_message_queue(self):
"""异步处理消息队列，批量写入数据库""" """异步处理消息队列，批量写入数据库"""
@@ -151,12 +166,16 @@ class MessageQueue:
for temp_uuid in message_uuids: for temp_uuid in message_uuids:
try: try:
# 获取消息数据 # 获取消息数据
message_data = await self._run_in_executor(self.redis.hgetall, f"msg:{temp_uuid}") message_data = await self._run_in_executor(
self.redis.hgetall, f"msg:{temp_uuid}"
)
if not message_data: if not message_data:
continue continue
# 更新状态为处理中 # 更新状态为处理中
await self._run_in_executor(self.redis.hset, f"msg:{temp_uuid}", "status", "processing") await self._run_in_executor(
self.redis.hset, f"msg:{temp_uuid}", "status", "processing"
)
# 创建数据库消息对象 # 创建数据库消息对象
msg = ChatMessage( msg = ChatMessage(
@@ -164,14 +183,16 @@ class MessageQueue:
content=message_data["content"], content=message_data["content"],
sender_id=int(message_data["sender_id"]), sender_id=int(message_data["sender_id"]),
type=MessageType(message_data["type"]), type=MessageType(message_data["type"]),
uuid=message_data.get("user_uuid") # 用户提供的 UUID（如果有） uuid=message_data.get("user_uuid"), # 用户提供的 UUID（如果有）
) )
messages_to_insert.append((msg, temp_uuid)) messages_to_insert.append((msg, temp_uuid))
except Exception as e: except Exception as e:
logger.error(f"Error preparing message {temp_uuid}: {e}") logger.error(f"Error preparing message {temp_uuid}: {e}")
await self._run_in_executor(self.redis.hset, f"msg:{temp_uuid}", "status", "failed") await self._run_in_executor(
self.redis.hset, f"msg:{temp_uuid}", "status", "failed"
)
if messages_to_insert: if messages_to_insert:
try: try:
@@ -185,14 +206,21 @@ class MessageQueue:
for msg, temp_uuid in messages_to_insert: for msg, temp_uuid in messages_to_insert:
await session.refresh(msg) await session.refresh(msg)
await self._run_in_executor( await self._run_in_executor(
lambda: self.redis.hset(f"msg:{temp_uuid}", mapping={ lambda: self.redis.hset(
"status": "completed", f"msg:{temp_uuid}",
"message_id": str(msg.message_id), mapping={
"created_at": msg.timestamp.isoformat() if msg.timestamp else "" "status": "completed",
}) "message_id": str(msg.message_id),
"created_at": msg.timestamp.isoformat()
if msg.timestamp
else "",
},
)
) )
logger.info(f"Message {temp_uuid} persisted to DB with ID {msg.message_id}") logger.info(
f"Message {temp_uuid} persisted to DB with ID {msg.message_id}"
)
except Exception as e: except Exception as e:
logger.error(f"Error inserting messages to database: {e}") logger.error(f"Error inserting messages to database: {e}")
@@ -200,7 +228,9 @@ class MessageQueue:
# 标记所有消息为失败 # 标记所有消息为失败
for _, temp_uuid in messages_to_insert: for _, temp_uuid in messages_to_insert:
await self._run_in_executor(self.redis.hset, f"msg:{temp_uuid}", "status", "failed") await self._run_in_executor(
self.redis.hset, f"msg:{temp_uuid}", "status", "failed"
)
# 全局消息队列实例 # 全局消息队列实例

124
app/service/message_queue_processor.py
View File

@@ -3,12 +3,12 @@
专门处理 Redis 消息队列的异步写入数据库 专门处理 Redis 消息队列的异步写入数据库
""" """
from __future__ import annotations
import asyncio import asyncio
import json
import uuid
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from datetime import datetime from datetime import datetime
from typing import Optional import json
from app.database.chat import ChatMessage, MessageType from app.database.chat import ChatMessage, MessageType
from app.dependencies.database import get_redis_message, with_db from app.dependencies.database import get_redis_message, with_db
@@ -33,40 +33,62 @@ class MessageQueueProcessor:
"""将消息缓存到 Redis""" """将消息缓存到 Redis"""
try: try:
# 存储消息数据 # 存储消息数据
await self._redis_exec(self.redis_message.hset, f"msg:{temp_uuid}", mapping=message_data) await self._redis_exec(
await self._redis_exec(self.redis_message.expire, f"msg:{temp_uuid}", 3600) # 1小时过期 self.redis_message.hset, f"msg:{temp_uuid}", mapping=message_data
)
await self._redis_exec(
self.redis_message.expire, f"msg:{temp_uuid}", 3600
) # 1小时过期
# 加入频道消息列表 # 加入频道消息列表
await self._redis_exec(self.redis_message.lpush, f"channel:{channel_id}:messages", temp_uuid) await self._redis_exec(
await self._redis_exec(self.redis_message.ltrim, f"channel:{channel_id}:messages", 0, 99) # 保持最新100条 self.redis_message.lpush, f"channel:{channel_id}:messages", temp_uuid
await self._redis_exec(self.redis_message.expire, f"channel:{channel_id}:messages", 86400) # 24小时过期 )
await self._redis_exec(
self.redis_message.ltrim, f"channel:{channel_id}:messages", 0, 99
) # 保持最新100条
await self._redis_exec(
self.redis_message.expire, f"channel:{channel_id}:messages", 86400
) # 24小时过期
# 加入异步处理队列 # 加入异步处理队列
await self._redis_exec(self.redis_message.lpush, "message_write_queue", temp_uuid) await self._redis_exec(
self.redis_message.lpush, "message_write_queue", temp_uuid
)
logger.info(f"Message cached to Redis: {temp_uuid}") logger.info(f"Message cached to Redis: {temp_uuid}")
except Exception as e: except Exception as e:
logger.error(f"Failed to cache message to Redis: {e}") logger.error(f"Failed to cache message to Redis: {e}")
async def get_cached_messages(self, channel_id: int, limit: int = 50, since: int = 0) -> list[dict]: async def get_cached_messages(
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[dict]:
"""从 Redis 获取缓存的消息""" """从 Redis 获取缓存的消息"""
try: try:
message_uuids = await self._redis_exec( message_uuids = await self._redis_exec(
self.redis_message.lrange, f"channel:{channel_id}:messages", 0, limit - 1 self.redis_message.lrange,
f"channel:{channel_id}:messages",
0,
limit - 1,
) )
messages = [] messages = []
for temp_uuid in message_uuids: for temp_uuid in message_uuids:
# 解码 UUID 如果它是字节类型 # 解码 UUID 如果它是字节类型
if isinstance(temp_uuid, bytes): if isinstance(temp_uuid, bytes):
temp_uuid = temp_uuid.decode('utf-8') temp_uuid = temp_uuid.decode("utf-8")
raw_data = await self._redis_exec(self.redis_message.hgetall, f"msg:{temp_uuid}") raw_data = await self._redis_exec(
self.redis_message.hgetall, f"msg:{temp_uuid}"
)
if raw_data: if raw_data:
# 解码 Redis 返回的字节数据 # 解码 Redis 返回的字节数据
message_data = { message_data = {
k.decode('utf-8') if isinstance(k, bytes) else k: k.decode("utf-8") if isinstance(k, bytes) else k: v.decode(
v.decode('utf-8') if isinstance(v, bytes) else v "utf-8"
)
if isinstance(v, bytes)
else v
for k, v in raw_data.items() for k, v in raw_data.items()
} }
@@ -81,7 +103,9 @@ class MessageQueueProcessor:
logger.error(f"Failed to get cached messages: {e}") logger.error(f"Failed to get cached messages: {e}")
return [] return []
async def update_message_status(self, temp_uuid: str, status: str, message_id: Optional[int] = None): async def update_message_status(
self, temp_uuid: str, status: str, message_id: int | None = None
):
"""更新消息状态""" """更新消息状态"""
try: try:
update_data = {"status": status} update_data = {"status": status}
@@ -89,21 +113,26 @@ class MessageQueueProcessor:
update_data["message_id"] = str(message_id) update_data["message_id"] = str(message_id)
update_data["db_timestamp"] = datetime.now().isoformat() update_data["db_timestamp"] = datetime.now().isoformat()
await self._redis_exec(self.redis_message.hset, f"msg:{temp_uuid}", mapping=update_data) await self._redis_exec(
self.redis_message.hset, f"msg:{temp_uuid}", mapping=update_data
)
except Exception as e: except Exception as e:
logger.error(f"Failed to update message status: {e}") logger.error(f"Failed to update message status: {e}")
async def get_message_status(self, temp_uuid: str) -> Optional[dict]: async def get_message_status(self, temp_uuid: str) -> dict | None:
"""获取消息状态""" """获取消息状态"""
try: try:
raw_data = await self._redis_exec(self.redis_message.hgetall, f"msg:{temp_uuid}") raw_data = await self._redis_exec(
self.redis_message.hgetall, f"msg:{temp_uuid}"
)
if not raw_data: if not raw_data:
return None return None
# 解码 Redis 返回的字节数据 # 解码 Redis 返回的字节数据
return { return {
k.decode('utf-8') if isinstance(k, bytes) else k: k.decode("utf-8") if isinstance(k, bytes) else k: v.decode("utf-8")
v.decode('utf-8') if isinstance(v, bytes) else v if isinstance(v, bytes)
else v
for k, v in raw_data.items() for k, v in raw_data.items()
} }
except Exception as e: except Exception as e:
@@ -126,7 +155,7 @@ class MessageQueueProcessor:
# result是 (queue_name, value) 的元组，需要解码 # result是 (queue_name, value) 的元组，需要解码
uuid_value = result[1] uuid_value = result[1]
if isinstance(uuid_value, bytes): if isinstance(uuid_value, bytes):
uuid_value = uuid_value.decode('utf-8') uuid_value = uuid_value.decode("utf-8")
message_uuids.append(uuid_value) message_uuids.append(uuid_value)
else: else:
break break
@@ -150,14 +179,19 @@ class MessageQueueProcessor:
for temp_uuid in message_uuids: for temp_uuid in message_uuids:
try: try:
# 获取消息数据并解码 # 获取消息数据并解码
raw_data = await self._redis_exec(self.redis_message.hgetall, f"msg:{temp_uuid}") raw_data = await self._redis_exec(
self.redis_message.hgetall, f"msg:{temp_uuid}"
)
if not raw_data: if not raw_data:
continue continue
# 解码 Redis 返回的字节数据 # 解码 Redis 返回的字节数据
message_data = { message_data = {
k.decode('utf-8') if isinstance(k, bytes) else k: k.decode("utf-8") if isinstance(k, bytes) else k: v.decode(
v.decode('utf-8') if isinstance(v, bytes) else v "utf-8"
)
if isinstance(v, bytes)
else v
for k, v in raw_data.items() for k, v in raw_data.items()
} }
@@ -182,7 +216,9 @@ class MessageQueueProcessor:
# 更新成功状态，包含临时消息ID映射 # 更新成功状态，包含临时消息ID映射
assert msg.message_id is not None assert msg.message_id is not None
await self.update_message_status(temp_uuid, "completed", msg.message_id) await self.update_message_status(
temp_uuid, "completed", msg.message_id
)
# 如果有临时消息ID，存储映射关系并通知客户端更新 # 如果有临时消息ID，存储映射关系并通知客户端更新
if message_data.get("temp_message_id"): if message_data.get("temp_message_id"):
@@ -191,20 +227,30 @@ class MessageQueueProcessor:
self.redis_message.set, self.redis_message.set,
f"temp_to_real:{temp_msg_id}", f"temp_to_real:{temp_msg_id}",
str(msg.message_id), str(msg.message_id),
ex=3600 # 1小时过期 ex=3600, # 1小时过期
) )
# 发送消息ID更新通知到频道 # 发送消息ID更新通知到频道
channel_id = int(message_data["channel_id"]) channel_id = int(message_data["channel_id"])
await self._notify_message_update(channel_id, temp_msg_id, msg.message_id, message_data) await self._notify_message_update(
channel_id, temp_msg_id, msg.message_id, message_data
)
logger.info(f"Message {temp_uuid} persisted to DB with ID {msg.message_id}, temp_id: {message_data.get('temp_message_id')}") logger.info(
f"Message {temp_uuid} persisted to DB with ID {msg.message_id}, temp_id: {message_data.get('temp_message_id')}"
)
except Exception as e: except Exception as e:
logger.error(f"Failed to process message {temp_uuid}: {e}") logger.error(f"Failed to process message {temp_uuid}: {e}")
await self.update_message_status(temp_uuid, "failed") await self.update_message_status(temp_uuid, "failed")
async def _notify_message_update(self, channel_id: int, temp_message_id: int, real_message_id: int, message_data: dict): async def _notify_message_update(
self,
channel_id: int,
temp_message_id: int,
real_message_id: int,
message_data: dict,
):
"""通知客户端消息ID已更新""" """通知客户端消息ID已更新"""
try: try:
# 这里我们需要通过 SignalR 发送消息更新通知 # 这里我们需要通过 SignalR 发送消息更新通知
@@ -215,18 +261,20 @@ class MessageQueueProcessor:
"channel_id": channel_id, "channel_id": channel_id,
"temp_message_id": temp_message_id, "temp_message_id": temp_message_id,
"real_message_id": real_message_id, "real_message_id": real_message_id,
"timestamp": message_data.get("timestamp") "timestamp": message_data.get("timestamp"),
} },
} }
# 发布到 Redis 频道，让 SignalR 服务处理 # 发布到 Redis 频道，让 SignalR 服务处理
await self._redis_exec( await self._redis_exec(
self.redis_message.publish, self.redis_message.publish,
f"chat_updates:{channel_id}", f"chat_updates:{channel_id}",
json.dumps(update_event) json.dumps(update_event),
) )
logger.info(f"Published message update: temp_id={temp_message_id}, real_id={real_message_id}") logger.info(
f"Published message update: temp_id={temp_message_id}, real_id={real_message_id}"
)
except Exception as e: except Exception as e:
logger.error(f"Failed to notify message update: {e}") logger.error(f"Failed to notify message update: {e}")
@@ -249,7 +297,7 @@ class MessageQueueProcessor:
def __del__(self): def __del__(self):
"""清理资源""" """清理资源"""
if hasattr(self, 'executor'): if hasattr(self, "executor"):
self.executor.shutdown(wait=False) self.executor.shutdown(wait=False)
@@ -272,11 +320,13 @@ async def cache_message_to_redis(channel_id: int, message_data: dict, temp_uuid:
await message_queue_processor.cache_message(channel_id, message_data, temp_uuid) await message_queue_processor.cache_message(channel_id, message_data, temp_uuid)
async def get_cached_messages(channel_id: int, limit: int = 50, since: int = 0) -> list[dict]: async def get_cached_messages(
channel_id: int, limit: int = 50, since: int = 0
) -> list[dict]:
"""从 Redis 获取缓存的消息 - 便捷接口""" """从 Redis 获取缓存的消息 - 便捷接口"""
return await message_queue_processor.get_cached_messages(channel_id, limit, since) return await message_queue_processor.get_cached_messages(channel_id, limit, since)
async def get_message_status(temp_uuid: str) -> Optional[dict]: async def get_message_status(temp_uuid: str) -> dict | None:
"""获取消息状态 - 便捷接口""" """获取消息状态 - 便捷接口"""
return await message_queue_processor.get_message_status(temp_uuid) return await message_queue_processor.get_message_status(temp_uuid)

35
app/service/optimized_message.py
View File

@@ -3,14 +3,17 @@
结合 Redis 缓存和异步数据库写入实现实时消息传送 结合 Redis 缓存和异步数据库写入实现实时消息传送
""" """
from typing import Optional from __future__ import annotations
from fastapi import HTTPException
from app.database.chat import ChatMessage, ChatChannel, MessageType, ChannelType, ChatMessageResp from app.database.chat import (
ChannelType,
ChatMessageResp,
MessageType,
)
from app.database.lazer_user import User from app.database.lazer_user import User
from app.router.notification.server import server
from app.service.message_queue import message_queue
from app.log import logger from app.log import logger
from app.service.message_queue import message_queue
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
@@ -28,8 +31,8 @@ class OptimizedMessageService:
content: str, content: str,
sender: User, sender: User,
is_action: bool = False, is_action: bool = False,
user_uuid: Optional[str] = None, user_uuid: str | None = None,
session: Optional[AsyncSession] = None session: AsyncSession | None = None,
) -> ChatMessageResp: ) -> ChatMessageResp:
""" """
快速发送消息（先缓存到 Redis，异步写入数据库） 快速发送消息（先缓存到 Redis，异步写入数据库）
@@ -57,7 +60,7 @@ class OptimizedMessageService:
"type": MessageType.ACTION.value if is_action else MessageType.PLAIN.value, "type": MessageType.ACTION.value if is_action else MessageType.PLAIN.value,
"user_uuid": user_uuid or "", "user_uuid": user_uuid or "",
"channel_type": channel_type.value, "channel_type": channel_type.value,
"channel_name": channel_name "channel_name": channel_name,
} }
# 立即将消息加入 Redis 队列（实时响应） # 立即将消息加入 Redis 队列（实时响应）
@@ -68,13 +71,14 @@ class OptimizedMessageService:
# 创建临时响应对象（简化版本，用于立即响应） # 创建临时响应对象（简化版本，用于立即响应）
from datetime import datetime from datetime import datetime
from app.database.lazer_user import UserResp from app.database.lazer_user import UserResp
# 创建基本的用户响应对象 # 创建基本的用户响应对象
user_resp = UserResp( user_resp = UserResp(
id=sender.id, id=sender.id,
username=sender.username, username=sender.username,
country_code=getattr(sender, 'country_code', 'XX'), country_code=getattr(sender, "country_code", "XX"),
# 基本字段，其他复杂字段可以后续异步加载 # 基本字段，其他复杂字段可以后续异步加载
) )
@@ -86,7 +90,7 @@ class OptimizedMessageService:
sender_id=sender.id, sender_id=sender.id,
sender=user_resp, sender=user_resp,
is_action=is_action, is_action=is_action,
uuid=user_uuid uuid=user_uuid,
) )
temp_response.temp_uuid = temp_uuid # 添加临时 UUID 用于后续更新 temp_response.temp_uuid = temp_uuid # 添加临时 UUID 用于后续更新
@@ -94,10 +98,7 @@ class OptimizedMessageService:
return temp_response return temp_response
async def get_cached_messages( async def get_cached_messages(
self, self, channel_id: int, limit: int = 50, since: int = 0
channel_id: int,
limit: int = 50,
since: int = 0
) -> list[dict]: ) -> list[dict]:
""" """
获取缓存的消息 获取缓存的消息
@@ -112,7 +113,7 @@ class OptimizedMessageService:
""" """
return await self.message_queue.get_cached_messages(channel_id, limit, since) return await self.message_queue.get_cached_messages(channel_id, limit, since)
async def get_message_status(self, temp_uuid: str) -> Optional[dict]: async def get_message_status(self, temp_uuid: str) -> dict | None:
""" """
获取消息状态 获取消息状态
@@ -124,7 +125,9 @@ class OptimizedMessageService:
""" """
return await self.message_queue.get_message_status(temp_uuid) return await self.message_queue.get_message_status(temp_uuid)
async def wait_for_message_persisted(self, temp_uuid: str, timeout: int = 30) -> Optional[dict]: async def wait_for_message_persisted(
self, temp_uuid: str, timeout: int = 30
) -> dict | None:
""" """
等待消息持久化到数据库 等待消息持久化到数据库

225
app/service/redis_message_system.py
View File

@@ -5,16 +5,17 @@
- 支持消息状态同步和故障恢复 - 支持消息状态同步和故障恢复
""" """
from __future__ import annotations
import asyncio import asyncio
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
import json import json
import time import time
import uuid from typing import Any
from datetime import datetime
from typing import Optional, List, Dict, Any
from concurrent.futures import ThreadPoolExecutor
from app.database.chat import ChatMessage, MessageType, ChatMessageResp from app.database.chat import ChatMessage, ChatMessageResp, MessageType
from app.database.lazer_user import User, UserResp, RANKING_INCLUDES from app.database.lazer_user import RANKING_INCLUDES, User, UserResp
from app.dependencies.database import get_redis_message, with_db from app.dependencies.database import get_redis_message, with_db
from app.log import logger from app.log import logger
@@ -25,7 +26,7 @@ class RedisMessageSystem:
def __init__(self): def __init__(self):
self.redis = get_redis_message() self.redis = get_redis_message()
self.executor = ThreadPoolExecutor(max_workers=2) self.executor = ThreadPoolExecutor(max_workers=2)
self._batch_timer: Optional[asyncio.Task] = None self._batch_timer: asyncio.Task | None = None
self._running = False self._running = False
self.batch_interval = 5.0 # 5秒批量存储一次 self.batch_interval = 5.0 # 5秒批量存储一次
self.max_batch_size = 100 # 每批最多处理100条消息 self.max_batch_size = 100 # 每批最多处理100条消息
@@ -35,8 +36,14 @@ class RedisMessageSystem:
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
return await loop.run_in_executor(self.executor, lambda: func(*args, **kwargs)) return await loop.run_in_executor(self.executor, lambda: func(*args, **kwargs))
async def send_message(self, channel_id: int, user: User, content: str, async def send_message(
is_action: bool = False, user_uuid: Optional[str] = None) -> ChatMessageResp: self,
channel_id: int,
user: User,
content: str,
is_action: bool = False,
user_uuid: str | None = None,
) -> ChatMessageResp:
""" """
发送消息 - 立即存储到 Redis 并返回 发送消息 - 立即存储到 Redis 并返回
@@ -68,7 +75,7 @@ class RedisMessageSystem:
"type": MessageType.ACTION.value if is_action else MessageType.PLAIN.value, "type": MessageType.ACTION.value if is_action else MessageType.PLAIN.value,
"uuid": user_uuid or "", "uuid": user_uuid or "",
"status": "cached", # Redis 缓存状态 "status": "cached", # Redis 缓存状态
"created_at": time.time() "created_at": time.time(),
} }
# 立即存储到 Redis # 立即存储到 Redis
@@ -81,6 +88,7 @@ class RedisMessageSystem:
# 确保 statistics 不为空 # 确保 statistics 不为空
if user_resp.statistics is None: if user_resp.statistics is None:
from app.database.statistics import UserStatisticsResp from app.database.statistics import UserStatisticsResp
user_resp.statistics = UserStatisticsResp( user_resp.statistics = UserStatisticsResp(
mode=user.playmode, mode=user.playmode,
global_rank=0, global_rank=0,
@@ -96,7 +104,7 @@ class RedisMessageSystem:
replays_watched_by_others=0, replays_watched_by_others=0,
is_ranked=False, is_ranked=False,
grade_counts={"ssh": 0, "ss": 0, "sh": 0, "s": 0, "a": 0}, grade_counts={"ssh": 0, "ss": 0, "sh": 0, "s": 0, "a": 0},
level={"current": 1, "progress": 0} level={"current": 1, "progress": 0},
) )
response = ChatMessageResp( response = ChatMessageResp(
@@ -107,13 +115,17 @@ class RedisMessageSystem:
sender_id=user.id, sender_id=user.id,
sender=user_resp, sender=user_resp,
is_action=is_action, is_action=is_action,
uuid=user_uuid uuid=user_uuid,
) )
logger.info(f"Message {message_id} sent to Redis cache for channel {channel_id}") logger.info(
f"Message {message_id} sent to Redis cache for channel {channel_id}"
)
return response return response
async def get_messages(self, channel_id: int, limit: int = 50, since: int = 0) -> List[ChatMessageResp]: async def get_messages(
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[ChatMessageResp]:
""" """
获取频道消息 - 优先从 Redis 获取最新消息 获取频道消息 - 优先从 Redis 获取最新消息
@@ -137,19 +149,35 @@ class RedisMessageSystem:
# 获取发送者信息 # 获取发送者信息
sender = await session.get(User, msg_data["sender_id"]) sender = await session.get(User, msg_data["sender_id"])
if sender: if sender:
user_resp = await UserResp.from_db(sender, session, RANKING_INCLUDES) user_resp = await UserResp.from_db(
sender, session, RANKING_INCLUDES
)
if user_resp.statistics is None: if user_resp.statistics is None:
from app.database.statistics import UserStatisticsResp from app.database.statistics import UserStatisticsResp
user_resp.statistics = UserStatisticsResp( user_resp.statistics = UserStatisticsResp(
mode=sender.playmode, mode=sender.playmode,
global_rank=0, country_rank=0, pp=0.0, global_rank=0,
ranked_score=0, hit_accuracy=0.0, play_count=0, country_rank=0,
play_time=0, total_score=0, total_hits=0, pp=0.0,
maximum_combo=0, replays_watched_by_others=0, ranked_score=0,
hit_accuracy=0.0,
play_count=0,
play_time=0,
total_score=0,
total_hits=0,
maximum_combo=0,
replays_watched_by_others=0,
is_ranked=False, is_ranked=False,
grade_counts={"ssh": 0, "ss": 0, "sh": 0, "s": 0, "a": 0}, grade_counts={
level={"current": 1, "progress": 0} "ssh": 0,
"ss": 0,
"sh": 0,
"s": 0,
"a": 0,
},
level={"current": 1, "progress": 0},
) )
message_resp = ChatMessageResp( message_resp = ChatMessageResp(
@@ -160,7 +188,7 @@ class RedisMessageSystem:
sender_id=msg_data["sender_id"], sender_id=msg_data["sender_id"],
sender=user_resp, sender=user_resp,
is_action=msg_data["type"] == MessageType.ACTION.value, is_action=msg_data["type"] == MessageType.ACTION.value,
uuid=msg_data.get("uuid") or None uuid=msg_data.get("uuid") or None,
) )
messages.append(message_resp) messages.append(message_resp)
@@ -178,26 +206,36 @@ class RedisMessageSystem:
async def _generate_message_id(self, channel_id: int) -> int: async def _generate_message_id(self, channel_id: int) -> int:
"""生成唯一的消息ID - 确保全局唯一且严格递增""" """生成唯一的消息ID - 确保全局唯一且严格递增"""
# 使用全局计数器确保所有频道的消息ID都是严格递增的 # 使用全局计数器确保所有频道的消息ID都是严格递增的
message_id = await self._redis_exec(self.redis.incr, "global_message_id_counter") message_id = await self._redis_exec(
self.redis.incr, "global_message_id_counter"
)
# 同时更新频道的最后消息ID，用于客户端状态同步 # 同时更新频道的最后消息ID，用于客户端状态同步
await self._redis_exec(self.redis.set, f"channel:{channel_id}:last_msg_id", message_id) await self._redis_exec(
self.redis.set, f"channel:{channel_id}:last_msg_id", message_id
)
return message_id return message_id
async def _store_to_redis(self, message_id: int, channel_id: int, message_data: Dict[str, Any]): async def _store_to_redis(
self, message_id: int, channel_id: int, message_data: dict[str, Any]
):
"""存储消息到 Redis""" """存储消息到 Redis"""
try: try:
# 存储消息数据 # 存储消息数据
await self._redis_exec( await self._redis_exec(
self.redis.hset, self.redis.hset,
f"msg:{channel_id}:{message_id}", f"msg:{channel_id}:{message_id}",
mapping={k: json.dumps(v) if isinstance(v, (dict, list)) else str(v) mapping={
for k, v in message_data.items()} k: json.dumps(v) if isinstance(v, (dict, list)) else str(v)
for k, v in message_data.items()
},
) )
# 设置消息过期时间（7天） # 设置消息过期时间（7天）
await self._redis_exec(self.redis.expire, f"msg:{channel_id}:{message_id}", 604800) await self._redis_exec(
self.redis.expire, f"msg:{channel_id}:{message_id}", 604800
)
# 清理可能存在的错误类型键，然后添加到频道消息列表（按时间排序） # 清理可能存在的错误类型键，然后添加到频道消息列表（按时间排序）
channel_messages_key = f"channel:{channel_id}:messages" channel_messages_key = f"channel:{channel_id}:messages"
@@ -206,10 +244,14 @@ class RedisMessageSystem:
try: try:
key_type = await self._redis_exec(self.redis.type, channel_messages_key) key_type = await self._redis_exec(self.redis.type, channel_messages_key)
if key_type and key_type != "zset": if key_type and key_type != "zset":
logger.warning(f"Deleting Redis key {channel_messages_key} with wrong type: {key_type}") logger.warning(
f"Deleting Redis key {channel_messages_key} with wrong type: {key_type}"
)
await self._redis_exec(self.redis.delete, channel_messages_key) await self._redis_exec(self.redis.delete, channel_messages_key)
except Exception as type_check_error: except Exception as type_check_error:
logger.warning(f"Failed to check key type for {channel_messages_key}: {type_check_error}") logger.warning(
f"Failed to check key type for {channel_messages_key}: {type_check_error}"
)
# 如果检查失败，直接删除键以确保清理 # 如果检查失败，直接删除键以确保清理
await self._redis_exec(self.redis.delete, channel_messages_key) await self._redis_exec(self.redis.delete, channel_messages_key)
@@ -217,20 +259,26 @@ class RedisMessageSystem:
await self._redis_exec( await self._redis_exec(
self.redis.zadd, self.redis.zadd,
channel_messages_key, channel_messages_key,
{f"msg:{channel_id}:{message_id}": message_id} {f"msg:{channel_id}:{message_id}": message_id},
) )
# 保持频道消息列表大小（最多1000条） # 保持频道消息列表大小（最多1000条）
await self._redis_exec(self.redis.zremrangebyrank, channel_messages_key, 0, -1001) await self._redis_exec(
self.redis.zremrangebyrank, channel_messages_key, 0, -1001
)
# 添加到待持久化队列 # 添加到待持久化队列
await self._redis_exec(self.redis.lpush, "pending_messages", f"{channel_id}:{message_id}") await self._redis_exec(
self.redis.lpush, "pending_messages", f"{channel_id}:{message_id}"
)
except Exception as e: except Exception as e:
logger.error(f"Failed to store message to Redis: {e}") logger.error(f"Failed to store message to Redis: {e}")
raise raise
async def _get_from_redis(self, channel_id: int, limit: int = 50, since: int = 0) -> List[Dict[str, Any]]: async def _get_from_redis(
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[dict[str, Any]]:
"""从 Redis 获取消息""" """从 Redis 获取消息"""
try: try:
# 获取消息键列表，按消息ID排序 # 获取消息键列表，按消息ID排序
@@ -239,21 +287,21 @@ class RedisMessageSystem:
message_keys = await self._redis_exec( message_keys = await self._redis_exec(
self.redis.zrangebyscore, self.redis.zrangebyscore,
f"channel:{channel_id}:messages", f"channel:{channel_id}:messages",
since + 1, "+inf", since + 1,
start=0, num=limit "+inf",
start=0,
num=limit,
) )
else: else:
# 获取最新的消息（倒序获取，然后反转） # 获取最新的消息（倒序获取，然后反转）
message_keys = await self._redis_exec( message_keys = await self._redis_exec(
self.redis.zrevrange, self.redis.zrevrange, f"channel:{channel_id}:messages", 0, limit - 1
f"channel:{channel_id}:messages",
0, limit - 1
) )
messages = [] messages = []
for key in message_keys: for key in message_keys:
if isinstance(key, bytes): if isinstance(key, bytes):
key = key.decode('utf-8') key = key.decode("utf-8")
# 获取消息数据 # 获取消息数据
raw_data = await self._redis_exec(self.redis.hgetall, key) raw_data = await self._redis_exec(self.redis.hgetall, key)
@@ -262,17 +310,19 @@ class RedisMessageSystem:
message_data = {} message_data = {}
for k, v in raw_data.items(): for k, v in raw_data.items():
if isinstance(k, bytes): if isinstance(k, bytes):
k = k.decode('utf-8') k = k.decode("utf-8")
if isinstance(v, bytes): if isinstance(v, bytes):
v = v.decode('utf-8') v = v.decode("utf-8")
# 尝试解析 JSON # 尝试解析 JSON
try: try:
if k in ['grade_counts', 'level'] or v.startswith(('{', '[')): if k in ["grade_counts", "level"] or v.startswith(
("{", "[")
):
message_data[k] = json.loads(v) message_data[k] = json.loads(v)
elif k in ['message_id', 'channel_id', 'sender_id']: elif k in ["message_id", "channel_id", "sender_id"]:
message_data[k] = int(v) message_data[k] = int(v)
elif k == 'created_at': elif k == "created_at":
message_data[k] = float(v) message_data[k] = float(v)
else: else:
message_data[k] = v message_data[k] = v
@@ -282,7 +332,7 @@ class RedisMessageSystem:
messages.append(message_data) messages.append(message_data)
# 确保消息按ID正序排序（时间顺序） # 确保消息按ID正序排序（时间顺序）
messages.sort(key=lambda x: x.get('message_id', 0)) messages.sort(key=lambda x: x.get("message_id", 0))
# 如果是获取最新消息（since=0），需要保持倒序（最新的在前面） # 如果是获取最新消息（since=0），需要保持倒序（最新的在前面）
if since == 0: if since == 0:
@@ -294,11 +344,13 @@ class RedisMessageSystem:
logger.error(f"Failed to get messages from Redis: {e}") logger.error(f"Failed to get messages from Redis: {e}")
return [] return []
async def _backfill_from_database(self, channel_id: int, existing_messages: List[ChatMessageResp], limit: int): async def _backfill_from_database(
self, channel_id: int, existing_messages: list[ChatMessageResp], limit: int
):
"""从数据库补充历史消息""" """从数据库补充历史消息"""
try: try:
# 找到最小的消息ID # 找到最小的消息ID
min_id = float('inf') min_id = float("inf")
if existing_messages: if existing_messages:
for msg in existing_messages: for msg in existing_messages:
if msg.message_id is not None and msg.message_id < min_id: if msg.message_id is not None and msg.message_id < min_id:
@@ -310,12 +362,11 @@ class RedisMessageSystem:
return return
async with with_db() as session: async with with_db() as session:
from sqlmodel import select, col from sqlmodel import col, select
query = select(ChatMessage).where(
ChatMessage.channel_id == channel_id
)
if min_id != float('inf'): query = select(ChatMessage).where(ChatMessage.channel_id == channel_id)
if min_id != float("inf"):
query = query.where(col(ChatMessage.message_id) < min_id) query = query.where(col(ChatMessage.message_id) < min_id)
query = query.order_by(col(ChatMessage.message_id).desc()).limit(needed) query = query.order_by(col(ChatMessage.message_id).desc()).limit(needed)
@@ -329,24 +380,33 @@ class RedisMessageSystem:
except Exception as e: except Exception as e:
logger.error(f"Failed to backfill from database: {e}") logger.error(f"Failed to backfill from database: {e}")
async def _get_from_database_only(self, channel_id: int, limit: int, since: int) -> List[ChatMessageResp]: async def _get_from_database_only(
self, channel_id: int, limit: int, since: int
) -> list[ChatMessageResp]:
"""仅从数据库获取消息（回退方案）""" """仅从数据库获取消息（回退方案）"""
try: try:
async with with_db() as session: async with with_db() as session:
from sqlmodel import select, col from sqlmodel import col, select
query = select(ChatMessage).where(ChatMessage.channel_id == channel_id) query = select(ChatMessage).where(ChatMessage.channel_id == channel_id)
if since > 0: if since > 0:
# 获取指定ID之后的消息，按ID正序 # 获取指定ID之后的消息，按ID正序
query = query.where(col(ChatMessage.message_id) > since) query = query.where(col(ChatMessage.message_id) > since)
query = query.order_by(col(ChatMessage.message_id).asc()).limit(limit) query = query.order_by(col(ChatMessage.message_id).asc()).limit(
limit
)
else: else:
# 获取最新消息，按ID倒序（最新的在前面） # 获取最新消息，按ID倒序（最新的在前面）
query = query.order_by(col(ChatMessage.message_id).desc()).limit(limit) query = query.order_by(col(ChatMessage.message_id).desc()).limit(
limit
)
messages = (await session.exec(query)).all() messages = (await session.exec(query)).all()
results = [await ChatMessageResp.from_db(msg, session) for msg in messages] results = [
await ChatMessageResp.from_db(msg, session) for msg in messages
]
# 如果是 since > 0，保持正序；否则反转为时间正序 # 如果是 since > 0，保持正序；否则反转为时间正序
if since == 0: if since == 0:
@@ -374,7 +434,7 @@ class RedisMessageSystem:
# key 是 (queue_name, value) 的元组 # key 是 (queue_name, value) 的元组
value = key[1] value = key[1]
if isinstance(value, bytes): if isinstance(value, bytes):
value = value.decode('utf-8') value = value.decode("utf-8")
message_keys.append(value) message_keys.append(value)
else: else:
break break
@@ -390,13 +450,13 @@ class RedisMessageSystem:
logger.info("Stopped batch persistence to database") logger.info("Stopped batch persistence to database")
async def _process_message_batch(self, message_keys: List[str]): async def _process_message_batch(self, message_keys: list[str]):
"""处理消息批次""" """处理消息批次"""
async with with_db() as session: async with with_db() as session:
for key in message_keys: for key in message_keys:
try: try:
# 解析频道ID和消息ID # 解析频道ID和消息ID
channel_id, message_id = map(int, key.split(':')) channel_id, message_id = map(int, key.split(":"))
# 从 Redis 获取消息数据 # 从 Redis 获取消息数据
raw_data = await self._redis_exec( raw_data = await self._redis_exec(
@@ -410,9 +470,9 @@ class RedisMessageSystem:
message_data = {} message_data = {}
for k, v in raw_data.items(): for k, v in raw_data.items():
if isinstance(k, bytes): if isinstance(k, bytes):
k = k.decode('utf-8') k = k.decode("utf-8")
if isinstance(v, bytes): if isinstance(v, bytes):
v = v.decode('utf-8') v = v.decode("utf-8")
message_data[k] = v message_data[k] = v
# 检查消息是否已存在于数据库 # 检查消息是否已存在于数据库
@@ -428,7 +488,7 @@ class RedisMessageSystem:
content=message_data["content"], content=message_data["content"],
timestamp=datetime.fromisoformat(message_data["timestamp"]), timestamp=datetime.fromisoformat(message_data["timestamp"]),
type=MessageType(message_data["type"]), type=MessageType(message_data["type"]),
uuid=message_data.get("uuid") or None uuid=message_data.get("uuid") or None,
) )
session.add(db_message) session.add(db_message)
@@ -437,7 +497,8 @@ class RedisMessageSystem:
await self._redis_exec( await self._redis_exec(
self.redis.hset, self.redis.hset,
f"msg:{channel_id}:{message_id}", f"msg:{channel_id}:{message_id}",
"status", "persisted" "status",
"persisted",
) )
logger.debug(f"Message {message_id} persisted to database") logger.debug(f"Message {message_id} persisted to database")
@@ -448,7 +509,9 @@ class RedisMessageSystem:
# 提交批次 # 提交批次
try: try:
await session.commit() await session.commit()
logger.info(f"Batch of {len(message_keys)} messages committed to database") logger.info(
f"Batch of {len(message_keys)} messages committed to database"
)
except Exception as e: except Exception as e:
logger.error(f"Failed to commit message batch: {e}") logger.error(f"Failed to commit message batch: {e}")
await session.rollback() await session.rollback()
@@ -469,28 +532,34 @@ class RedisMessageSystem:
await self._cleanup_redis_keys() await self._cleanup_redis_keys()
async with with_db() as session: async with with_db() as session:
from sqlmodel import select, func from sqlmodel import func, select
# 获取数据库中最大的消息ID # 获取数据库中最大的消息ID
result = await session.exec( result = await session.exec(select(func.max(ChatMessage.message_id)))
select(func.max(ChatMessage.message_id))
)
max_id = result.one() or 0 max_id = result.one() or 0
# 检查 Redis 中的计数器值 # 检查 Redis 中的计数器值
current_counter = await self._redis_exec(self.redis.get, "global_message_id_counter") current_counter = await self._redis_exec(
self.redis.get, "global_message_id_counter"
)
current_counter = int(current_counter) if current_counter else 0 current_counter = int(current_counter) if current_counter else 0
# 设置计数器为两者中的最大值 # 设置计数器为两者中的最大值
initial_counter = max(max_id, current_counter) initial_counter = max(max_id, current_counter)
await self._redis_exec(self.redis.set, "global_message_id_counter", initial_counter) await self._redis_exec(
self.redis.set, "global_message_id_counter", initial_counter
)
logger.info(f"Initialized global message ID counter to {initial_counter}") logger.info(
f"Initialized global message ID counter to {initial_counter}"
)
except Exception as e: except Exception as e:
logger.error(f"Failed to initialize message counter: {e}") logger.error(f"Failed to initialize message counter: {e}")
# 如果初始化失败，设置一个安全的起始值 # 如果初始化失败，设置一个安全的起始值
await self._redis_exec(self.redis.setnx, "global_message_id_counter", 1000000) await self._redis_exec(
self.redis.setnx, "global_message_id_counter", 1000000
)
async def _cleanup_redis_keys(self): async def _cleanup_redis_keys(self):
"""清理可能存在问题的 Redis 键""" """清理可能存在问题的 Redis 键"""
@@ -501,12 +570,14 @@ class RedisMessageSystem:
for key in keys: for key in keys:
if isinstance(key, bytes): if isinstance(key, bytes):
key = key.decode('utf-8') key = key.decode("utf-8")
try: try:
key_type = await self._redis_exec(self.redis.type, key) key_type = await self._redis_exec(self.redis.type, key)
if key_type and key_type != "zset": if key_type and key_type != "zset":
logger.warning(f"Cleaning up Redis key {key} with wrong type: {key_type}") logger.warning(
f"Cleaning up Redis key {key} with wrong type: {key_type}"
)
await self._redis_exec(self.redis.delete, key) await self._redis_exec(self.redis.delete, key)
except Exception as cleanup_error: except Exception as cleanup_error:
logger.warning(f"Failed to cleanup key {key}: {cleanup_error}") logger.warning(f"Failed to cleanup key {key}: {cleanup_error}")
@@ -529,7 +600,7 @@ class RedisMessageSystem:
def __del__(self): def __del__(self):
"""清理资源""" """清理资源"""
if hasattr(self, 'executor'): if hasattr(self, "executor"):
self.executor.shutdown(wait=False) self.executor.shutdown(wait=False)

23
app/service/stats_cleanup.py
View File

@@ -1,9 +1,14 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime, timedelta from datetime import datetime, timedelta
from app.dependencies.database import get_redis, get_redis_message from app.dependencies.database import get_redis, get_redis_message
from app.log import logger from app.log import logger
from app.router.v2.stats import REDIS_ONLINE_USERS_KEY, REDIS_PLAYING_USERS_KEY, _redis_exec from app.router.v2.stats import (
REDIS_ONLINE_USERS_KEY,
REDIS_PLAYING_USERS_KEY,
_redis_exec,
)
async def cleanup_stale_online_users() -> tuple[int, int]: async def cleanup_stale_online_users() -> tuple[int, int]:
@@ -26,7 +31,9 @@ async def cleanup_stale_online_users() -> tuple[int, int]:
# 对于在线用户，我们检查metadata在线标记 # 对于在线用户，我们检查metadata在线标记
stale_online_users = [] stale_online_users = []
for user_id in online_users: for user_id in online_users:
user_id_str = user_id.decode() if isinstance(user_id, bytes) else str(user_id) user_id_str = (
user_id.decode() if isinstance(user_id, bytes) else str(user_id)
)
metadata_key = f"metadata:online:{user_id_str}" metadata_key = f"metadata:online:{user_id_str}"
# 如果metadata标记不存在，说明用户已经离线 # 如果metadata标记不存在，说明用户已经离线
@@ -35,14 +42,18 @@ async def cleanup_stale_online_users() -> tuple[int, int]:
# 清理过期的在线用户 # 清理过期的在线用户
if stale_online_users: if stale_online_users:
await _redis_exec(redis_sync.srem, REDIS_ONLINE_USERS_KEY, *stale_online_users) await _redis_exec(
redis_sync.srem, REDIS_ONLINE_USERS_KEY, *stale_online_users
)
online_cleaned = len(stale_online_users) online_cleaned = len(stale_online_users)
logger.info(f"Cleaned {online_cleaned} stale online users") logger.info(f"Cleaned {online_cleaned} stale online users")
# 对于游玩用户，我们也检查对应的spectator状态 # 对于游玩用户，我们也检查对应的spectator状态
stale_playing_users = [] stale_playing_users = []
for user_id in playing_users: for user_id in playing_users:
user_id_str = user_id.decode() if isinstance(user_id, bytes) else str(user_id) user_id_str = (
user_id.decode() if isinstance(user_id, bytes) else str(user_id)
)
# 如果用户不在在线用户列表中，说明已经离线，也应该从游玩列表中移除 # 如果用户不在在线用户列表中，说明已经离线，也应该从游玩列表中移除
if user_id_str in stale_online_users or user_id_str not in [ if user_id_str in stale_online_users or user_id_str not in [
@@ -52,7 +63,9 @@ async def cleanup_stale_online_users() -> tuple[int, int]:
# 清理过期的游玩用户 # 清理过期的游玩用户
if stale_playing_users: if stale_playing_users:
await _redis_exec(redis_sync.srem, REDIS_PLAYING_USERS_KEY, *stale_playing_users) await _redis_exec(
redis_sync.srem, REDIS_PLAYING_USERS_KEY, *stale_playing_users
)
playing_cleaned = len(stale_playing_users) playing_cleaned = len(stale_playing_users)
logger.info(f"Cleaned {playing_cleaned} stale playing users") logger.info(f"Cleaned {playing_cleaned} stale playing users")

29
app/service/stats_scheduler.py
View File

@@ -5,8 +5,11 @@ from datetime import datetime, timedelta
from app.log import logger from app.log import logger
from app.router.v2.stats import record_hourly_stats, update_registered_users_count from app.router.v2.stats import record_hourly_stats, update_registered_users_count
from app.service.stats_cleanup import cleanup_stale_online_users, refresh_redis_key_expiry
from app.service.enhanced_interval_stats import EnhancedIntervalStatsManager from app.service.enhanced_interval_stats import EnhancedIntervalStatsManager
from app.service.stats_cleanup import (
cleanup_stale_online_users,
refresh_redis_key_expiry,
)
class StatsScheduler: class StatsScheduler:
@@ -61,7 +64,9 @@ class StatsScheduler:
# 计算当前区间边界 # 计算当前区间边界
current_minute = (now.minute // 30) * 30 current_minute = (now.minute // 30) * 30
current_interval_end = now.replace(minute=current_minute, second=0, microsecond=0) + timedelta(minutes=30) current_interval_end = now.replace(
minute=current_minute, second=0, microsecond=0
) + timedelta(minutes=30)
# 如果已经过了当前区间结束时间，立即处理 # 如果已经过了当前区间结束时间，立即处理
if now >= current_interval_end: if now >= current_interval_end:
@@ -73,7 +78,9 @@ class StatsScheduler:
# 确保至少等待1分钟，最多等待31分钟 # 确保至少等待1分钟，最多等待31分钟
sleep_seconds = max(min(sleep_seconds, 31 * 60), 60) sleep_seconds = max(min(sleep_seconds, 31 * 60), 60)
logger.debug(f"Next interval finalization in {sleep_seconds/60:.1f} minutes at {current_interval_end.strftime('%H:%M:%S')}") logger.debug(
f"Next interval finalization in {sleep_seconds / 60:.1f} minutes at {current_interval_end.strftime('%H:%M:%S')}"
)
await asyncio.sleep(sleep_seconds) await asyncio.sleep(sleep_seconds)
if not self._running: if not self._running:
@@ -82,11 +89,15 @@ class StatsScheduler:
# 完成当前区间并记录到历史 # 完成当前区间并记录到历史
finalized_stats = await EnhancedIntervalStatsManager.finalize_interval() finalized_stats = await EnhancedIntervalStatsManager.finalize_interval()
if finalized_stats: if finalized_stats:
logger.info(f"Finalized enhanced interval statistics at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}") logger.info(
f"Finalized enhanced interval statistics at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}"
)
else: else:
# 如果区间完成失败，使用原有方式记录 # 如果区间完成失败，使用原有方式记录
await record_hourly_stats() await record_hourly_stats()
logger.info(f"Recorded hourly statistics (fallback) at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}") logger.info(
f"Recorded hourly statistics (fallback) at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}"
)
# 开始新的区间统计 # 开始新的区间统计
await EnhancedIntervalStatsManager.initialize_current_interval() await EnhancedIntervalStatsManager.initialize_current_interval()
@@ -124,7 +135,9 @@ class StatsScheduler:
try: try:
online_cleaned, playing_cleaned = await cleanup_stale_online_users() online_cleaned, playing_cleaned = await cleanup_stale_online_users()
if online_cleaned > 0 or playing_cleaned > 0: if online_cleaned > 0 or playing_cleaned > 0:
logger.info(f"Initial cleanup: removed {online_cleaned} stale online users, {playing_cleaned} stale playing users") logger.info(
f"Initial cleanup: removed {online_cleaned} stale online users, {playing_cleaned} stale playing users"
)
await refresh_redis_key_expiry() await refresh_redis_key_expiry()
except Exception as e: except Exception as e:
@@ -141,7 +154,9 @@ class StatsScheduler:
# 清理过期用户 # 清理过期用户
online_cleaned, playing_cleaned = await cleanup_stale_online_users() online_cleaned, playing_cleaned = await cleanup_stale_online_users()
if online_cleaned > 0 or playing_cleaned > 0: if online_cleaned > 0 or playing_cleaned > 0:
logger.info(f"Cleanup: removed {online_cleaned} stale online users, {playing_cleaned} stale playing users") logger.info(
f"Cleanup: removed {online_cleaned} stale online users, {playing_cleaned} stale playing users"
)
# 刷新Redis key过期时间 # 刷新Redis key过期时间
await refresh_redis_key_expiry() await refresh_redis_key_expiry()

2
app/signalr/hub/metadata.py
View File

@@ -95,6 +95,7 @@ class MetadataHub(Hub[MetadataClientState]):
# Remove from online user tracking # Remove from online user tracking
from app.router.v2.stats import remove_online_user from app.router.v2.stats import remove_online_user
asyncio.create_task(remove_online_user(user_id)) asyncio.create_task(remove_online_user(user_id))
if state.pushable: if state.pushable:
@@ -125,6 +126,7 @@ class MetadataHub(Hub[MetadataClientState]):
# Track online user # Track online user
from app.router.v2.stats import add_online_user from app.router.v2.stats import add_online_user
asyncio.create_task(add_online_user(user_id)) asyncio.create_task(add_online_user(user_id))
async with with_db() as session: async with with_db() as session:

49
app/signalr/hub/multiplayer.py
View File

@@ -166,6 +166,7 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
# Remove from online user tracking # Remove from online user tracking
from app.router.v2.stats import remove_online_user from app.router.v2.stats import remove_online_user
asyncio.create_task(remove_online_user(user_id)) asyncio.create_task(remove_online_user(user_id))
if state.room_id != 0 and state.room_id in self.rooms: if state.room_id != 0 and state.room_id in self.rooms:
@@ -183,6 +184,7 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
# Track online user # Track online user
from app.router.v2.stats import add_online_user from app.router.v2.stats import add_online_user
asyncio.create_task(add_online_user(client.user_id)) asyncio.create_task(add_online_user(client.user_id))
def _ensure_in_room(self, client: Client) -> ServerMultiplayerRoom: def _ensure_in_room(self, client: Client) -> ServerMultiplayerRoom:
@@ -674,11 +676,17 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
MultiplayerUserState.RESULTS, # Allow spectating after results MultiplayerUserState.RESULTS, # Allow spectating after results
): ):
# Allow spectating during gameplay states only if the room is in appropriate state # Allow spectating during gameplay states only if the room is in appropriate state
if not (old.is_playing and room.room.state in ( if not (
MultiplayerRoomState.WAITING_FOR_LOAD, old.is_playing
MultiplayerRoomState.PLAYING and room.room.state
)): in (
raise InvokeException(f"Cannot change state from {old} to {new}") MultiplayerRoomState.WAITING_FOR_LOAD,
MultiplayerRoomState.PLAYING,
)
):
raise InvokeException(
f"Cannot change state from {old} to {new}"
)
case _: case _:
raise InvokeException(f"Invalid state transition from {old} to {new}") raise InvokeException(f"Invalid state transition from {old} to {new}")
@@ -738,10 +746,7 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
) )
async def handle_spectator_state_change( async def handle_spectator_state_change(
self, self, client: Client, room: ServerMultiplayerRoom, user: MultiplayerRoomUser
client: Client,
room: ServerMultiplayerRoom,
user: MultiplayerRoomUser
): ):
""" """
Handle special logic for users entering spectator mode during ongoing gameplay. Handle special logic for users entering spectator mode during ongoing gameplay.
@@ -763,9 +768,7 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
await self.call_noblock(client, "LoadRequested") await self.call_noblock(client, "LoadRequested")
async def _send_current_gameplay_state_to_spectator( async def _send_current_gameplay_state_to_spectator(
self, self, client: Client, room: ServerMultiplayerRoom
client: Client,
room: ServerMultiplayerRoom
): ):
""" """
Send current gameplay state information to a newly joined spectator. Send current gameplay state information to a newly joined spectator.
@@ -773,11 +776,7 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
""" """
try: try:
# Send current room state # Send current room state
await self.call_noblock( await self.call_noblock(client, "RoomStateChanged", room.room.state)
client,
"RoomStateChanged",
room.room.state
)
# Send current user states for all players # Send current user states for all players
for room_user in room.room.users: for room_user in room.room.users:
@@ -798,9 +797,7 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
) )
async def _send_room_state_to_new_user( async def _send_room_state_to_new_user(
self, self, client: Client, room: ServerMultiplayerRoom
client: Client,
room: ServerMultiplayerRoom
): ):
""" """
Send complete room state to a newly joined user. Send complete room state to a newly joined user.
@@ -809,16 +806,12 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
try: try:
# Send current room state # Send current room state
if room.room.state != MultiplayerRoomState.OPEN: if room.room.state != MultiplayerRoomState.OPEN:
await self.call_noblock( await self.call_noblock(client, "RoomStateChanged", room.room.state)
client,
"RoomStateChanged",
room.room.state
)
# If room is in gameplay state, send LoadRequested immediately # If room is in gameplay state, send LoadRequested immediately
if room.room.state in ( if room.room.state in (
MultiplayerRoomState.WAITING_FOR_LOAD, MultiplayerRoomState.WAITING_FOR_LOAD,
MultiplayerRoomState.PLAYING MultiplayerRoomState.PLAYING,
): ):
logger.info( logger.info(
f"[MultiplayerHub] Sending LoadRequested to user {client.user_id} " f"[MultiplayerHub] Sending LoadRequested to user {client.user_id} "
@@ -849,9 +842,7 @@ class MultiplayerHub(Hub[MultiplayerClientState]):
) )
async def _sync_with_spectator_hub( async def _sync_with_spectator_hub(
self, self, client: Client, room: ServerMultiplayerRoom
client: Client,
room: ServerMultiplayerRoom
): ):
""" """
Sync with SpectatorHub to ensure cross-hub spectating works properly. Sync with SpectatorHub to ensure cross-hub spectating works properly.

53
app/signalr/hub/spectator.py
View File

@@ -173,6 +173,7 @@ class SpectatorHub(Hub[StoreClientState]):
# Remove from online and playing tracking # Remove from online and playing tracking
from app.router.v2.stats import remove_online_user from app.router.v2.stats import remove_online_user
asyncio.create_task(remove_online_user(user_id)) asyncio.create_task(remove_online_user(user_id))
if state.state: if state.state:
@@ -181,10 +182,10 @@ class SpectatorHub(Hub[StoreClientState]):
# Critical fix: Notify all watched users that this spectator has disconnected # Critical fix: Notify all watched users that this spectator has disconnected
# This matches the official CleanUpState implementation # This matches the official CleanUpState implementation
for watched_user_id in state.watched_user: for watched_user_id in state.watched_user:
if (target_client := self.get_client_by_id(str(watched_user_id))) is not None: if (
await self.call_noblock( target_client := self.get_client_by_id(str(watched_user_id))
target_client, "UserEndedWatching", user_id ) is not None:
) await self.call_noblock(target_client, "UserEndedWatching", user_id)
logger.debug( logger.debug(
f"[SpectatorHub] Notified {watched_user_id} that {user_id} stopped watching" f"[SpectatorHub] Notified {watched_user_id} that {user_id} stopped watching"
) )
@@ -198,6 +199,7 @@ class SpectatorHub(Hub[StoreClientState]):
# Track online user # Track online user
from app.router.v2.stats import add_online_user from app.router.v2.stats import add_online_user
asyncio.create_task(add_online_user(client.user_id)) asyncio.create_task(add_online_user(client.user_id))
# Send all current player states to the new client # Send all current player states to the new client
@@ -216,7 +218,9 @@ class SpectatorHub(Hub[StoreClientState]):
try: try:
await self.call_noblock(client, "UserBeganPlaying", user_id, state) await self.call_noblock(client, "UserBeganPlaying", user_id, state)
except Exception as e: except Exception as e:
logger.debug(f"[SpectatorHub] Failed to send state for user {user_id}: {e}") logger.debug(
f"[SpectatorHub] Failed to send state for user {user_id}: {e}"
)
# Also sync with MultiplayerHub for cross-hub spectating # Also sync with MultiplayerHub for cross-hub spectating
await self._sync_with_multiplayer_hub(client) await self._sync_with_multiplayer_hub(client)
@@ -234,9 +238,10 @@ class SpectatorHub(Hub[StoreClientState]):
for room_id, server_room in MultiplayerHubs.rooms.items(): for room_id, server_room in MultiplayerHubs.rooms.items():
for room_user in server_room.room.users: for room_user in server_room.room.users:
# If user is playing in multiplayer but we don't have their spectator state # If user is playing in multiplayer but we don't have their spectator state
if (room_user.state.is_playing and if (
room_user.user_id not in self.state): room_user.state.is_playing
and room_user.user_id not in self.state
):
# Create a synthetic SpectatorState for multiplayer players # Create a synthetic SpectatorState for multiplayer players
# This helps with cross-hub spectating # This helps with cross-hub spectating
try: try:
@@ -245,7 +250,7 @@ class SpectatorHub(Hub[StoreClientState]):
ruleset_id=room_user.ruleset_id or 0, # Default to osu! ruleset_id=room_user.ruleset_id or 0, # Default to osu!
mods=room_user.mods, mods=room_user.mods,
state=SpectatedUserState.Playing, state=SpectatedUserState.Playing,
maximum_statistics={} maximum_statistics={},
) )
await self.call_noblock( await self.call_noblock(
@@ -258,7 +263,9 @@ class SpectatorHub(Hub[StoreClientState]):
f"[SpectatorHub] Sent synthetic multiplayer state for user {room_user.user_id}" f"[SpectatorHub] Sent synthetic multiplayer state for user {room_user.user_id}"
) )
except Exception as e: except Exception as e:
logger.debug(f"[SpectatorHub] Failed to create synthetic state: {e}") logger.debug(
f"[SpectatorHub] Failed to create synthetic state: {e}"
)
except Exception as e: except Exception as e:
logger.debug(f"[SpectatorHub] Failed to sync with MultiplayerHub: {e}") logger.debug(f"[SpectatorHub] Failed to sync with MultiplayerHub: {e}")
@@ -306,6 +313,7 @@ class SpectatorHub(Hub[StoreClientState]):
# Track playing user # Track playing user
from app.router.v2.stats import add_playing_user from app.router.v2.stats import add_playing_user
asyncio.create_task(add_playing_user(user_id)) asyncio.create_task(add_playing_user(user_id))
# # 预缓存beatmap文件以加速后续PP计算 # # 预缓存beatmap文件以加速后续PP计算
@@ -359,6 +367,7 @@ class SpectatorHub(Hub[StoreClientState]):
# Remove from playing user tracking # Remove from playing user tracking
from app.router.v2.stats import remove_playing_user from app.router.v2.stats import remove_playing_user
asyncio.create_task(remove_playing_user(user_id)) asyncio.create_task(remove_playing_user(user_id))
store.state = None store.state = None
@@ -473,7 +482,9 @@ class SpectatorHub(Hub[StoreClientState]):
if state.state == SpectatedUserState.Playing: if state.state == SpectatedUserState.Playing:
state.state = SpectatedUserState.Quit state.state = SpectatedUserState.Quit
logger.debug(f"[SpectatorHub] Changed state from Playing to Quit for user {user_id}") logger.debug(
f"[SpectatorHub] Changed state from Playing to Quit for user {user_id}"
)
# Calculate exit time safely # Calculate exit time safely
exit_time = 0 exit_time = 0
@@ -526,7 +537,9 @@ class SpectatorHub(Hub[StoreClientState]):
# Get target user's current state if it exists # Get target user's current state if it exists
target_store = self.state.get(target_id) target_store = self.state.get(target_id)
if target_store and target_store.state: if target_store and target_store.state:
logger.debug(f"[SpectatorHub] {target_id} is currently {target_store.state.state}") logger.debug(
f"[SpectatorHub] {target_id} is currently {target_store.state.state}"
)
# Send current state to the watcher immediately # Send current state to the watcher immediately
await self.call_noblock( await self.call_noblock(
client, client,
@@ -552,7 +565,9 @@ class SpectatorHub(Hub[StoreClientState]):
await session.exec(select(User.username).where(User.id == user_id)) await session.exec(select(User.username).where(User.id == user_id))
).first() ).first()
if not username: if not username:
logger.warning(f"[SpectatorHub] Could not find username for user {user_id}") logger.warning(
f"[SpectatorHub] Could not find username for user {user_id}"
)
return return
# Notify target user that someone started watching # Notify target user that someone started watching
@@ -562,7 +577,9 @@ class SpectatorHub(Hub[StoreClientState]):
await self.call_noblock( await self.call_noblock(
target_client, "UserStartedWatching", watcher_info target_client, "UserStartedWatching", watcher_info
) )
logger.debug(f"[SpectatorHub] Notified {target_id} that {username} started watching") logger.debug(
f"[SpectatorHub] Notified {target_id} that {username} started watching"
)
except Exception as e: except Exception as e:
logger.error(f"[SpectatorHub] Error notifying target user {target_id}: {e}") logger.error(f"[SpectatorHub] Error notifying target user {target_id}: {e}")
@@ -585,6 +602,10 @@ class SpectatorHub(Hub[StoreClientState]):
# Notify target user that watcher stopped watching # Notify target user that watcher stopped watching
if (target_client := self.get_client_by_id(str(target_id))) is not None: if (target_client := self.get_client_by_id(str(target_id))) is not None:
await self.call_noblock(target_client, "UserEndedWatching", user_id) await self.call_noblock(target_client, "UserEndedWatching", user_id)
logger.debug(f"[SpectatorHub] Notified {target_id} that {user_id} stopped watching") logger.debug(
f"[SpectatorHub] Notified {target_id} that {user_id} stopped watching"
)
else: else:
logger.debug(f"[SpectatorHub] Target user {target_id} not found for end watching notification") logger.debug(
f"[SpectatorHub] Target user {target_id} not found for end watching notification"
)