Bennett/g0v0-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

整理代码

Browse Source
This commit is contained in:
咕谷酱
2025-08-22 05:57:28 +08:00
parent ad131c0158
commit ce465aa049
20 changed files with 1078 additions and 799 deletions
Download Patch File Download Diff File Expand all files Collapse all files

360
app/service/enhanced_interval_stats.py
View File

@@ -4,11 +4,9 @@
from __future__ import annotations
import json
import asyncio
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Dict, Set, Optional, List
from dataclasses import dataclass, asdict
import json
from app.dependencies.database import get_redis, get_redis_message
from app.log import logger
@@ -16,7 +14,7 @@ from app.router.v2.stats import (
REDIS_ONLINE_HISTORY_KEY,
_get_online_users_count,
_get_playing_users_count,
_redis_exec
_redis_exec,
)
# Redis keys for interval statistics
@@ -29,34 +27,36 @@ CURRENT_INTERVAL_INFO_KEY = "server:current_interval_info" # 当前区间信息
@dataclass
class IntervalInfo:
"""区间信息"""
start_time: datetime
end_time: datetime
interval_key: str
def is_current(self) -> bool:
"""检查是否是当前区间"""
now = datetime.utcnow()
return self.start_time <= now < self.end_time
def to_dict(self) -> Dict:
def to_dict(self) -> dict:
return {
'start_time': self.start_time.isoformat(),
'end_time': self.end_time.isoformat(),
'interval_key': self.interval_key
"start_time": self.start_time.isoformat(),
"end_time": self.end_time.isoformat(),
"interval_key": self.interval_key,
}
@classmethod
def from_dict(cls, data: Dict) -> 'IntervalInfo':
def from_dict(cls, data: dict) -> "IntervalInfo":
return cls(
start_time=datetime.fromisoformat(data['start_time']),
end_time=datetime.fromisoformat(data['end_time']),
interval_key=data['interval_key']
start_time=datetime.fromisoformat(data["start_time"]),
end_time=datetime.fromisoformat(data["end_time"]),
interval_key=data["interval_key"],
)
@dataclass
class IntervalStats:
"""区间统计数据"""
interval_key: str
start_time: datetime
end_time: datetime
@@ -66,38 +66,38 @@ class IntervalStats:
peak_playing_count: int # 区间内游玩用户数峰值
total_samples: int # 采样次数
created_at: datetime
def to_dict(self) -> Dict:
def to_dict(self) -> dict:
return {
'interval_key': self.interval_key,
'start_time': self.start_time.isoformat(),
'end_time': self.end_time.isoformat(),
'unique_online_users': self.unique_online_users,
'unique_playing_users': self.unique_playing_users,
'peak_online_count': self.peak_online_count,
'peak_playing_count': self.peak_playing_count,
'total_samples': self.total_samples,
'created_at': self.created_at.isoformat()
"interval_key": self.interval_key,
"start_time": self.start_time.isoformat(),
"end_time": self.end_time.isoformat(),
"unique_online_users": self.unique_online_users,
"unique_playing_users": self.unique_playing_users,
"peak_online_count": self.peak_online_count,
"peak_playing_count": self.peak_playing_count,
"total_samples": self.total_samples,
"created_at": self.created_at.isoformat(),
}
@classmethod
def from_dict(cls, data: Dict) -> 'IntervalStats':
def from_dict(cls, data: dict) -> "IntervalStats":
return cls(
interval_key=data['interval_key'],
start_time=datetime.fromisoformat(data['start_time']),
end_time=datetime.fromisoformat(data['end_time']),
unique_online_users=data['unique_online_users'],
unique_playing_users=data['unique_playing_users'],
peak_online_count=data['peak_online_count'],
peak_playing_count=data['peak_playing_count'],
total_samples=data['total_samples'],
created_at=datetime.fromisoformat(data['created_at'])
interval_key=data["interval_key"],
start_time=datetime.fromisoformat(data["start_time"]),
end_time=datetime.fromisoformat(data["end_time"]),
unique_online_users=data["unique_online_users"],
unique_playing_users=data["unique_playing_users"],
peak_online_count=data["peak_online_count"],
peak_playing_count=data["peak_playing_count"],
total_samples=data["total_samples"],
created_at=datetime.fromisoformat(data["created_at"]),
)
class EnhancedIntervalStatsManager:
"""增强的区间统计管理器 - 真正统计半小时区间内的用户活跃情况"""
@staticmethod
def get_current_interval_boundaries() -> tuple[datetime, datetime]:
"""获取当前30分钟区间的边界"""
@@ -108,49 +108,53 @@ class EnhancedIntervalStatsManager:
# 区间结束时间
end_time = start_time + timedelta(minutes=30)
return start_time, end_time
@staticmethod
def generate_interval_key(start_time: datetime) -> str:
"""生成区间唯一标识"""
return f"{INTERVAL_STATS_BASE_KEY}:{start_time.strftime('%Y%m%d_%H%M')}"
@staticmethod
async def get_current_interval_info() -> IntervalInfo:
"""获取当前区间信息"""
start_time, end_time = EnhancedIntervalStatsManager.get_current_interval_boundaries()
interval_key = EnhancedIntervalStatsManager.generate_interval_key(start_time)
return IntervalInfo(
start_time=start_time,
end_time=end_time,
interval_key=interval_key
start_time, end_time = (
EnhancedIntervalStatsManager.get_current_interval_boundaries()
)
interval_key = EnhancedIntervalStatsManager.generate_interval_key(start_time)
return IntervalInfo(
start_time=start_time, end_time=end_time, interval_key=interval_key
)
@staticmethod
async def initialize_current_interval() -> None:
"""初始化当前区间"""
redis_sync = get_redis_message()
redis_async = get_redis()
try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info()
current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
# 存储当前区间信息
await _redis_exec(
redis_sync.set,
CURRENT_INTERVAL_INFO_KEY,
json.dumps(current_interval.to_dict())
redis_sync.set,
CURRENT_INTERVAL_INFO_KEY,
json.dumps(current_interval.to_dict()),
)
await redis_async.expire(CURRENT_INTERVAL_INFO_KEY, 35 * 60) # 35分钟过期
# 初始化区间用户集合（如果不存在）
online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}"
playing_key = f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
playing_key = (
f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
)
# 设置过期时间为35分钟
await redis_async.expire(online_key, 35 * 60)
await redis_async.expire(playing_key, 35 * 60)
# 初始化区间统计记录
stats = IntervalStats(
interval_key=current_interval.interval_key,
@@ -161,157 +165,193 @@ class EnhancedIntervalStatsManager:
peak_online_count=0,
peak_playing_count=0,
total_samples=0,
created_at=datetime.utcnow()
created_at=datetime.utcnow(),
)
await _redis_exec(
redis_sync.set,
current_interval.interval_key,
json.dumps(stats.to_dict())
json.dumps(stats.to_dict()),
)
await redis_async.expire(current_interval.interval_key, 35 * 60)
# 如果历史记录为空，自动填充前24小时数据为0
await EnhancedIntervalStatsManager._ensure_24h_history_exists()
logger.info(f"Initialized interval stats for {current_interval.start_time.strftime('%H:%M')} - {current_interval.end_time.strftime('%H:%M')}")
logger.info(
f"Initialized interval stats for {current_interval.start_time.strftime('%H:%M')} - {current_interval.end_time.strftime('%H:%M')}"
)
except Exception as e:
logger.error(f"Error initializing current interval: {e}")
@staticmethod
async def _ensure_24h_history_exists() -> None:
"""确保24小时历史数据存在，不存在则用0填充"""
redis_sync = get_redis_message()
redis_async = get_redis()
try:
# 检查现有历史数据数量
history_length = await _redis_exec(redis_sync.llen, REDIS_ONLINE_HISTORY_KEY)
history_length = await _redis_exec(
redis_sync.llen, REDIS_ONLINE_HISTORY_KEY
)
if history_length < 48: # 少于48个数据点（24小时*2）
logger.info(f"History has only {history_length} points, filling with zeros for 24h")
logger.info(
f"History has only {history_length} points, filling with zeros for 24h"
)
# 计算需要填充的数据点数量
needed_points = 48 - history_length
# 从当前时间往前推，创建缺失的时间点（都填充为0）
current_time = datetime.utcnow()
current_interval_start, _ = EnhancedIntervalStatsManager.get_current_interval_boundaries()
current_interval_start, _ = (
EnhancedIntervalStatsManager.get_current_interval_boundaries()
)
# 从当前区间开始往前推，创建历史数据点（确保时间对齐到30分钟边界）
fill_points = []
for i in range(needed_points):
# 每次往前推30分钟，确保时间对齐
point_time = current_interval_start - timedelta(minutes=30 * (i + 1))
point_time = current_interval_start - timedelta(
minutes=30 * (i + 1)
)
# 确保时间对齐到30分钟边界
aligned_minute = (point_time.minute // 30) * 30
point_time = point_time.replace(minute=aligned_minute, second=0, microsecond=0)
point_time = point_time.replace(
minute=aligned_minute, second=0, microsecond=0
)
history_point = {
"timestamp": point_time.isoformat(),
"online_count": 0,
"playing_count": 0,
"peak_online": 0,
"peak_playing": 0,
"total_samples": 0
"total_samples": 0,
}
fill_points.append(json.dumps(history_point))
# 将填充数据添加到历史记录末尾（最旧的数据）
if fill_points:
# 先将现有数据转移到临时位置
temp_key = f"{REDIS_ONLINE_HISTORY_KEY}_temp"
if history_length > 0:
# 复制现有数据到临时key
existing_data = await _redis_exec(redis_sync.lrange, REDIS_ONLINE_HISTORY_KEY, 0, -1)
existing_data = await _redis_exec(
redis_sync.lrange, REDIS_ONLINE_HISTORY_KEY, 0, -1
)
if existing_data:
for data in existing_data:
await _redis_exec(redis_sync.rpush, temp_key, data)
# 清空原有key
await redis_async.delete(REDIS_ONLINE_HISTORY_KEY)
# 先添加填充数据（最旧的）
for point in reversed(fill_points): # 反向添加，最旧的在最后
await _redis_exec(redis_sync.rpush, REDIS_ONLINE_HISTORY_KEY, point)
await _redis_exec(
redis_sync.rpush, REDIS_ONLINE_HISTORY_KEY, point
)
# 再添加原有数据（较新的）
if history_length > 0:
existing_data = await _redis_exec(redis_sync.lrange, temp_key, 0, -1)
existing_data = await _redis_exec(
redis_sync.lrange, temp_key, 0, -1
)
for data in existing_data:
await _redis_exec(redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, data)
await _redis_exec(
redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, data
)
# 清理临时key
await redis_async.delete(temp_key)
# 确保只保留48个数据点
await _redis_exec(redis_sync.ltrim, REDIS_ONLINE_HISTORY_KEY, 0, 47)
# 设置过期时间
await redis_async.expire(REDIS_ONLINE_HISTORY_KEY, 26 * 3600)
logger.info(f"Filled {len(fill_points)} historical data points with zeros")
logger.info(
f"Filled {len(fill_points)} historical data points with zeros"
)
except Exception as e:
logger.error(f"Error ensuring 24h history exists: {e}")
@staticmethod
async def add_user_to_interval(user_id: int, is_playing: bool = False) -> None:
"""添加用户到当前区间统计 - 实时更新当前运行的区间"""
redis_sync = get_redis_message()
redis_async = get_redis()
try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info()
current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
# 添加到区间在线用户集合
online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}"
await _redis_exec(redis_sync.sadd, online_key, str(user_id))
await redis_async.expire(online_key, 35 * 60)
# 如果用户在游玩，也添加到游玩用户集合
if is_playing:
playing_key = f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
playing_key = (
f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
)
await _redis_exec(redis_sync.sadd, playing_key, str(user_id))
await redis_async.expire(playing_key, 35 * 60)
# 立即更新区间统计（同步更新，确保数据实时性）
await EnhancedIntervalStatsManager._update_interval_stats()
logger.debug(f"Added user {user_id} to current interval {current_interval.start_time.strftime('%H:%M')}-{current_interval.end_time.strftime('%H:%M')}")
logger.debug(
f"Added user {user_id} to current interval {current_interval.start_time.strftime('%H:%M')}-{current_interval.end_time.strftime('%H:%M')}"
)
except Exception as e:
logger.error(f"Error adding user {user_id} to interval: {e}")
@staticmethod
async def _update_interval_stats() -> None:
"""更新当前区间统计 - 立即同步更新"""
redis_sync = get_redis_message()
redis_async = get_redis()
try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info()
current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
# 获取区间内独特用户数
online_key = f"{INTERVAL_ONLINE_USERS_KEY}:{current_interval.interval_key}"
playing_key = f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
playing_key = (
f"{INTERVAL_PLAYING_USERS_KEY}:{current_interval.interval_key}"
)
unique_online = await _redis_exec(redis_sync.scard, online_key)
unique_playing = await _redis_exec(redis_sync.scard, playing_key)
# 获取当前实时用户数作为峰值参考
current_online = await _get_online_users_count(redis_async)
current_playing = await _get_playing_users_count(redis_async)
# 获取现有统计数据
existing_data = await _redis_exec(redis_sync.get, current_interval.interval_key)
existing_data = await _redis_exec(
redis_sync.get, current_interval.interval_key
)
if existing_data:
stats = IntervalStats.from_dict(json.loads(existing_data))
# 更新峰值
stats.peak_online_count = max(stats.peak_online_count, current_online)
stats.peak_playing_count = max(stats.peak_playing_count, current_playing)
stats.peak_playing_count = max(
stats.peak_playing_count, current_playing
)
stats.total_samples += 1
else:
# 创建新的统计记录
@@ -324,46 +364,52 @@ class EnhancedIntervalStatsManager:
peak_online_count=current_online,
peak_playing_count=current_playing,
total_samples=1,
created_at=datetime.utcnow()
created_at=datetime.utcnow(),
)
# 更新独特用户数
stats.unique_online_users = unique_online
stats.unique_playing_users = unique_playing
# 立即保存更新的统计数据
await _redis_exec(
redis_sync.set,
current_interval.interval_key,
json.dumps(stats.to_dict())
json.dumps(stats.to_dict()),
)
await redis_async.expire(current_interval.interval_key, 35 * 60)
logger.debug(f"Updated interval stats: online={unique_online}, playing={unique_playing}, peak_online={stats.peak_online_count}, peak_playing={stats.peak_playing_count}")
logger.debug(
f"Updated interval stats: online={unique_online}, playing={unique_playing}, peak_online={stats.peak_online_count}, peak_playing={stats.peak_playing_count}"
)
except Exception as e:
logger.error(f"Error updating interval stats: {e}")
@staticmethod
async def finalize_interval() -> Optional[IntervalStats]:
async def finalize_interval() -> IntervalStats | None:
"""完成当前区间统计并保存到历史"""
redis_sync = get_redis_message()
redis_async = get_redis()
try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info()
current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
# 最后一次更新统计
await EnhancedIntervalStatsManager._update_interval_stats()
# 获取最终统计数据
stats_data = await _redis_exec(redis_sync.get, current_interval.interval_key)
stats_data = await _redis_exec(
redis_sync.get, current_interval.interval_key
)
if not stats_data:
logger.warning("No interval stats found to finalize")
return None
stats = IntervalStats.from_dict(json.loads(stats_data))
# 创建历史记录点（使用区间结束时间作为时间戳，确保时间对齐）
history_point = {
"timestamp": current_interval.end_time.isoformat(),
@@ -371,16 +417,18 @@ class EnhancedIntervalStatsManager:
"playing_count": stats.unique_playing_users,
"peak_online": stats.peak_online_count,
"peak_playing": stats.peak_playing_count,
"total_samples": stats.total_samples
"total_samples": stats.total_samples,
}
# 添加到历史记录
await _redis_exec(redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, json.dumps(history_point))
await _redis_exec(
redis_sync.lpush, REDIS_ONLINE_HISTORY_KEY, json.dumps(history_point)
)
# 只保留48个数据点（24小时，每30分钟一个点）
await _redis_exec(redis_sync.ltrim, REDIS_ONLINE_HISTORY_KEY, 0, 47)
# 设置过期时间为26小时，确保有足够缓冲
await redis_async.expire(REDIS_ONLINE_HISTORY_KEY, 26 * 3600)
logger.info(
f"Finalized interval stats: "
f"unique_online={stats.unique_online_users}, "
@@ -390,64 +438,70 @@ class EnhancedIntervalStatsManager:
f"samples={stats.total_samples} "
f"for {stats.start_time.strftime('%H:%M')}-{stats.end_time.strftime('%H:%M')}"
)
return stats
except Exception as e:
logger.error(f"Error finalizing interval stats: {e}")
return None
@staticmethod
async def get_current_interval_stats() -> Optional[IntervalStats]:
async def get_current_interval_stats() -> IntervalStats | None:
"""获取当前区间统计"""
redis_sync = get_redis_message()
try:
current_interval = await EnhancedIntervalStatsManager.get_current_interval_info()
stats_data = await _redis_exec(redis_sync.get, current_interval.interval_key)
current_interval = (
await EnhancedIntervalStatsManager.get_current_interval_info()
)
stats_data = await _redis_exec(
redis_sync.get, current_interval.interval_key
)
if stats_data:
return IntervalStats.from_dict(json.loads(stats_data))
return None
except Exception as e:
logger.error(f"Error getting current interval stats: {e}")
return None
@staticmethod
async def cleanup_old_intervals() -> None:
"""清理过期的区间数据"""
redis_async = get_redis()
try:
# 删除过期的区间统计数据（超过2小时的）
cutoff_time = datetime.utcnow() - timedelta(hours=2)
pattern = f"{INTERVAL_STATS_BASE_KEY}:*"
keys = await redis_async.keys(pattern)
for key in keys:
try:
# 从key中提取时间
time_part = key.decode().split(':')[-1] # YYYYMMDD_HHMM格式
key_time = datetime.strptime(time_part, '%Y%m%d_%H%M')
time_part = key.decode().split(":")[-1] # YYYYMMDD_HHMM格式
key_time = datetime.strptime(time_part, "%Y%m%d_%H%M")
if key_time < cutoff_time:
await redis_async.delete(key)
# 也删除对应的用户集合
await redis_async.delete(f"{INTERVAL_ONLINE_USERS_KEY}:{key}")
await redis_async.delete(f"{INTERVAL_PLAYING_USERS_KEY}:{key}")
except (ValueError, IndexError):
# 忽略解析错误的key
continue
logger.debug("Cleaned up old interval data")
except Exception as e:
logger.error(f"Error cleaning up old intervals: {e}")
# 便捷函数，用于替换现有的统计更新函数
async def update_user_activity_in_interval(user_id: int, is_playing: bool = False) -> None:
async def update_user_activity_in_interval(
user_id: int, is_playing: bool = False
) -> None:
"""用户活动时更新区间统计（在登录、开始游玩等时调用）"""
await EnhancedIntervalStatsManager.add_user_to_interval(user_id, is_playing)

154
app/service/message_queue.py
View File

@@ -3,53 +3,52 @@ Redis 消息队列服务
用于实现实时消息推送和异步数据库持久化
"""
import asyncio
import json
import uuid
from datetime import datetime
from functools import partial
from typing import Optional, Union
import concurrent.futures
from __future__ import annotations
from app.database.chat import ChatMessage, ChatChannel, MessageType, ChannelType
import asyncio
import concurrent.futures
from datetime import datetime
import uuid
from app.database.chat import ChatMessage, MessageType
from app.dependencies.database import get_redis, with_db
from app.log import logger
class MessageQueue:
"""Redis 消息队列服务"""
def __init__(self):
self.redis = get_redis()
self._processing = False
self._batch_size = 50 # 批量处理大小
self._batch_timeout = 1.0 # 批量处理超时时间（秒）
self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
async def _run_in_executor(self, func, *args):
"""在线程池中运行同步 Redis 操作"""
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self._executor, func, *args)
async def start_processing(self):
"""启动消息处理任务"""
if not self._processing:
self._processing = True
asyncio.create_task(self._process_message_queue())
logger.info("Message queue processing started")
async def stop_processing(self):
"""停止消息处理"""
self._processing = False
logger.info("Message queue processing stopped")
async def enqueue_message(self, message_data: dict) -> str:
"""
将消息加入 Redis 队列（实时响应）
Args:
message_data: 消息数据字典，包含所有必要的字段
Returns:
消息的临时 UUID
"""
@@ -58,36 +57,42 @@ class MessageQueue:
message_data["temp_uuid"] = temp_uuid
message_data["timestamp"] = datetime.now().isoformat()
message_data["status"] = "pending" # pending, processing, completed, failed
# 将消息存储到 Redis
await self._run_in_executor(
lambda: self.redis.hset(f"msg:{temp_uuid}", mapping=message_data)
)
await self._run_in_executor(self.redis.expire, f"msg:{temp_uuid}", 3600) # 1小时过期
await self._run_in_executor(
self.redis.expire, f"msg:{temp_uuid}", 3600
) # 1小时过期
# 加入处理队列
await self._run_in_executor(self.redis.lpush, "message_queue", temp_uuid)
logger.info(f"Message enqueued with temp_uuid: {temp_uuid}")
return temp_uuid
async def get_message_status(self, temp_uuid: str) -> Optional[dict]:
async def get_message_status(self, temp_uuid: str) -> dict | None:
"""获取消息状态"""
message_data = await self._run_in_executor(self.redis.hgetall, f"msg:{temp_uuid}")
message_data = await self._run_in_executor(
self.redis.hgetall, f"msg:{temp_uuid}"
)
if not message_data:
return None
return message_data
async def get_cached_messages(self, channel_id: int, limit: int = 50, since: int = 0) -> list[dict]:
async def get_cached_messages(
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[dict]:
"""
从 Redis 获取缓存的消息
Args:
channel_id: 频道 ID
limit: 限制数量
since: 获取自此消息 ID 之后的消息
Returns:
消息列表
"""
@@ -95,29 +100,39 @@ class MessageQueue:
message_uuids = await self._run_in_executor(
self.redis.lrange, f"channel:{channel_id}:messages", 0, limit - 1
)
messages = []
for uuid_str in message_uuids:
message_data = await self._run_in_executor(self.redis.hgetall, f"msg:{uuid_str}")
message_data = await self._run_in_executor(
self.redis.hgetall, f"msg:{uuid_str}"
)
if message_data:
# 检查是否满足 since 条件
if since > 0 and "message_id" in message_data:
if int(message_data["message_id"]) <= since:
continue
messages.append(message_data)
return messages[::-1] # 返回时间顺序
async def cache_channel_message(self, channel_id: int, temp_uuid: str, max_cache: int = 100):
async def cache_channel_message(
self, channel_id: int, temp_uuid: str, max_cache: int = 100
):
"""将消息 UUID 缓存到频道消息列表"""
# 添加到频道消息列表开头
await self._run_in_executor(self.redis.lpush, f"channel:{channel_id}:messages", temp_uuid)
await self._run_in_executor(
self.redis.lpush, f"channel:{channel_id}:messages", temp_uuid
)
# 限制缓存大小
await self._run_in_executor(self.redis.ltrim, f"channel:{channel_id}:messages", 0, max_cache - 1)
await self._run_in_executor(
self.redis.ltrim, f"channel:{channel_id}:messages", 0, max_cache - 1
)
# 设置过期时间（24小时）
await self._run_in_executor(self.redis.expire, f"channel:{channel_id}:messages", 86400)
await self._run_in_executor(
self.redis.expire, f"channel:{channel_id}:messages", 86400
)
async def _process_message_queue(self):
"""异步处理消息队列，批量写入数据库"""
while self._processing:
@@ -132,75 +147,90 @@ class MessageQueue:
message_uuids.append(result[1])
else:
break
if message_uuids:
await self._process_message_batch(message_uuids)
else:
# 没有消息时短暂等待
await asyncio.sleep(0.1)
except Exception as e:
logger.error(f"Error processing message queue: {e}")
await asyncio.sleep(1) # 错误时等待1秒再重试
async def _process_message_batch(self, message_uuids: list[str]):
"""批量处理消息写入数据库"""
async with with_db() as session:
messages_to_insert = []
for temp_uuid in message_uuids:
try:
# 获取消息数据
message_data = await self._run_in_executor(self.redis.hgetall, f"msg:{temp_uuid}")
message_data = await self._run_in_executor(
self.redis.hgetall, f"msg:{temp_uuid}"
)
if not message_data:
continue
# 更新状态为处理中
await self._run_in_executor(self.redis.hset, f"msg:{temp_uuid}", "status", "processing")
await self._run_in_executor(
self.redis.hset, f"msg:{temp_uuid}", "status", "processing"
)
# 创建数据库消息对象
msg = ChatMessage(
channel_id=int(message_data["channel_id"]),
content=message_data["content"],
sender_id=int(message_data["sender_id"]),
type=MessageType(message_data["type"]),
uuid=message_data.get("user_uuid") # 用户提供的 UUID（如果有）
uuid=message_data.get("user_uuid"), # 用户提供的 UUID（如果有）
)
messages_to_insert.append((msg, temp_uuid))
except Exception as e:
logger.error(f"Error preparing message {temp_uuid}: {e}")
await self._run_in_executor(self.redis.hset, f"msg:{temp_uuid}", "status", "failed")
await self._run_in_executor(
self.redis.hset, f"msg:{temp_uuid}", "status", "failed"
)
if messages_to_insert:
try:
# 批量插入数据库
for msg, temp_uuid in messages_to_insert:
session.add(msg)
await session.commit()
# 更新所有消息状态和真实 ID
for msg, temp_uuid in messages_to_insert:
await session.refresh(msg)
await self._run_in_executor(
lambda: self.redis.hset(f"msg:{temp_uuid}", mapping={
"status": "completed",
"message_id": str(msg.message_id),
"created_at": msg.timestamp.isoformat() if msg.timestamp else ""
})
lambda: self.redis.hset(
f"msg:{temp_uuid}",
mapping={
"status": "completed",
"message_id": str(msg.message_id),
"created_at": msg.timestamp.isoformat()
if msg.timestamp
else "",
},
)
)
logger.info(f"Message {temp_uuid} persisted to DB with ID {msg.message_id}")
logger.info(
f"Message {temp_uuid} persisted to DB with ID {msg.message_id}"
)
except Exception as e:
logger.error(f"Error inserting messages to database: {e}")
await session.rollback()
# 标记所有消息为失败
for _, temp_uuid in messages_to_insert:
await self._run_in_executor(self.redis.hset, f"msg:{temp_uuid}", "status", "failed")
await self._run_in_executor(
self.redis.hset, f"msg:{temp_uuid}", "status", "failed"
)
# 全局消息队列实例

204
app/service/message_queue_processor.py
View File

@@ -3,12 +3,12 @@
专门处理 Redis 消息队列的异步写入数据库
"""
from __future__ import annotations
import asyncio
import json
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from typing import Optional
import json
from app.database.chat import ChatMessage, MessageType
from app.dependencies.database import get_redis_message, with_db
@@ -17,103 +17,132 @@ from app.log import logger
class MessageQueueProcessor:
"""消息队列处理器"""
def __init__(self):
self.redis_message = get_redis_message()
self.executor = ThreadPoolExecutor(max_workers=2)
self._processing = False
self._queue_task = None
async def _redis_exec(self, func, *args, **kwargs):
"""在线程池中执行 Redis 操作"""
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self.executor, lambda: func(*args, **kwargs))
async def cache_message(self, channel_id: int, message_data: dict, temp_uuid: str):
"""将消息缓存到 Redis"""
try:
# 存储消息数据
await self._redis_exec(self.redis_message.hset, f"msg:{temp_uuid}", mapping=message_data)
await self._redis_exec(self.redis_message.expire, f"msg:{temp_uuid}", 3600) # 1小时过期
await self._redis_exec(
self.redis_message.hset, f"msg:{temp_uuid}", mapping=message_data
)
await self._redis_exec(
self.redis_message.expire, f"msg:{temp_uuid}", 3600
) # 1小时过期
# 加入频道消息列表
await self._redis_exec(self.redis_message.lpush, f"channel:{channel_id}:messages", temp_uuid)
await self._redis_exec(self.redis_message.ltrim, f"channel:{channel_id}:messages", 0, 99) # 保持最新100条
await self._redis_exec(self.redis_message.expire, f"channel:{channel_id}:messages", 86400) # 24小时过期
await self._redis_exec(
self.redis_message.lpush, f"channel:{channel_id}:messages", temp_uuid
)
await self._redis_exec(
self.redis_message.ltrim, f"channel:{channel_id}:messages", 0, 99
) # 保持最新100条
await self._redis_exec(
self.redis_message.expire, f"channel:{channel_id}:messages", 86400
) # 24小时过期
# 加入异步处理队列
await self._redis_exec(self.redis_message.lpush, "message_write_queue", temp_uuid)
await self._redis_exec(
self.redis_message.lpush, "message_write_queue", temp_uuid
)
logger.info(f"Message cached to Redis: {temp_uuid}")
except Exception as e:
logger.error(f"Failed to cache message to Redis: {e}")
async def get_cached_messages(self, channel_id: int, limit: int = 50, since: int = 0) -> list[dict]:
async def get_cached_messages(
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[dict]:
"""从 Redis 获取缓存的消息"""
try:
message_uuids = await self._redis_exec(
self.redis_message.lrange, f"channel:{channel_id}:messages", 0, limit - 1
self.redis_message.lrange,
f"channel:{channel_id}:messages",
0,
limit - 1,
)
messages = []
for temp_uuid in message_uuids:
# 解码 UUID 如果它是字节类型
if isinstance(temp_uuid, bytes):
temp_uuid = temp_uuid.decode('utf-8')
raw_data = await self._redis_exec(self.redis_message.hgetall, f"msg:{temp_uuid}")
temp_uuid = temp_uuid.decode("utf-8")
raw_data = await self._redis_exec(
self.redis_message.hgetall, f"msg:{temp_uuid}"
)
if raw_data:
# 解码 Redis 返回的字节数据
message_data = {
k.decode('utf-8') if isinstance(k, bytes) else k:
v.decode('utf-8') if isinstance(v, bytes) else v
k.decode("utf-8") if isinstance(k, bytes) else k: v.decode(
"utf-8"
)
if isinstance(v, bytes)
else v
for k, v in raw_data.items()
}
# 检查 since 条件
if since > 0 and message_data.get("message_id"):
if int(message_data["message_id"]) <= since:
continue
messages.append(message_data)
return messages[::-1] # 按时间顺序返回
except Exception as e:
logger.error(f"Failed to get cached messages: {e}")
return []
async def update_message_status(self, temp_uuid: str, status: str, message_id: Optional[int] = None):
async def update_message_status(
self, temp_uuid: str, status: str, message_id: int | None = None
):
"""更新消息状态"""
try:
update_data = {"status": status}
if message_id:
update_data["message_id"] = str(message_id)
update_data["db_timestamp"] = datetime.now().isoformat()
await self._redis_exec(self.redis_message.hset, f"msg:{temp_uuid}", mapping=update_data)
await self._redis_exec(
self.redis_message.hset, f"msg:{temp_uuid}", mapping=update_data
)
except Exception as e:
logger.error(f"Failed to update message status: {e}")
async def get_message_status(self, temp_uuid: str) -> Optional[dict]:
async def get_message_status(self, temp_uuid: str) -> dict | None:
"""获取消息状态"""
try:
raw_data = await self._redis_exec(self.redis_message.hgetall, f"msg:{temp_uuid}")
raw_data = await self._redis_exec(
self.redis_message.hgetall, f"msg:{temp_uuid}"
)
if not raw_data:
return None
# 解码 Redis 返回的字节数据
return {
k.decode('utf-8') if isinstance(k, bytes) else k:
v.decode('utf-8') if isinstance(v, bytes) else v
k.decode("utf-8") if isinstance(k, bytes) else k: v.decode("utf-8")
if isinstance(v, bytes)
else v
for k, v in raw_data.items()
}
except Exception as e:
logger.error(f"Failed to get message status: {e}")
return None
async def _process_message_queue(self):
"""处理消息队列，异步写入数据库"""
logger.info("Message queue processing started")
while self._processing:
try:
# 批量获取消息
@@ -126,47 +155,52 @@ class MessageQueueProcessor:
# result是 (queue_name, value) 的元组，需要解码
uuid_value = result[1]
if isinstance(uuid_value, bytes):
uuid_value = uuid_value.decode('utf-8')
uuid_value = uuid_value.decode("utf-8")
message_uuids.append(uuid_value)
else:
break
if not message_uuids:
await asyncio.sleep(0.5)
continue
# 批量写入数据库
await self._process_message_batch(message_uuids)
except Exception as e:
logger.error(f"Error in message queue processing: {e}")
await asyncio.sleep(1)
logger.info("Message queue processing stopped")
async def _process_message_batch(self, message_uuids: list[str]):
"""批量处理消息写入数据库"""
async with with_db() as session:
for temp_uuid in message_uuids:
try:
# 获取消息数据并解码
raw_data = await self._redis_exec(self.redis_message.hgetall, f"msg:{temp_uuid}")
raw_data = await self._redis_exec(
self.redis_message.hgetall, f"msg:{temp_uuid}"
)
if not raw_data:
continue
# 解码 Redis 返回的字节数据
message_data = {
k.decode('utf-8') if isinstance(k, bytes) else k:
v.decode('utf-8') if isinstance(v, bytes) else v
k.decode("utf-8") if isinstance(k, bytes) else k: v.decode(
"utf-8"
)
if isinstance(v, bytes)
else v
for k, v in raw_data.items()
}
if message_data.get("status") != "pending":
continue
# 更新状态为处理中
await self.update_message_status(temp_uuid, "processing")
# 创建数据库消息
msg = ChatMessage(
channel_id=int(message_data["channel_id"]),
@@ -175,15 +209,17 @@ class MessageQueueProcessor:
type=MessageType(message_data["type"]),
uuid=message_data.get("user_uuid") or None,
)
session.add(msg)
await session.commit()
await session.refresh(msg)
# 更新成功状态，包含临时消息ID映射
assert msg.message_id is not None
await self.update_message_status(temp_uuid, "completed", msg.message_id)
await self.update_message_status(
temp_uuid, "completed", msg.message_id
)
# 如果有临时消息ID，存储映射关系并通知客户端更新
if message_data.get("temp_message_id"):
temp_msg_id = int(message_data["temp_message_id"])
@@ -191,53 +227,65 @@ class MessageQueueProcessor:
self.redis_message.set,
f"temp_to_real:{temp_msg_id}",
str(msg.message_id),
ex=3600 # 1小时过期
ex=3600, # 1小时过期
)
# 发送消息ID更新通知到频道
channel_id = int(message_data["channel_id"])
await self._notify_message_update(channel_id, temp_msg_id, msg.message_id, message_data)
logger.info(f"Message {temp_uuid} persisted to DB with ID {msg.message_id}, temp_id: {message_data.get('temp_message_id')}")
await self._notify_message_update(
channel_id, temp_msg_id, msg.message_id, message_data
)
logger.info(
f"Message {temp_uuid} persisted to DB with ID {msg.message_id}, temp_id: {message_data.get('temp_message_id')}"
)
except Exception as e:
logger.error(f"Failed to process message {temp_uuid}: {e}")
await self.update_message_status(temp_uuid, "failed")
async def _notify_message_update(self, channel_id: int, temp_message_id: int, real_message_id: int, message_data: dict):
async def _notify_message_update(
self,
channel_id: int,
temp_message_id: int,
real_message_id: int,
message_data: dict,
):
"""通知客户端消息ID已更新"""
try:
# 这里我们需要通过 SignalR 发送消息更新通知
# 但为了避免循环依赖，我们将通过 Redis 发布消息更新事件
update_event = {
"event": "chat.message.update",
"event": "chat.message.update",
"data": {
"channel_id": channel_id,
"temp_message_id": temp_message_id,
"real_message_id": real_message_id,
"timestamp": message_data.get("timestamp")
}
"timestamp": message_data.get("timestamp"),
},
}
# 发布到 Redis 频道，让 SignalR 服务处理
await self._redis_exec(
self.redis_message.publish,
f"chat_updates:{channel_id}",
json.dumps(update_event)
json.dumps(update_event),
)
logger.info(f"Published message update: temp_id={temp_message_id}, real_id={real_message_id}")
logger.info(
f"Published message update: temp_id={temp_message_id}, real_id={real_message_id}"
)
except Exception as e:
logger.error(f"Failed to notify message update: {e}")
def start_processing(self):
"""启动消息队列处理"""
if not self._processing:
self._processing = True
self._queue_task = asyncio.create_task(self._process_message_queue())
logger.info("Message queue processor started")
def stop_processing(self):
"""停止消息队列处理"""
if self._processing:
@@ -246,10 +294,10 @@ class MessageQueueProcessor:
self._queue_task.cancel()
self._queue_task = None
logger.info("Message queue processor stopped")
def __del__(self):
"""清理资源"""
if hasattr(self, 'executor'):
if hasattr(self, "executor"):
self.executor.shutdown(wait=False)
@@ -272,11 +320,13 @@ async def cache_message_to_redis(channel_id: int, message_data: dict, temp_uuid:
await message_queue_processor.cache_message(channel_id, message_data, temp_uuid)
async def get_cached_messages(channel_id: int, limit: int = 50, since: int = 0) -> list[dict]:
async def get_cached_messages(
channel_id: int, limit: int = 50, since: int = 0
) -> list[dict]:
"""从 Redis 获取缓存的消息 - 便捷接口"""
return await message_queue_processor.get_cached_messages(channel_id, limit, since)
async def get_message_status(temp_uuid: str) -> Optional[dict]:
async def get_message_status(temp_uuid: str) -> dict | None:
"""获取消息状态 - 便捷接口"""
return await message_queue_processor.get_message_status(temp_uuid)

79
app/service/optimized_message.py
View File

@@ -3,23 +3,26 @@
结合 Redis 缓存和异步数据库写入实现实时消息传送
"""
from typing import Optional
from fastapi import HTTPException
from __future__ import annotations
from app.database.chat import ChatMessage, ChatChannel, MessageType, ChannelType, ChatMessageResp
from app.database.chat import (
ChannelType,
ChatMessageResp,
MessageType,
)
from app.database.lazer_user import User
from app.router.notification.server import server
from app.service.message_queue import message_queue
from app.log import logger
from app.service.message_queue import message_queue
from sqlalchemy.ext.asyncio import AsyncSession
class OptimizedMessageService:
"""优化的消息服务"""
def __init__(self):
self.message_queue = message_queue
async def send_message_fast(
self,
channel_id: int,
@@ -28,12 +31,12 @@ class OptimizedMessageService:
content: str,
sender: User,
is_action: bool = False,
user_uuid: Optional[str] = None,
session: Optional[AsyncSession] = None
user_uuid: str | None = None,
session: AsyncSession | None = None,
) -> ChatMessageResp:
"""
快速发送消息（先缓存到 Redis，异步写入数据库）
Args:
channel_id: 频道 ID
channel_type: 频道类型
@@ -43,12 +46,12 @@ class OptimizedMessageService:
is_action: 是否为动作消息
user_uuid: 用户提供的 UUID
session: 数据库会话（可选，用于一些验证）
Returns:
消息响应对象
"""
assert sender.id is not None
# 准备消息数据
message_data = {
"channel_id": str(channel_id),
@@ -57,27 +60,28 @@ class OptimizedMessageService:
"type": MessageType.ACTION.value if is_action else MessageType.PLAIN.value,
"user_uuid": user_uuid or "",
"channel_type": channel_type.value,
"channel_name": channel_name
"channel_name": channel_name,
}
# 立即将消息加入 Redis 队列（实时响应）
temp_uuid = await self.message_queue.enqueue_message(message_data)
# 缓存到频道消息列表
await self.message_queue.cache_channel_message(channel_id, temp_uuid)
# 创建临时响应对象（简化版本，用于立即响应）
from datetime import datetime
from app.database.lazer_user import UserResp
# 创建基本的用户响应对象
user_resp = UserResp(
id=sender.id,
username=sender.username,
country_code=getattr(sender, 'country_code', 'XX'),
country_code=getattr(sender, "country_code", "XX"),
# 基本字段，其他复杂字段可以后续异步加载
)
temp_response = ChatMessageResp(
message_id=0, # 临时 ID，等数据库写入后会更新
channel_id=channel_id,
@@ -86,63 +90,62 @@ class OptimizedMessageService:
sender_id=sender.id,
sender=user_resp,
is_action=is_action,
uuid=user_uuid
uuid=user_uuid,
)
temp_response.temp_uuid = temp_uuid # 添加临时 UUID 用于后续更新
logger.info(f"Message sent to channel {channel_id} with temp_uuid {temp_uuid}")
return temp_response
async def get_cached_messages(
self,
channel_id: int,
limit: int = 50,
since: int = 0
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[dict]:
"""
获取缓存的消息
Args:
channel_id: 频道 ID
limit: 限制数量
since: 获取自此消息 ID 之后的消息
Returns:
消息列表
"""
return await self.message_queue.get_cached_messages(channel_id, limit, since)
async def get_message_status(self, temp_uuid: str) -> Optional[dict]:
async def get_message_status(self, temp_uuid: str) -> dict | None:
"""
获取消息状态
Args:
temp_uuid: 临时消息 UUID
Returns:
消息状态信息
"""
return await self.message_queue.get_message_status(temp_uuid)
async def wait_for_message_persisted(self, temp_uuid: str, timeout: int = 30) -> Optional[dict]:
async def wait_for_message_persisted(
self, temp_uuid: str, timeout: int = 30
) -> dict | None:
"""
等待消息持久化到数据库
Args:
temp_uuid: 临时消息 UUID
timeout: 超时时间（秒）
Returns:
完成后的消息状态
"""
import asyncio
for _ in range(timeout * 10): # 每100ms检查一次
status = await self.get_message_status(temp_uuid)
if status and status.get("status") in ["completed", "failed"]:
return status
await asyncio.sleep(0.1)
return None

411
app/service/redis_message_system.py
View File

@@ -5,59 +5,66 @@
- 支持消息状态同步和故障恢复
"""
from __future__ import annotations
import asyncio
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
import json
import time
import uuid
from datetime import datetime
from typing import Optional, List, Dict, Any
from concurrent.futures import ThreadPoolExecutor
from typing import Any
from app.database.chat import ChatMessage, MessageType, ChatMessageResp
from app.database.lazer_user import User, UserResp, RANKING_INCLUDES
from app.database.chat import ChatMessage, ChatMessageResp, MessageType
from app.database.lazer_user import RANKING_INCLUDES, User, UserResp
from app.dependencies.database import get_redis_message, with_db
from app.log import logger
class RedisMessageSystem:
"""Redis 消息系统"""
def __init__(self):
self.redis = get_redis_message()
self.executor = ThreadPoolExecutor(max_workers=2)
self._batch_timer: Optional[asyncio.Task] = None
self._batch_timer: asyncio.Task | None = None
self._running = False
self.batch_interval = 5.0 # 5秒批量存储一次
self.max_batch_size = 100 # 每批最多处理100条消息
async def _redis_exec(self, func, *args, **kwargs):
"""在线程池中执行 Redis 操作"""
loop = asyncio.get_event_loop()
return await loop.run_in_executor(self.executor, lambda: func(*args, **kwargs))
async def send_message(self, channel_id: int, user: User, content: str,
is_action: bool = False, user_uuid: Optional[str] = None) -> ChatMessageResp:
async def send_message(
self,
channel_id: int,
user: User,
content: str,
is_action: bool = False,
user_uuid: str | None = None,
) -> ChatMessageResp:
"""
发送消息 - 立即存储到 Redis 并返回
Args:
channel_id: 频道ID
user: 发送用户
content: 消息内容
is_action: 是否为动作消息
user_uuid: 用户UUID
Returns:
ChatMessageResp: 消息响应对象
"""
# 生成消息ID和时间戳
message_id = await self._generate_message_id(channel_id)
timestamp = datetime.now()
# 确保用户ID存在
if not user.id:
raise ValueError("User ID is required")
# 准备消息数据
message_data = {
"message_id": message_id,
@@ -68,19 +75,20 @@ class RedisMessageSystem:
"type": MessageType.ACTION.value if is_action else MessageType.PLAIN.value,
"uuid": user_uuid or "",
"status": "cached", # Redis 缓存状态
"created_at": time.time()
"created_at": time.time(),
}
# 立即存储到 Redis
await self._store_to_redis(message_id, channel_id, message_data)
# 创建响应对象
async with with_db() as session:
user_resp = await UserResp.from_db(user, session, RANKING_INCLUDES)
# 确保 statistics 不为空
if user_resp.statistics is None:
from app.database.statistics import UserStatisticsResp
user_resp.statistics = UserStatisticsResp(
mode=user.playmode,
global_rank=0,
@@ -96,9 +104,9 @@ class RedisMessageSystem:
replays_watched_by_others=0,
is_ranked=False,
grade_counts={"ssh": 0, "ss": 0, "sh": 0, "s": 0, "a": 0},
level={"current": 1, "progress": 0}
level={"current": 1, "progress": 0},
)
response = ChatMessageResp(
message_id=message_id,
channel_id=channel_id,
@@ -107,51 +115,71 @@ class RedisMessageSystem:
sender_id=user.id,
sender=user_resp,
is_action=is_action,
uuid=user_uuid
uuid=user_uuid,
)
logger.info(
f"Message {message_id} sent to Redis cache for channel {channel_id}"
)
logger.info(f"Message {message_id} sent to Redis cache for channel {channel_id}")
return response
async def get_messages(self, channel_id: int, limit: int = 50, since: int = 0) -> List[ChatMessageResp]:
async def get_messages(
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[ChatMessageResp]:
"""
获取频道消息 - 优先从 Redis 获取最新消息
Args:
channel_id: 频道ID
limit: 消息数量限制
since: 起始消息ID
Returns:
List[ChatMessageResp]: 消息列表
"""
messages = []
try:
# 从 Redis 获取最新消息
redis_messages = await self._get_from_redis(channel_id, limit, since)
# 为每条消息构建响应对象
async with with_db() as session:
for msg_data in redis_messages:
# 获取发送者信息
sender = await session.get(User, msg_data["sender_id"])
if sender:
user_resp = await UserResp.from_db(sender, session, RANKING_INCLUDES)
user_resp = await UserResp.from_db(
sender, session, RANKING_INCLUDES
)
if user_resp.statistics is None:
from app.database.statistics import UserStatisticsResp
user_resp.statistics = UserStatisticsResp(
mode=sender.playmode,
global_rank=0, country_rank=0, pp=0.0,
ranked_score=0, hit_accuracy=0.0, play_count=0,
play_time=0, total_score=0, total_hits=0,
maximum_combo=0, replays_watched_by_others=0,
global_rank=0,
country_rank=0,
pp=0.0,
ranked_score=0,
hit_accuracy=0.0,
play_count=0,
play_time=0,
total_score=0,
total_hits=0,
maximum_combo=0,
replays_watched_by_others=0,
is_ranked=False,
grade_counts={"ssh": 0, "ss": 0, "sh": 0, "s": 0, "a": 0},
level={"current": 1, "progress": 0}
grade_counts={
"ssh": 0,
"ss": 0,
"sh": 0,
"s": 0,
"a": 0,
},
level={"current": 1, "progress": 0},
)
message_resp = ChatMessageResp(
message_id=msg_data["message_id"],
channel_id=msg_data["channel_id"],
@@ -160,77 +188,97 @@ class RedisMessageSystem:
sender_id=msg_data["sender_id"],
sender=user_resp,
is_action=msg_data["type"] == MessageType.ACTION.value,
uuid=msg_data.get("uuid") or None
uuid=msg_data.get("uuid") or None,
)
messages.append(message_resp)
# 如果 Redis 消息不够，从数据库补充
if len(messages) < limit and since == 0:
await self._backfill_from_database(channel_id, messages, limit)
except Exception as e:
logger.error(f"Failed to get messages from Redis: {e}")
# 回退到数据库查询
messages = await self._get_from_database_only(channel_id, limit, since)
return messages[:limit]
async def _generate_message_id(self, channel_id: int) -> int:
"""生成唯一的消息ID - 确保全局唯一且严格递增"""
# 使用全局计数器确保所有频道的消息ID都是严格递增的
message_id = await self._redis_exec(self.redis.incr, "global_message_id_counter")
message_id = await self._redis_exec(
self.redis.incr, "global_message_id_counter"
)
# 同时更新频道的最后消息ID，用于客户端状态同步
await self._redis_exec(self.redis.set, f"channel:{channel_id}:last_msg_id", message_id)
await self._redis_exec(
self.redis.set, f"channel:{channel_id}:last_msg_id", message_id
)
return message_id
async def _store_to_redis(self, message_id: int, channel_id: int, message_data: Dict[str, Any]):
async def _store_to_redis(
self, message_id: int, channel_id: int, message_data: dict[str, Any]
):
"""存储消息到 Redis"""
try:
# 存储消息数据
await self._redis_exec(
self.redis.hset,
f"msg:{channel_id}:{message_id}",
mapping={k: json.dumps(v) if isinstance(v, (dict, list)) else str(v)
for k, v in message_data.items()}
self.redis.hset,
f"msg:{channel_id}:{message_id}",
mapping={
k: json.dumps(v) if isinstance(v, (dict, list)) else str(v)
for k, v in message_data.items()
},
)
# 设置消息过期时间（7天）
await self._redis_exec(self.redis.expire, f"msg:{channel_id}:{message_id}", 604800)
await self._redis_exec(
self.redis.expire, f"msg:{channel_id}:{message_id}", 604800
)
# 清理可能存在的错误类型键，然后添加到频道消息列表（按时间排序）
channel_messages_key = f"channel:{channel_id}:messages"
# 检查键的类型，如果不是 zset 类型则删除
try:
key_type = await self._redis_exec(self.redis.type, channel_messages_key)
if key_type and key_type != "zset":
logger.warning(f"Deleting Redis key {channel_messages_key} with wrong type: {key_type}")
logger.warning(
f"Deleting Redis key {channel_messages_key} with wrong type: {key_type}"
)
await self._redis_exec(self.redis.delete, channel_messages_key)
except Exception as type_check_error:
logger.warning(f"Failed to check key type for {channel_messages_key}: {type_check_error}")
logger.warning(
f"Failed to check key type for {channel_messages_key}: {type_check_error}"
)
# 如果检查失败，直接删除键以确保清理
await self._redis_exec(self.redis.delete, channel_messages_key)
# 添加到频道消息列表（sorted set）
await self._redis_exec(
self.redis.zadd,
channel_messages_key,
{f"msg:{channel_id}:{message_id}": message_id}
self.redis.zadd,
channel_messages_key,
{f"msg:{channel_id}:{message_id}": message_id},
)
# 保持频道消息列表大小（最多1000条）
await self._redis_exec(self.redis.zremrangebyrank, channel_messages_key, 0, -1001)
await self._redis_exec(
self.redis.zremrangebyrank, channel_messages_key, 0, -1001
)
# 添加到待持久化队列
await self._redis_exec(self.redis.lpush, "pending_messages", f"{channel_id}:{message_id}")
await self._redis_exec(
self.redis.lpush, "pending_messages", f"{channel_id}:{message_id}"
)
except Exception as e:
logger.error(f"Failed to store message to Redis: {e}")
raise
async def _get_from_redis(self, channel_id: int, limit: int = 50, since: int = 0) -> List[Dict[str, Any]]:
async def _get_from_redis(
self, channel_id: int, limit: int = 50, since: int = 0
) -> list[dict[str, Any]]:
"""从 Redis 获取消息"""
try:
# 获取消息键列表，按消息ID排序
@@ -239,22 +287,22 @@ class RedisMessageSystem:
message_keys = await self._redis_exec(
self.redis.zrangebyscore,
f"channel:{channel_id}:messages",
since + 1, "+inf",
start=0, num=limit
since + 1,
"+inf",
start=0,
num=limit,
)
else:
# 获取最新的消息（倒序获取，然后反转）
message_keys = await self._redis_exec(
self.redis.zrevrange,
f"channel:{channel_id}:messages",
0, limit - 1
self.redis.zrevrange, f"channel:{channel_id}:messages", 0, limit - 1
)
messages = []
for key in message_keys:
if isinstance(key, bytes):
key = key.decode('utf-8')
key = key.decode("utf-8")
# 获取消息数据
raw_data = await self._redis_exec(self.redis.hgetall, key)
if raw_data:
@@ -262,106 +310,118 @@ class RedisMessageSystem:
message_data = {}
for k, v in raw_data.items():
if isinstance(k, bytes):
k = k.decode('utf-8')
k = k.decode("utf-8")
if isinstance(v, bytes):
v = v.decode('utf-8')
v = v.decode("utf-8")
# 尝试解析 JSON
try:
if k in ['grade_counts', 'level'] or v.startswith(('{', '[')):
if k in ["grade_counts", "level"] or v.startswith(
("{", "[")
):
message_data[k] = json.loads(v)
elif k in ['message_id', 'channel_id', 'sender_id']:
elif k in ["message_id", "channel_id", "sender_id"]:
message_data[k] = int(v)
elif k == 'created_at':
elif k == "created_at":
message_data[k] = float(v)
else:
message_data[k] = v
except (json.JSONDecodeError, ValueError):
message_data[k] = v
messages.append(message_data)
# 确保消息按ID正序排序（时间顺序）
messages.sort(key=lambda x: x.get('message_id', 0))
messages.sort(key=lambda x: x.get("message_id", 0))
# 如果是获取最新消息（since=0），需要保持倒序（最新的在前面）
if since == 0:
messages.reverse()
return messages
except Exception as e:
logger.error(f"Failed to get messages from Redis: {e}")
return []
async def _backfill_from_database(self, channel_id: int, existing_messages: List[ChatMessageResp], limit: int):
async def _backfill_from_database(
self, channel_id: int, existing_messages: list[ChatMessageResp], limit: int
):
"""从数据库补充历史消息"""
try:
# 找到最小的消息ID
min_id = float('inf')
min_id = float("inf")
if existing_messages:
for msg in existing_messages:
if msg.message_id is not None and msg.message_id < min_id:
min_id = msg.message_id
needed = limit - len(existing_messages)
if needed <= 0:
return
async with with_db() as session:
from sqlmodel import select, col
query = select(ChatMessage).where(
ChatMessage.channel_id == channel_id
)
if min_id != float('inf'):
from sqlmodel import col, select
query = select(ChatMessage).where(ChatMessage.channel_id == channel_id)
if min_id != float("inf"):
query = query.where(col(ChatMessage.message_id) < min_id)
query = query.order_by(col(ChatMessage.message_id).desc()).limit(needed)
db_messages = (await session.exec(query)).all()
for msg in reversed(db_messages): # 按时间正序插入
msg_resp = await ChatMessageResp.from_db(msg, session)
existing_messages.insert(0, msg_resp)
except Exception as e:
logger.error(f"Failed to backfill from database: {e}")
async def _get_from_database_only(self, channel_id: int, limit: int, since: int) -> List[ChatMessageResp]:
async def _get_from_database_only(
self, channel_id: int, limit: int, since: int
) -> list[ChatMessageResp]:
"""仅从数据库获取消息（回退方案）"""
try:
async with with_db() as session:
from sqlmodel import select, col
from sqlmodel import col, select
query = select(ChatMessage).where(ChatMessage.channel_id == channel_id)
if since > 0:
# 获取指定ID之后的消息，按ID正序
query = query.where(col(ChatMessage.message_id) > since)
query = query.order_by(col(ChatMessage.message_id).asc()).limit(limit)
query = query.order_by(col(ChatMessage.message_id).asc()).limit(
limit
)
else:
# 获取最新消息，按ID倒序（最新的在前面）
query = query.order_by(col(ChatMessage.message_id).desc()).limit(limit)
query = query.order_by(col(ChatMessage.message_id).desc()).limit(
limit
)
messages = (await session.exec(query)).all()
results = [await ChatMessageResp.from_db(msg, session) for msg in messages]
results = [
await ChatMessageResp.from_db(msg, session) for msg in messages
]
# 如果是 since > 0，保持正序；否则反转为时间正序
if since == 0:
results.reverse()
return results
except Exception as e:
logger.error(f"Failed to get messages from database: {e}")
return []
async def _batch_persist_to_database(self):
"""批量持久化消息到数据库"""
logger.info("Starting batch persistence to database")
while self._running:
try:
# 获取待处理的消息
@@ -374,52 +434,52 @@ class RedisMessageSystem:
# key 是 (queue_name, value) 的元组
value = key[1]
if isinstance(value, bytes):
value = value.decode('utf-8')
value = value.decode("utf-8")
message_keys.append(value)
else:
break
if message_keys:
await self._process_message_batch(message_keys)
else:
await asyncio.sleep(self.batch_interval)
except Exception as e:
logger.error(f"Error in batch persistence: {e}")
await asyncio.sleep(1)
logger.info("Stopped batch persistence to database")
async def _process_message_batch(self, message_keys: List[str]):
async def _process_message_batch(self, message_keys: list[str]):
"""处理消息批次"""
async with with_db() as session:
for key in message_keys:
try:
# 解析频道ID和消息ID
channel_id, message_id = map(int, key.split(':'))
channel_id, message_id = map(int, key.split(":"))
# 从 Redis 获取消息数据
raw_data = await self._redis_exec(
self.redis.hgetall, f"msg:{channel_id}:{message_id}"
)
if not raw_data:
continue
# 解码数据
message_data = {}
for k, v in raw_data.items():
if isinstance(k, bytes):
k = k.decode('utf-8')
k = k.decode("utf-8")
if isinstance(v, bytes):
v = v.decode('utf-8')
v = v.decode("utf-8")
message_data[k] = v
# 检查消息是否已存在于数据库
existing = await session.get(ChatMessage, int(message_id))
if existing:
continue
# 创建数据库消息 - 使用 Redis 生成的正数ID
db_message = ChatMessage(
message_id=int(message_id), # 使用 Redis 系统生成的正数ID
@@ -428,31 +488,34 @@ class RedisMessageSystem:
content=message_data["content"],
timestamp=datetime.fromisoformat(message_data["timestamp"]),
type=MessageType(message_data["type"]),
uuid=message_data.get("uuid") or None
uuid=message_data.get("uuid") or None,
)
session.add(db_message)
# 更新 Redis 中的状态
await self._redis_exec(
self.redis.hset,
f"msg:{channel_id}:{message_id}",
"status", "persisted"
"status",
"persisted",
)
logger.debug(f"Message {message_id} persisted to database")
except Exception as e:
logger.error(f"Failed to process message {key}: {e}")
# 提交批次
try:
await session.commit()
logger.info(f"Batch of {len(message_keys)} messages committed to database")
logger.info(
f"Batch of {len(message_keys)} messages committed to database"
)
except Exception as e:
logger.error(f"Failed to commit message batch: {e}")
await session.rollback()
def start(self):
"""启动系统"""
if not self._running:
@@ -461,63 +524,71 @@ class RedisMessageSystem:
# 启动时初始化消息ID计数器
asyncio.create_task(self._initialize_message_counter())
logger.info("Redis message system started")
async def _initialize_message_counter(self):
"""初始化全局消息ID计数器，确保从数据库最大ID开始"""
try:
# 清理可能存在的问题键
await self._cleanup_redis_keys()
async with with_db() as session:
from sqlmodel import select, func
from sqlmodel import func, select
# 获取数据库中最大的消息ID
result = await session.exec(
select(func.max(ChatMessage.message_id))
)
result = await session.exec(select(func.max(ChatMessage.message_id)))
max_id = result.one() or 0
# 检查 Redis 中的计数器值
current_counter = await self._redis_exec(self.redis.get, "global_message_id_counter")
current_counter = await self._redis_exec(
self.redis.get, "global_message_id_counter"
)
current_counter = int(current_counter) if current_counter else 0
# 设置计数器为两者中的最大值
initial_counter = max(max_id, current_counter)
await self._redis_exec(self.redis.set, "global_message_id_counter", initial_counter)
logger.info(f"Initialized global message ID counter to {initial_counter}")
await self._redis_exec(
self.redis.set, "global_message_id_counter", initial_counter
)
logger.info(
f"Initialized global message ID counter to {initial_counter}"
)
except Exception as e:
logger.error(f"Failed to initialize message counter: {e}")
# 如果初始化失败，设置一个安全的起始值
await self._redis_exec(self.redis.setnx, "global_message_id_counter", 1000000)
await self._redis_exec(
self.redis.setnx, "global_message_id_counter", 1000000
)
async def _cleanup_redis_keys(self):
"""清理可能存在问题的 Redis 键"""
try:
# 扫描所有 channel:*:messages 键并检查类型
keys_pattern = "channel:*:messages"
keys = await self._redis_exec(self.redis.keys, keys_pattern)
for key in keys:
if isinstance(key, bytes):
key = key.decode('utf-8')
key = key.decode("utf-8")
try:
key_type = await self._redis_exec(self.redis.type, key)
if key_type and key_type != "zset":
logger.warning(f"Cleaning up Redis key {key} with wrong type: {key_type}")
logger.warning(
f"Cleaning up Redis key {key} with wrong type: {key_type}"
)
await self._redis_exec(self.redis.delete, key)
except Exception as cleanup_error:
logger.warning(f"Failed to cleanup key {key}: {cleanup_error}")
# 强制删除问题键
await self._redis_exec(self.redis.delete, key)
logger.info("Redis keys cleanup completed")
except Exception as e:
logger.error(f"Failed to cleanup Redis keys: {e}")
def stop(self):
"""停止系统"""
if self._running:
@@ -526,10 +597,10 @@ class RedisMessageSystem:
self._batch_timer.cancel()
self._batch_timer = None
logger.info("Redis message system stopped")
def __del__(self):
"""清理资源"""
if hasattr(self, 'executor'):
if hasattr(self, "executor"):
self.executor.shutdown(wait=False)

53
app/service/stats_cleanup.py
View File

@@ -1,81 +1,94 @@
from __future__ import annotations
from datetime import datetime, timedelta
from app.dependencies.database import get_redis, get_redis_message
from app.log import logger
from app.router.v2.stats import REDIS_ONLINE_USERS_KEY, REDIS_PLAYING_USERS_KEY, _redis_exec
from app.router.v2.stats import (
REDIS_ONLINE_USERS_KEY,
REDIS_PLAYING_USERS_KEY,
_redis_exec,
)
async def cleanup_stale_online_users() -> tuple[int, int]:
"""清理过期的在线和游玩用户，返回清理的用户数"""
redis_sync = get_redis_message()
redis_async = get_redis()
online_cleaned = 0
playing_cleaned = 0
try:
# 获取所有在线用户
online_users = await _redis_exec(redis_sync.smembers, REDIS_ONLINE_USERS_KEY)
playing_users = await _redis_exec(redis_sync.smembers, REDIS_PLAYING_USERS_KEY)
# 检查在线用户的最后活动时间
current_time = datetime.utcnow()
stale_threshold = current_time - timedelta(hours=2) # 2小时无活动视为过期
# 对于在线用户，我们检查metadata在线标记
stale_online_users = []
for user_id in online_users:
user_id_str = user_id.decode() if isinstance(user_id, bytes) else str(user_id)
user_id_str = (
user_id.decode() if isinstance(user_id, bytes) else str(user_id)
)
metadata_key = f"metadata:online:{user_id_str}"
# 如果metadata标记不存在，说明用户已经离线
if not await redis_async.exists(metadata_key):
stale_online_users.append(user_id_str)
# 清理过期的在线用户
if stale_online_users:
await _redis_exec(redis_sync.srem, REDIS_ONLINE_USERS_KEY, *stale_online_users)
await _redis_exec(
redis_sync.srem, REDIS_ONLINE_USERS_KEY, *stale_online_users
)
online_cleaned = len(stale_online_users)
logger.info(f"Cleaned {online_cleaned} stale online users")
# 对于游玩用户，我们也检查对应的spectator状态
stale_playing_users = []
for user_id in playing_users:
user_id_str = user_id.decode() if isinstance(user_id, bytes) else str(user_id)
user_id_str = (
user_id.decode() if isinstance(user_id, bytes) else str(user_id)
)
# 如果用户不在在线用户列表中，说明已经离线，也应该从游玩列表中移除
if user_id_str in stale_online_users or user_id_str not in [
u.decode() if isinstance(u, bytes) else str(u) for u in online_users
]:
stale_playing_users.append(user_id_str)
# 清理过期的游玩用户
if stale_playing_users:
await _redis_exec(redis_sync.srem, REDIS_PLAYING_USERS_KEY, *stale_playing_users)
await _redis_exec(
redis_sync.srem, REDIS_PLAYING_USERS_KEY, *stale_playing_users
)
playing_cleaned = len(stale_playing_users)
logger.info(f"Cleaned {playing_cleaned} stale playing users")
except Exception as e:
logger.error(f"Error cleaning stale users: {e}")
return online_cleaned, playing_cleaned
async def refresh_redis_key_expiry() -> None:
"""刷新Redis键的过期时间，防止数据丢失"""
redis_async = get_redis()
try:
# 刷新在线用户key的过期时间
if await redis_async.exists(REDIS_ONLINE_USERS_KEY):
await redis_async.expire(REDIS_ONLINE_USERS_KEY, 6 * 3600) # 6小时
# 刷新游玩用户key的过期时间
if await redis_async.exists(REDIS_PLAYING_USERS_KEY):
await redis_async.expire(REDIS_PLAYING_USERS_KEY, 6 * 3600) # 6小时
logger.debug("Refreshed Redis key expiry times")
except Exception as e:
logger.error(f"Error refreshing Redis key expiry: {e}")

87
app/service/stats_scheduler.py
View File

@@ -5,46 +5,49 @@ from datetime import datetime, timedelta
from app.log import logger
from app.router.v2.stats import record_hourly_stats, update_registered_users_count
from app.service.stats_cleanup import cleanup_stale_online_users, refresh_redis_key_expiry
from app.service.enhanced_interval_stats import EnhancedIntervalStatsManager
from app.service.stats_cleanup import (
cleanup_stale_online_users,
refresh_redis_key_expiry,
)
class StatsScheduler:
"""统计数据调度器"""
def __init__(self):
self._running = False
self._stats_task: asyncio.Task | None = None
self._registered_task: asyncio.Task | None = None
self._cleanup_task: asyncio.Task | None = None
def start(self) -> None:
"""启动调度器"""
if self._running:
return
self._running = True
self._stats_task = asyncio.create_task(self._stats_loop())
self._registered_task = asyncio.create_task(self._registered_users_loop())
self._cleanup_task = asyncio.create_task(self._cleanup_loop())
logger.info("Stats scheduler started")
def stop(self) -> None:
"""停止调度器"""
if not self._running:
return
self._running = False
if self._stats_task:
self._stats_task.cancel()
if self._registered_task:
self._registered_task.cancel()
if self._cleanup_task:
self._cleanup_task.cancel()
logger.info("Stats scheduler stopped")
async def _stats_loop(self) -> None:
"""统计数据记录循环 - 每30分钟记录一次"""
# 启动时立即记录一次统计数据
@@ -53,49 +56,57 @@ class StatsScheduler:
logger.info("Initial enhanced interval statistics initialized on startup")
except Exception as e:
logger.error(f"Error initializing enhanced interval stats: {e}")
while self._running:
try:
# 计算下次记录时间（下个30分钟整点）
now = datetime.utcnow()
# 计算当前区间边界
current_minute = (now.minute // 30) * 30
current_interval_end = now.replace(minute=current_minute, second=0, microsecond=0) + timedelta(minutes=30)
current_interval_end = now.replace(
minute=current_minute, second=0, microsecond=0
) + timedelta(minutes=30)
# 如果已经过了当前区间结束时间，立即处理
if now >= current_interval_end:
current_interval_end += timedelta(minutes=30)
# 计算需要等待的时间（到下个区间结束）
sleep_seconds = (current_interval_end - now).total_seconds()
# 确保至少等待1分钟，最多等待31分钟
sleep_seconds = max(min(sleep_seconds, 31 * 60), 60)
logger.debug(f"Next interval finalization in {sleep_seconds/60:.1f} minutes at {current_interval_end.strftime('%H:%M:%S')}")
logger.debug(
f"Next interval finalization in {sleep_seconds / 60:.1f} minutes at {current_interval_end.strftime('%H:%M:%S')}"
)
await asyncio.sleep(sleep_seconds)
if not self._running:
break
# 完成当前区间并记录到历史
finalized_stats = await EnhancedIntervalStatsManager.finalize_interval()
if finalized_stats:
logger.info(f"Finalized enhanced interval statistics at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}")
logger.info(
f"Finalized enhanced interval statistics at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}"
)
else:
# 如果区间完成失败，使用原有方式记录
await record_hourly_stats()
logger.info(f"Recorded hourly statistics (fallback) at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}")
logger.info(
f"Recorded hourly statistics (fallback) at {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}"
)
# 开始新的区间统计
await EnhancedIntervalStatsManager.initialize_current_interval()
except Exception as e:
logger.error(f"Error in stats loop: {e}")
# 出错时等待5分钟再重试
await asyncio.sleep(5 * 60)
async def _registered_users_loop(self) -> None:
"""注册用户数更新循环 - 每5分钟更新一次"""
# 启动时立即更新一次注册用户数
@@ -104,14 +115,14 @@ class StatsScheduler:
logger.info("Initial registered users count updated on startup")
except Exception as e:
logger.error(f"Error updating initial registered users count: {e}")
while self._running:
# 等待5分钟
await asyncio.sleep(5 * 60)
if not self._running:
break
try:
await update_registered_users_count()
logger.debug("Updated registered users count")
@@ -124,31 +135,35 @@ class StatsScheduler:
try:
online_cleaned, playing_cleaned = await cleanup_stale_online_users()
if online_cleaned > 0 or playing_cleaned > 0:
logger.info(f"Initial cleanup: removed {online_cleaned} stale online users, {playing_cleaned} stale playing users")
logger.info(
f"Initial cleanup: removed {online_cleaned} stale online users, {playing_cleaned} stale playing users"
)
await refresh_redis_key_expiry()
except Exception as e:
logger.error(f"Error in initial cleanup: {e}")
while self._running:
# 等待10分钟
await asyncio.sleep(10 * 60)
if not self._running:
break
try:
# 清理过期用户
online_cleaned, playing_cleaned = await cleanup_stale_online_users()
if online_cleaned > 0 or playing_cleaned > 0:
logger.info(f"Cleanup: removed {online_cleaned} stale online users, {playing_cleaned} stale playing users")
logger.info(
f"Cleanup: removed {online_cleaned} stale online users, {playing_cleaned} stale playing users"
)
# 刷新Redis key过期时间
await refresh_redis_key_expiry()
# 清理过期的区间数据
await EnhancedIntervalStatsManager.cleanup_old_intervals()
except Exception as e:
logger.error(f"Error in cleanup loop: {e}")
# 出错时等待2分钟再重试