Bennett/g0v0-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(task): move schedulers and startup/shutdown task into tasks directory

Browse Source
This commit is contained in:
MingxuanGame
2025-10-03 10:15:22 +00:00
parent afd5018bcd
commit fce88272b5
26 changed files with 464 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/service/__init__.py
View File

@@ -1,14 +1,8 @@
from __future__ import annotations
from .daily_challenge import create_daily_challenge_room
from .recalculate_banned_beatmap import recalculate_banned_beatmap
from .recalculate_failed_score import recalculate_failed_score
from .room import create_playlist_room, create_playlist_room_from_api
__all__ = [
"create_daily_challenge_room",
"create_playlist_room",
"create_playlist_room_from_api",
"recalculate_banned_beatmap",
"recalculate_failed_score",
]

15
app/service/beatmapset_update_service.py
View File

@@ -1,6 +1,6 @@
from __future__ import annotations
from datetime import datetime, timedelta
from datetime import timedelta
from enum import Enum
import math
import random
@@ -12,7 +12,6 @@ from app.database.beatmap_sync import BeatmapSync, SavedBeatmapMeta
from app.database.beatmapset import Beatmapset, BeatmapsetResp
from app.database.score import Score
from app.dependencies.database import with_db
from app.dependencies.scheduler import get_scheduler
from app.dependencies.storage import get_storage_service
from app.log import logger
from app.models.beatmap import BeatmapRankStatus
@@ -347,15 +346,3 @@ def init_beatmapset_update_service(fetcher: "Fetcher") -> BeatmapsetUpdateServic
def get_beatmapset_update_service() -> BeatmapsetUpdateService:
assert service is not None, "BeatmapsetUpdateService is not initialized"
return service
@get_scheduler().scheduled_job(
"interval",
id="update_beatmaps",
minutes=SCHEDULER_INTERVAL_MINUTES,
next_run_time=datetime.now() + timedelta(minutes=1),
)
async def beatmapset_update_job():
if service is not None:
bg_tasks.add_task(service.add_missing_beatmapsets)
await service._update_beatmaps()

85
app/service/calculate_all_user_rank.py
View File

@@ -1,85 +0,0 @@
from __future__ import annotations
from datetime import timedelta
from app.database import RankHistory, UserStatistics
from app.database.rank_history import RankTop
from app.dependencies.database import with_db
from app.dependencies.scheduler import get_scheduler
from app.models.score import GameMode
from app.utils import utcnow
from sqlmodel import col, exists, select, update
@get_scheduler().scheduled_job("cron", hour=0, minute=0, second=0, id="calculate_user_rank")
async def calculate_user_rank(is_today: bool = False):
today = utcnow().date()
target_date = today if is_today else today - timedelta(days=1)
async with with_db() as session:
for gamemode in GameMode:
users = await session.exec(
select(UserStatistics)
.where(
UserStatistics.mode == gamemode,
UserStatistics.pp > 0,
col(UserStatistics.is_ranked).is_(True),
)
.order_by(
col(UserStatistics.pp).desc(),
col(UserStatistics.total_score).desc(),
)
)
rank = 1
for user in users:
is_exist = (
await session.exec(
select(exists()).where(
RankHistory.user_id == user.user_id,
RankHistory.mode == gamemode,
RankHistory.date == target_date,
)
)
).first()
if not is_exist:
rank_history = RankHistory(
user_id=user.user_id,
mode=gamemode,
rank=rank,
date=today,
)
session.add(rank_history)
else:
await session.execute(
update(RankHistory)
.where(
col(RankHistory.user_id) == user.user_id,
col(RankHistory.mode) == gamemode,
col(RankHistory.date) == target_date,
)
.values(rank=rank)
)
rank_top = (
await session.exec(
select(RankTop).where(
RankTop.user_id == user.user_id,
RankTop.mode == gamemode,
)
)
).first()
if not rank_top:
rank_top = RankTop(
user_id=user.user_id,
mode=gamemode,
rank=rank,
date=today,
)
session.add(rank_top)
else:
if rank_top.rank > rank:
rank_top.rank = rank
rank_top.date = today
rank += 1
await session.commit()

29
app/service/create_banchobot.py
View File

@@ -1,29 +0,0 @@
from __future__ import annotations
from app.const import BANCHOBOT_ID
from app.database.statistics import UserStatistics
from app.database.user import User
from app.dependencies.database import with_db
from app.models.score import GameMode
from sqlmodel import exists, select
async def create_banchobot():
async with with_db() as session:
is_exist = (await session.exec(select(exists()).where(User.id == BANCHOBOT_ID))).first()
if not is_exist:
banchobot = User(
username="BanchoBot",
email="banchobot@ppy.sh",
is_bot=True,
pw_bcrypt="0",
id=BANCHOBOT_ID,
avatar_url="https://a.ppy.sh/3",
country_code="SH",
website="https://twitter.com/banchoboat",
)
session.add(banchobot)
statistics = UserStatistics(user_id=BANCHOBOT_ID, mode=GameMode.OSU)
session.add(statistics)
await session.commit()

182
app/service/daily_challenge.py
View File

@@ -1,182 +0,0 @@
from __future__ import annotations
from datetime import UTC, timedelta
import json
from math import ceil
from app.const import BANCHOBOT_ID
from app.database.daily_challenge import DailyChallengeStats
from app.database.playlist_best_score import PlaylistBestScore
from app.database.playlists import Playlist
from app.database.room import Room
from app.database.score import Score
from app.database.user import User
from app.dependencies.database import get_redis, with_db
from app.dependencies.scheduler import get_scheduler
from app.log import logger
from app.models.metadata_hub import DailyChallengeInfo
from app.models.mods import APIMod, get_available_mods
from app.models.room import RoomCategory
from app.utils import are_same_weeks, utcnow
from .room import create_playlist_room
from sqlmodel import col, select
async def create_daily_challenge_room(
beatmap: int,
ruleset_id: int,
duration: int,
required_mods: list[APIMod] = [],
allowed_mods: list[APIMod] = [],
) -> Room:
async with with_db() as session:
today = utcnow().date()
return await create_playlist_room(
session=session,
name=str(today),
host_id=BANCHOBOT_ID,
playlist=[
Playlist(
id=0,
room_id=0,
owner_id=BANCHOBOT_ID,
ruleset_id=ruleset_id,
beatmap_id=beatmap,
required_mods=required_mods,
allowed_mods=allowed_mods,
)
],
category=RoomCategory.DAILY_CHALLENGE,
duration=duration,
)
@get_scheduler().scheduled_job("cron", hour=0, minute=0, second=0, id="daily_challenge")
async def daily_challenge_job():
from app.signalr.hub import MetadataHubs
now = utcnow()
redis = get_redis()
key = f"daily_challenge:{now.date()}"
if not await redis.exists(key):
return
async with with_db() as session:
room = (
await session.exec(
select(Room).where(
Room.category == RoomCategory.DAILY_CHALLENGE,
col(Room.ends_at) > utcnow(),
)
)
).first()
if room:
return
try:
beatmap = await redis.hget(key, "beatmap") # pyright: ignore[reportGeneralTypeIssues]
ruleset_id = await redis.hget(key, "ruleset_id") # pyright: ignore[reportGeneralTypeIssues]
required_mods = await redis.hget(key, "required_mods") # pyright: ignore[reportGeneralTypeIssues]
allowed_mods = await redis.hget(key, "allowed_mods") # pyright: ignore[reportGeneralTypeIssues]
if beatmap is None or ruleset_id is None:
logger.warning(
f"[DailyChallenge] Missing required data for daily challenge {now}. Will try again in 5 minutes."
)
get_scheduler().add_job(
daily_challenge_job,
"date",
run_date=utcnow() + timedelta(minutes=5),
)
return
beatmap_int = int(beatmap)
ruleset_id_int = int(ruleset_id)
required_mods_list = []
allowed_mods_list = []
if required_mods:
required_mods_list = json.loads(required_mods)
if allowed_mods:
allowed_mods_list = json.loads(allowed_mods)
else:
allowed_mods_list = get_available_mods(ruleset_id_int, required_mods_list)
next_day = (now + timedelta(days=1)).replace(hour=0, minute=0, second=0, microsecond=0)
room = await create_daily_challenge_room(
beatmap=beatmap_int,
ruleset_id=ruleset_id_int,
required_mods=required_mods_list,
allowed_mods=allowed_mods_list,
duration=int((next_day - now - timedelta(minutes=2)).total_seconds() / 60),
)
await MetadataHubs.broadcast_call("DailyChallengeUpdated", DailyChallengeInfo(room_id=room.id))
logger.success(f"[DailyChallenge] Added today's daily challenge: {beatmap=}, {ruleset_id=}, {required_mods=}")
return
except (ValueError, json.JSONDecodeError) as e:
logger.warning(f"[DailyChallenge] Error processing daily challenge data: {e} Will try again in 5 minutes.")
except Exception as e:
logger.exception(f"[DailyChallenge] Unexpected error in daily challenge job: {e} Will try again in 5 minutes.")
get_scheduler().add_job(
daily_challenge_job,
"date",
run_date=utcnow() + timedelta(minutes=5),
)
@get_scheduler().scheduled_job("cron", hour=0, minute=1, second=0, id="daily_challenge_last_top")
async def process_daily_challenge_top():
async with with_db() as session:
now = utcnow()
room = (
await session.exec(
select(Room).where(
Room.category == RoomCategory.DAILY_CHALLENGE,
col(Room.ends_at) > now - timedelta(days=1),
col(Room.ends_at) < now,
)
)
).first()
participated_users = []
if room is not None:
scores = (
await session.exec(
select(PlaylistBestScore)
.where(
PlaylistBestScore.room_id == room.id,
PlaylistBestScore.playlist_id == 0,
col(PlaylistBestScore.score).has(col(Score.passed).is_(True)),
)
.order_by(col(PlaylistBestScore.total_score).desc())
)
).all()
total_score_count = len(scores)
s = []
for i, score in enumerate(scores):
stats = await session.get(DailyChallengeStats, score.user_id)
if stats is None: # not execute
continue
if stats.last_update is None or stats.last_update.replace(tzinfo=UTC).date() != now.date():
if total_score_count < 10 or ceil(i + 1 / total_score_count) <= 0.1:
stats.top_10p_placements += 1
if total_score_count < 2 or ceil(i + 1 / total_score_count) <= 0.5:
stats.top_50p_placements += 1
s.append(s)
participated_users.append(score.user_id)
stats.last_update = now
await session.commit()
del s
user_ids = (await session.exec(select(User.id).where(col(User.id).not_in(participated_users)))).all()
for id in user_ids:
stats = await session.get(DailyChallengeStats, id)
if stats is None: # not execute
continue
stats.daily_streak_current = 0
if stats.last_weekly_streak and not are_same_weeks(
stats.last_weekly_streak.replace(tzinfo=UTC), now - timedelta(days=7)
):
stats.weekly_streak_current = 0
stats.last_update = now
await session.commit()

17
app/service/database_cleanup_service.py
View File

@@ -8,8 +8,6 @@ from datetime import timedelta
from app.database.auth import OAuthToken
from app.database.verification import EmailVerification, LoginSession, TrustedDevice
from app.dependencies.database import with_db
from app.dependencies.scheduler import get_scheduler
from app.log import logger
from app.utils import utcnow
@@ -434,18 +432,3 @@ class DatabaseCleanupService:
"outdated_trusted_devices": 0,
"total_cleanable": 0,
}
@get_scheduler().scheduled_job(
"interval",
id="cleanup_database",
hours=1,
)
async def scheduled_cleanup_job():
async with with_db() as session:
logger.debug("Starting database cleanup...")
results = await DatabaseCleanupService.run_full_cleanup(session)
total = sum(results.values())
if total > 0:
logger.debug(f"Cleanup completed, total records cleaned: {total}")
return results

2
app/service/email_queue.py
View File

@@ -17,7 +17,7 @@ import uuid
from app.config import settings
from app.log import logger
from app.utils import bg_tasks # 添加同步Redis导入
from app.utils import bg_tasks
import redis as sync_redis

55
app/service/geoip_scheduler.py
View File

@@ -1,55 +0,0 @@
"""
[GeoIP] Scheduled Update Service
Periodically update the MaxMind GeoIP database
"""
from __future__ import annotations
import asyncio
from app.config import settings
from app.dependencies.geoip import get_geoip_helper
from app.dependencies.scheduler import get_scheduler
from app.log import logger
async def update_geoip_database():
"""
Asynchronous task to update the GeoIP database
"""
try:
logger.info("[GeoIP] Starting scheduled GeoIP database update...")
geoip = get_geoip_helper()
# Run the synchronous update method in a background thread
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, lambda: geoip.update(force=False))
logger.info("[GeoIP] Scheduled GeoIP database update completed successfully")
except Exception as e:
logger.error(f"[GeoIP] Scheduled GeoIP database update failed: {e}")
def schedule_geoip_updates():
"""
Schedule the GeoIP database update task
"""
scheduler = get_scheduler()
# Use settings to configure the update time: update once a week
scheduler.add_job(
update_geoip_database,
"cron",
day_of_week=settings.geoip_update_day,
hour=settings.geoip_update_hour,
minute=0,
id="geoip_weekly_update",
name="Weekly GeoIP database update",
replace_existing=True,
)
logger.info(
f"[GeoIP] Scheduled update task registered: "
f"every week on day {settings.geoip_update_day} "
f"at {settings.geoip_update_hour}:00"
)

30
app/service/init_geoip.py
View File

@@ -1,30 +0,0 @@
"""
[GeoIP] Initialization Service
Initialize the GeoIP database when the application starts
"""
from __future__ import annotations
import asyncio
from app.dependencies.geoip import get_geoip_helper
from app.log import logger
async def init_geoip():
"""
Asynchronously initialize the GeoIP database
"""
try:
geoip = get_geoip_helper()
logger.info("[GeoIP] Initializing GeoIP database...")
# Run the synchronous update method in a background thread
# force=False means only download if files don't exist or are expired
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, lambda: geoip.update(force=False))
logger.info("[GeoIP] GeoIP database initialization completed")
except Exception as e:
logger.error(f"[GeoIP] GeoIP database initialization failed: {e}")
# Do not raise an exception to avoid blocking application startup

18
app/service/load_achievements.py
View File

@@ -1,18 +0,0 @@
from __future__ import annotations
import importlib
from app.log import logger
from app.models.achievement import MEDALS, Medals
from app.path import ACHIEVEMENTS_DIR
def load_achievements() -> Medals:
for module in ACHIEVEMENTS_DIR.iterdir():
if module.is_file() and module.suffix == ".py":
module_name = module.stem
module_achievements = importlib.import_module(f"app.achievements.{module_name}")
medals = getattr(module_achievements, "MEDALS", {})
MEDALS.update(medals)
logger.success(f"Successfully loaded {len(medals)} achievements from {module_name}.py")
return MEDALS

50
app/service/osu_rx_statistics.py
View File

@@ -1,50 +0,0 @@
from __future__ import annotations
from app.config import settings
from app.const import BANCHOBOT_ID
from app.database.statistics import UserStatistics
from app.database.user import User
from app.dependencies.database import with_db
from app.models.score import GameMode
from sqlalchemy import exists
from sqlmodel import select
async def create_rx_statistics():
async with with_db() as session:
users = (await session.exec(select(User.id))).all()
for i in users:
if i == BANCHOBOT_ID:
continue
if settings.enable_rx:
for mode in (
GameMode.OSURX,
GameMode.TAIKORX,
GameMode.FRUITSRX,
):
is_exist = (
await session.exec(
select(exists()).where(
UserStatistics.user_id == i,
UserStatistics.mode == mode,
)
)
).first()
if not is_exist:
statistics_rx = UserStatistics(mode=mode, user_id=i)
session.add(statistics_rx)
if settings.enable_ap:
is_exist = (
await session.exec(
select(exists()).where(
UserStatistics.user_id == i,
UserStatistics.mode == GameMode.OSUAP,
)
)
).first()
if not is_exist:
statistics_ap = UserStatistics(mode=GameMode.OSUAP, user_id=i)
session.add(statistics_ap)
await session.commit()

130
app/service/recalculate_banned_beatmap.py
View File

@@ -1,130 +0,0 @@
from __future__ import annotations
import asyncio
import json
from app.calculator import calculate_pp
from app.config import settings
from app.database.beatmap import BannedBeatmaps, Beatmap
from app.database.best_scores import PPBestScore
from app.database.score import Score, calculate_user_pp
from app.database.statistics import UserStatistics
from app.dependencies.database import get_redis, with_db
from app.dependencies.fetcher import get_fetcher
from app.dependencies.scheduler import get_scheduler
from app.log import logger
from app.models.mods import mods_can_get_pp
from sqlmodel import col, delete, select
@get_scheduler().scheduled_job("interval", id="recalculate_banned_beatmap", hours=1)
async def recalculate_banned_beatmap():
redis = get_redis()
last_banned_beatmaps = set()
last_banned = await redis.get("last_banned_beatmap")
if last_banned:
last_banned_beatmaps = set(json.loads(last_banned))
affected_users = set()
async with with_db() as session:
query = select(BannedBeatmaps.beatmap_id).distinct()
if last_banned_beatmaps:
query = query.where(col(BannedBeatmaps.beatmap_id).not_in(last_banned_beatmaps))
new_banned_beatmaps = (await session.exec(query)).all()
current_banned = (await session.exec(select(BannedBeatmaps.beatmap_id).distinct())).all()
unbanned_beatmaps = [b for b in last_banned_beatmaps if b not in current_banned]
for i in new_banned_beatmaps:
last_banned_beatmaps.add(i)
await session.execute(delete(PPBestScore).where(col(PPBestScore.beatmap_id) == i))
scores = (await session.exec(select(Score).where(Score.beatmap_id == i, Score.pp > 0))).all()
for score in scores:
score.pp = 0
affected_users.add((score.user_id, score.gamemode))
if unbanned_beatmaps:
fetcher = await get_fetcher()
for beatmap_id in unbanned_beatmaps:
last_banned_beatmaps.discard(beatmap_id)
try:
scores = (
await session.exec(
select(Score).where(
Score.beatmap_id == beatmap_id,
col(Score.passed).is_(True),
)
)
).all()
except Exception:
logger.exception(f"Failed to query scores for unbanned beatmap {beatmap_id}")
continue
prev: dict[tuple[int, int], PPBestScore] = {}
for score in scores:
attempts = 3
while attempts > 0:
try:
db_beatmap = await fetcher.get_or_fetch_beatmap_raw(redis, beatmap_id)
break
except Exception:
attempts -= 1
await asyncio.sleep(1)
else:
logger.warning(f"Could not fetch beatmap raw for {beatmap_id}, skipping pp calc")
continue
try:
beatmap_obj = await Beatmap.get_or_fetch(session, fetcher, bid=beatmap_id)
except Exception:
beatmap_obj = None
ranked = (
beatmap_obj.beatmap_status.has_pp() if beatmap_obj else False
) | settings.enable_all_beatmap_pp
if not ranked or not mods_can_get_pp(int(score.gamemode), score.mods):
continue
try:
pp = await calculate_pp(score, db_beatmap, session)
if not pp or pp == 0:
continue
key = (score.beatmap_id, score.user_id)
if key not in prev or prev[key].pp < pp:
best_score = PPBestScore(
user_id=score.user_id,
beatmap_id=beatmap_id,
acc=score.accuracy,
score_id=score.id,
pp=pp,
gamemode=score.gamemode,
)
prev[key] = best_score
affected_users.add((score.user_id, score.gamemode))
score.pp = pp
except Exception:
logger.exception(f"Error calculating pp for score {score.id} on unbanned beatmap {beatmap_id}")
continue
for best in prev.values():
session.add(best)
for user_id, gamemode in affected_users:
statistics = (
await session.exec(
select(UserStatistics)
.where(UserStatistics.user_id == user_id)
.where(col(UserStatistics.mode) == gamemode)
)
).first()
if not statistics:
continue
statistics.pp, statistics.hit_accuracy = await calculate_user_pp(session, statistics.user_id, gamemode)
await session.commit()
logger.info(
f"Recalculated banned beatmaps, banned {len(new_banned_beatmaps)} beatmaps, "
f"unbanned {len(unbanned_beatmaps)} beatmaps, affected {len(affected_users)} users"
)
await redis.set("last_banned_beatmap", json.dumps(list(last_banned_beatmaps)))

53
app/service/recalculate_failed_score.py
View File

@@ -1,53 +0,0 @@
from __future__ import annotations
from app.calculator import pre_fetch_and_calculate_pp
from app.database.score import Score, calculate_user_pp
from app.database.statistics import UserStatistics
from app.dependencies.database import get_redis, with_db
from app.dependencies.fetcher import get_fetcher
from app.dependencies.scheduler import get_scheduler
from app.log import logger
from sqlmodel import select
@get_scheduler().scheduled_job("interval", id="recalculate_banned_beatmap", minutes=5)
async def recalculate_failed_score():
redis = get_redis()
fetcher = await get_fetcher()
need_add = set()
affected_user = set()
while True:
scores = await redis.lpop("score:need_recalculate", 100) # pyright: ignore[reportGeneralTypeIssues]
if not scores:
break
if isinstance(scores, bytes):
scores = [scores]
async with with_db() as session:
for score_id in scores:
score_id = int(score_id)
score = await session.get(Score, score_id)
if score is None:
continue
pp, successed = await pre_fetch_and_calculate_pp(score, session, redis, fetcher)
if not successed:
need_add.add(score_id)
else:
score.pp = pp
logger.info(
f"Recalculated PP for score {score.id} (user: {score.user_id}) at {score.ended_at}: {pp}"
)
affected_user.add((score.user_id, score.gamemode))
await session.commit()
for user_id, gamemode in affected_user:
stats = (
await session.exec(
select(UserStatistics).where(UserStatistics.user_id == user_id, UserStatistics.mode == gamemode)
)
).first()
if not stats:
continue
stats.pp, stats.hit_accuracy = await calculate_user_pp(session, user_id, gamemode)
await session.commit()
if need_add:
await redis.rpush("score:need_recalculate", *need_add) # pyright: ignore[reportGeneralTypeIssues]