feat(score): auto recalculate for banned beatmaps

Running every hour

app/database/score.py

@@ -583,6 +583,20 @@ async def get_user_best_pp_in_beatmap(
     ).first()
 
 
+async def calculate_user_pp(session: AsyncSession, user_id: int, mode: GameMode) -> tuple[float, float]:
+    pp_sum = 0
+    acc_sum = 0
+    bps = await get_user_best_pp(session, user_id, mode)
+    for i, s in enumerate(bps):
+        pp_sum += calculate_weighted_pp(s.pp, i)
+        acc_sum += calculate_weighted_acc(s.acc, i)
+    if len(bps):
+        # https://github.com/ppy/osu-queue-score-statistics/blob/c538ae/osu.Server.Queues.ScoreStatisticsProcessor/Helpers/UserTotalPerformanceAggregateHelper.cs#L41-L45
+        acc_sum *= 100 / (20 * (1 - math.pow(0.95, len(bps))))
+    acc_sum = clamp(acc_sum, 0.0, 100.0)
+    return pp_sum, acc_sum
+
+
 async def get_user_best_pp(
     session: AsyncSession,
     user: int,
@@ -791,18 +805,9 @@ async def process_user(
 
     if score.passed and ranked:
         with session.no_autoflush:
-            best_pp_scores = await get_user_best_pp(session, user.id, score.gamemode)
+            statistics.pp, statistics.hit_accuracy = await calculate_user_pp(
-            pp_sum = 0.0
+                session, statistics.user_id, score.gamemode
-            acc_sum = 0.0
+            )
-            for i, bp in enumerate(best_pp_scores):
-                pp_sum += calculate_weighted_pp(bp.pp, i)
-                acc_sum += calculate_weighted_acc(bp.acc, i)
-            if len(best_pp_scores):
-                # https://github.com/ppy/osu-queue-score-statistics/blob/c538ae/osu.Server.Queues.ScoreStatisticsProcessor/Helpers/UserTotalPerformanceAggregateHelper.cs#L41-L45
-                acc_sum *= 100 / (20 * (1 - math.pow(0.95, len(best_pp_scores))))
-            acc_sum = clamp(acc_sum, 0.0, 100.0)
-            statistics.pp = pp_sum
-            statistics.hit_accuracy = acc_sum
     if add_to_db:
         session.add(mouthly_playcount)
     with session.no_autoflush:

app/service/__init__.py

@@ -1,10 +1,12 @@
 from __future__ import annotations
 
 from .daily_challenge import create_daily_challenge_room
+from .recalculate_banned_beatmap import recalculate_banned_beatmap
 from .room import create_playlist_room, create_playlist_room_from_api
 
 __all__ = [
     "create_daily_challenge_room",
     "create_playlist_room",
     "create_playlist_room_from_api",
+    "recalculate_banned_beatmap",
 ]

app/service/recalculate_banned_beatmap.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import asyncio
+import json
+
+from app.calculator import calculate_pp
+from app.config import settings
+from app.database.beatmap import BannedBeatmaps, Beatmap
+from app.database.pp_best_score import PPBestScore
+from app.database.score import Score, calculate_user_pp
+from app.database.statistics import UserStatistics
+from app.dependencies.database import get_redis, with_db
+from app.dependencies.fetcher import get_fetcher
+from app.dependencies.scheduler import get_scheduler
+from app.log import logger
+from app.models.mods import mods_can_get_pp
+
+from sqlmodel import col, delete, select
+
+
+@get_scheduler().scheduled_job("interval", id="recalculate_banned_beatmap", hours=1)
+async def recalculate_banned_beatmap():
+    redis = get_redis()
+    last_banned_beatmaps = set()
+    last_banned = await redis.get("last_banned_beatmap")
+    if last_banned:
+        last_banned_beatmaps = set(json.loads(last_banned))
+    affected_users = set()
+
+    async with with_db() as session:
+        query = select(BannedBeatmaps.beatmap_id).distinct()
+        if last_banned_beatmaps:
+            query = query.where(col(BannedBeatmaps.beatmap_id).not_in(last_banned_beatmaps))
+        new_banned_beatmaps = (await session.exec(query)).all()
+
+        current_banned = (await session.exec(select(BannedBeatmaps.beatmap_id).distinct())).all()
+        unbanned_beatmaps = [b for b in last_banned_beatmaps if b not in current_banned]
+        for i in new_banned_beatmaps:
+            last_banned_beatmaps.add(i)
+            await session.execute(delete(PPBestScore).where(col(PPBestScore.beatmap_id) == i))
+            scores = (await session.exec(select(Score).where(Score.beatmap_id == i, Score.pp > 0))).all()
+            for score in scores:
+                score.pp = 0
+                affected_users.add((score.user_id, score.gamemode))
+
+        if unbanned_beatmaps:
+            fetcher = await get_fetcher()
+            for beatmap_id in unbanned_beatmaps:
+                last_banned_beatmaps.discard(beatmap_id)
+                try:
+                    scores = (
+                        await session.exec(
+                            select(Score).where(
+                                Score.beatmap_id == beatmap_id,
+                                col(Score.passed).is_(True),
+                            )
+                        )
+                    ).all()
+                except Exception:
+                    logger.exception(f"Failed to query scores for unbanned beatmap {beatmap_id}")
+                    continue
+
+                prev: dict[tuple[int, int], PPBestScore] = {}
+                for score in scores:
+                    attempts = 3
+                    while attempts > 0:
+                        try:
+                            db_beatmap = await fetcher.get_or_fetch_beatmap_raw(redis, beatmap_id)
+                            break
+                        except Exception:
+                            attempts -= 1
+                            await asyncio.sleep(1)
+                    else:
+                        logger.warning(f"Could not fetch beatmap raw for {beatmap_id}, skipping pp calc")
+                        continue
+
+                    try:
+                        beatmap_obj = await Beatmap.get_or_fetch(session, fetcher, bid=beatmap_id)
+                    except Exception:
+                        beatmap_obj = None
+
+                    ranked = (
+                        beatmap_obj.beatmap_status.has_pp() if beatmap_obj else False
+                    ) | settings.enable_all_beatmap_pp
+
+                    if not ranked or not mods_can_get_pp(int(score.gamemode), score.mods):
+                        continue
+
+                    try:
+                        pp = await calculate_pp(score, db_beatmap, session)
+                        if not pp or pp == 0:
+                            continue
+                        key = (score.beatmap_id, score.user_id)
+                        if key not in prev or prev[key].pp < pp:
+                            best_score = PPBestScore(
+                                user_id=score.user_id,
+                                beatmap_id=beatmap_id,
+                                acc=score.accuracy,
+                                score_id=score.id,
+                                pp=pp,
+                                gamemode=score.gamemode,
+                            )
+                            prev[key] = best_score
+                            affected_users.add((score.user_id, score.gamemode))
+                            score.pp = pp
+                    except Exception:
+                        logger.exception(f"Error calculating pp for score {score.id} on unbanned beatmap {beatmap_id}")
+                        continue
+
+                for best in prev.values():
+                    session.add(best)
+
+        for user_id, gamemode in affected_users:
+            statistics = (
+                await session.exec(
+                    select(UserStatistics)
+                    .where(UserStatistics.user_id == user_id)
+                    .where(col(UserStatistics.mode) == gamemode)
+                )
+            ).first()
+            if not statistics:
+                continue
+            statistics.pp, statistics.hit_accuracy = await calculate_user_pp(session, statistics.user_id, gamemode)
+
+        await session.commit()
+    logger.info(
+        f"Recalculated banned beatmaps, banned {len(new_banned_beatmaps)} beatmaps, "
+        f"unbanned {len(unbanned_beatmaps)} beatmaps, affected {len(affected_users)} users"
+    )
+    await redis.set("last_banned_beatmap", json.dumps(list(last_banned_beatmaps)))

tools/recalculate.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import asyncio
-import math
 import os
 import sys
 
@@ -10,16 +9,13 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 from app.calculator import (
     calculate_pp,
     calculate_score_to_level,
-    calculate_weighted_acc,
-    calculate_weighted_pp,
-    clamp,
 )
 from app.config import settings
 from app.const import BANCHOBOT_ID
 from app.database import BestScore, UserStatistics
 from app.database.beatmap import Beatmap
 from app.database.pp_best_score import PPBestScore
-from app.database.score import Score, calculate_playtime, get_user_best_pp
+from app.database.score import Score, calculate_playtime, calculate_user_pp
 from app.dependencies.database import engine, get_redis
 from app.dependencies.fetcher import get_fetcher
 from app.fetcher import Fetcher
@@ -212,18 +208,7 @@ async def _recalculate_best_score(
 
 async def _recalculate_statistics(statistics: UserStatistics, session: AsyncSession):
     async with SEMAPHORE:
-        pp_sum = 0
+        statistics.pp, statistics.hit_accuracy = await calculate_user_pp(session, statistics.user_id, statistics.mode)
-        acc_sum = 0
-        bps = await get_user_best_pp(session, statistics.user_id, statistics.mode)
-        for i, s in enumerate(bps):
-            pp_sum += calculate_weighted_pp(s.pp, i)
-            acc_sum += calculate_weighted_acc(s.acc, i)
-        if len(bps):
-            # https://github.com/ppy/osu-queue-score-statistics/blob/c538ae/osu.Server.Queues.ScoreStatisticsProcessor/Helpers/UserTotalPerformanceAggregateHelper.cs#L41-L45
-            acc_sum *= 100 / (20 * (1 - math.pow(0.95, len(bps))))
-        acc_sum = clamp(acc_sum, 0.0, 100.0)
-        statistics.pp = pp_sum
-        statistics.hit_accuracy = acc_sum
 
         statistics.play_count = 0
         statistics.total_score = 0