From 4b5aefb94644b0fe95ae1aacafbe831e33f4696b Mon Sep 17 00:00:00 2001 From: MingxuanGame Date: Wed, 13 Aug 2025 07:55:48 +0000 Subject: [PATCH] feat(beatmapset): support search beatmapset --- app/database/beatmap.py | 10 +- app/database/beatmapset.py | 65 ++++++--- app/fetcher/beatmapset.py | 24 +++- app/models/beatmap.py | 125 +++++++++++++++++- app/router/v2/beatmapset.py | 68 +++++++++- .../59c9a0827de0_beatmap_add_indexes.py | 124 +++++++++++++++++ 6 files changed, 386 insertions(+), 30 deletions(-) create mode 100644 migrations/versions/59c9a0827de0_beatmap_add_indexes.py diff --git a/app/database/beatmap.py b/app/database/beatmap.py index 7e770b7..88ff61c 100644 --- a/app/database/beatmap.py +++ b/app/database/beatmap.py @@ -28,10 +28,10 @@ class BeatmapBase(SQLModel): url: str mode: GameMode beatmapset_id: int = Field(foreign_key="beatmapsets.id", index=True) - difficulty_rating: float = Field(default=0.0) + difficulty_rating: float = Field(default=0.0, index=True) total_length: int - user_id: int - version: str + user_id: int = Field(index=True) + version: str = Field(index=True) # optional checksum: str = Field(sa_column=Column(VARCHAR(32), index=True)) @@ -50,14 +50,14 @@ class BeatmapBase(SQLModel): count_spinners: int = Field(default=0) deleted_at: datetime | None = Field(default=None, sa_column=Column(DateTime)) hit_length: int = Field(default=0) - last_updated: datetime = Field(sa_column=Column(DateTime)) + last_updated: datetime = Field(sa_column=Column(DateTime, index=True)) class Beatmap(BeatmapBase, table=True): __tablename__ = "beatmaps" # pyright: ignore[reportAssignmentType] id: int = Field(primary_key=True, index=True) beatmapset_id: int = Field(foreign_key="beatmapsets.id", index=True) - beatmap_status: BeatmapRankStatus + beatmap_status: BeatmapRankStatus = Field(index=True) # optional beatmapset: Beatmapset = Relationship( back_populates="beatmaps", sa_relationship_kwargs={"lazy": "joined"} diff --git a/app/database/beatmapset.py b/app/database/beatmapset.py index 1ed5d13..6a5ca2b 100644 --- a/app/database/beatmapset.py +++ b/app/database/beatmapset.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import TYPE_CHECKING, NotRequired, TypedDict +from typing import TYPE_CHECKING, NotRequired, Self, TypedDict from app.config import settings from app.models.beatmap import BeatmapRankStatus, Genre, Language @@ -7,7 +7,7 @@ from app.models.score import GameMode from .lazer_user import BASE_INCLUDES, User, UserResp -from pydantic import BaseModel +from pydantic import BaseModel, model_validator from sqlalchemy import JSON, Column, DateTime, Text from sqlalchemy.ext.asyncio import AsyncAttrs from sqlmodel import Field, Relationship, SQLModel, col, exists, func, select @@ -72,17 +72,17 @@ class BeatmapsetBase(SQLModel): artist: str = Field(index=True) artist_unicode: str = Field(index=True) covers: BeatmapCovers | None = Field(sa_column=Column(JSON)) - creator: str + creator: str = Field(index=True) nsfw: bool = Field(default=False) - play_count: int + play_count: int = Field(index=True) preview_url: str source: str = Field(default="") spotlight: bool = Field(default=False) - title: str - title_unicode: str - user_id: int - video: bool + title: str = Field(index=True) + title_unicode: str = Field(index=True) + user_id: int = Field(index=True) + video: bool = Field(index=True) # optional # converts: list[Beatmap] = Relationship(back_populates="beatmapset") @@ -95,19 +95,21 @@ class BeatmapsetBase(SQLModel): # TODO: events: Optional[list[BeatmapsetEvent]] = None pack_tags: list[str] = Field(default=[], sa_column=Column(JSON)) - ratings: list[int] = Field(default=None, sa_column=Column(JSON)) + ratings: list[int] | None = Field(default=None, sa_column=Column(JSON)) # TODO: related_users: Optional[list[User]] = None # TODO: user: Optional[User] = Field(default=None) - track_id: int | None = Field(default=None) # feature artist? + track_id: int | None = Field(default=None, index=True) # feature artist? # BeatmapsetExtended bpm: float = Field(default=0.0) can_be_hyped: bool = Field(default=False) discussion_locked: bool = Field(default=False) - last_updated: datetime = Field(sa_column=Column(DateTime)) - ranked_date: datetime | None = Field(default=None, sa_column=Column(DateTime)) - storyboard: bool = Field(default=False) - submitted_date: datetime = Field(sa_column=Column(DateTime)) + last_updated: datetime = Field(sa_column=Column(DateTime, index=True)) + ranked_date: datetime | None = Field( + default=None, sa_column=Column(DateTime, index=True) + ) + storyboard: bool = Field(default=False, index=True) + submitted_date: datetime = Field(sa_column=Column(DateTime, index=True)) tags: str = Field(default="", sa_column=Column(Text)) @@ -117,13 +119,13 @@ class Beatmapset(AsyncAttrs, BeatmapsetBase, table=True): id: int | None = Field(default=None, primary_key=True, index=True) # Beatmapset beatmap_status: BeatmapRankStatus = Field( - default=BeatmapRankStatus.GRAVEYARD, + default=BeatmapRankStatus.GRAVEYARD, index=True ) # optional beatmaps: list["Beatmap"] = Relationship(back_populates="beatmapset") - beatmap_genre: Genre = Field(default=Genre.UNSPECIFIED) - beatmap_language: Language = Field(default=Language.UNSPECIFIED) + beatmap_genre: Genre = Field(default=Genre.UNSPECIFIED, index=True) + beatmap_language: Language = Field(default=Language.UNSPECIFIED, index=True) nominations_required: int = Field(default=0) nominations_current: int = Field(default=0) @@ -148,9 +150,13 @@ class Beatmapset(AsyncAttrs, BeatmapsetBase, table=True): if resp.hype: update["hype_current"] = resp.hype.current update["hype_required"] = resp.hype.required - if resp.genre: + if resp.genre_id: + update["beatmap_genre"] = Genre(resp.genre_id) + elif resp.genre: update["beatmap_genre"] = Genre(resp.genre.id) - if resp.language: + if resp.language_id: + update["beatmap_language"] = Language(resp.language_id) + elif resp.language: update["beatmap_language"] = Language(resp.language.id) beatmapset = Beatmapset.model_validate( { @@ -191,12 +197,26 @@ class BeatmapsetResp(BeatmapsetBase): hype: BeatmapHype | None = None availability: BeatmapAvailability genre: BeatmapTranslationText | None = None + genre_id: int language: BeatmapTranslationText | None = None + language_id: int nominations: BeatmapNominations | None = None has_favourited: bool = False favourite_count: int = 0 recent_favourites: list[UserResp] = Field(default_factory=list) + @model_validator(mode="after") + def fix_genre_language(self) -> Self: + if self.genre is None: + self.genre = BeatmapTranslationText( + name=Genre(self.genre_id).name, id=self.genre_id + ) + if self.language is None: + self.language = BeatmapTranslationText( + name=Language(self.language_id).name, id=self.language_id + ) + return self + @classmethod async def from_db( cls, @@ -228,6 +248,8 @@ class BeatmapsetResp(BeatmapsetBase): name=beatmapset.beatmap_language.name, id=beatmapset.beatmap_language.value, ), + "genre_id": beatmapset.beatmap_genre.value, + "language_id": beatmapset.beatmap_language.value, "nominations": BeatmapNominations( required=beatmapset.nominations_required, current=beatmapset.nominations_current, @@ -288,3 +310,8 @@ class BeatmapsetResp(BeatmapsetBase): return cls.model_validate( update, ) + + +class SearchBeatmapsetsResp(SQLModel): + beatmapsets: list[BeatmapsetResp] + total: int diff --git a/app/fetcher/beatmapset.py b/app/fetcher/beatmapset.py index 8062825..181eb05 100644 --- a/app/fetcher/beatmapset.py +++ b/app/fetcher/beatmapset.py @@ -1,7 +1,9 @@ from __future__ import annotations -from app.database.beatmapset import BeatmapsetResp +from app.database.beatmapset import BeatmapsetResp, SearchBeatmapsetsResp from app.log import logger +from app.models.beatmap import SearchQueryModel +from app.models.model import Cursor from ._base import BaseFetcher @@ -17,3 +19,23 @@ class BeatmapsetFetcher(BaseFetcher): f"https://osu.ppy.sh/api/v2/beatmapsets/{beatmap_set_id}" ) ) + + async def search_beatmapset( + self, query: SearchQueryModel, cursor: Cursor + ) -> SearchBeatmapsetsResp: + logger.opt(colors=True).debug( + f"[BeatmapsetFetcher] search_beatmapset: {query}" + ) + + params = query.model_dump( + exclude_none=True, exclude_unset=True, exclude_defaults=True + ) + for k, v in cursor.items(): + params[f"cursor[{k}]"] = v + resp = SearchBeatmapsetsResp.model_validate( + await self.request_api( + "https://osu.ppy.sh/api/v2/beatmapsets/search", + params=params, + ) + ) + return resp diff --git a/app/models/beatmap.py b/app/models/beatmap.py index d9bdd5c..b639467 100644 --- a/app/models/beatmap.py +++ b/app/models/beatmap.py @@ -1,8 +1,11 @@ from __future__ import annotations from enum import IntEnum +from typing import Annotated, Any, Literal -from pydantic import BaseModel +from .score import Rank + +from pydantic import BaseModel, BeforeValidator, Field, PlainSerializer class BeatmapRankStatus(IntEnum): @@ -79,3 +82,123 @@ class BeatmapAttributes(BaseModel): # taiko mono_stamina_factor: float | None = None + + +def _parse_list(v: Any): + if isinstance(v, str): + return v.split(".") + return v + + +class SearchQueryModel(BaseModel): + # model_config = ConfigDict(serialize_by_alias=True) + + q: str = Field("", description="搜索关键词") + c: Annotated[ + list[ + Literal[ + "recommended", "converts", "follows", "spotlights", "featured_artists" + ] + ], + BeforeValidator(_parse_list), + PlainSerializer(lambda x: ".".join(x)), + ] = Field( + default_factory=list, + description=( + "常规：recommended 推荐难度 / converts 包括转谱 / follows 已关注谱师 / " + "spotlights 聚光灯谱面 / featured_artists 精选艺术家" + ), + ) + m: int | None = Field(None, description="模式", alias="m") + s: Literal[ + "any", + "leaderboard", + "ranked", + "qualified", + "loved", + "favourites", + "pending", + "wip", + "graveyard", + "mine", + ] = Field( + default="leaderboard", + description=( + "分类：any 全部 / leaderboard 拥有排行榜 / ranked 上架 / " + "qualified 过审 / loved 社区喜爱 / favourites 收藏 / " + "pending 待定 / wip 制作中 / graveyard 坟场 / mine 我做的谱面" + ), + ) + l: Literal[ # noqa: E741 + "any", + "unspecified", + "english", + "japanese", + "chinese", + "instrumental", + "korean", + "french", + "german", + "swedish", + "spanish", + "italian", + "russian", + "polish", + "other", + ] = Field( + default="any", + description=( + "语言：any 全部 / unspecified 未指定 / english 英语 / japanese 日语 / " + "chinese 中文 / instrumental 器乐 / korean 韩语 / " + "french 法语 / german 德语 / swedish 瑞典语 / " + "spanish 西班牙语 / italian 意大利语 / russian 俄语 / " + "polish 波兰语 / other 其他" + ), + ) + sort: Literal[ + "title_asc", + "artist_asc", + "difficulty_asc", + "updated_asc", + "ranked_asc", + "rating_asc", + "plays_asc", + "favourites_asc", + "relevance_asc", + "nominations_asc", + "title_desc", + "artist_desc", + "difficulty_desc", + "updated_desc", + "ranked_desc", + "rating_desc", + "plays_desc", + "favourites_desc", + "relevance_desc", + "nominations_desc", + ] = Field( + ..., + description=( + "排序方式： title 标题 / artist 艺术家 / difficulty 难度 / updated 更新时间" + " / ranked 上架时间 / rating 评分 / plays 游玩次数 / favourites 收藏量" + " / relevance 相关性 / nominations 提名" + ), + ) + e: Annotated[ + list[Literal["video", "storyboard"]], + BeforeValidator(_parse_list), + PlainSerializer(lambda x: ".".join(x)), + ] = Field( + default_factory=list, description=("其他：video 有视频 / storyboard 有故事板") + ) + r: Annotated[ + list[Rank], BeforeValidator(_parse_list), PlainSerializer(lambda x: ".".join(x)) + ] = Field(default_factory=list, description="成绩") + played: bool = Field( + default=False, + description="玩过", + ) + nsfw: bool = Field( + default=False, + description="不良内容", + ) diff --git a/app/router/v2/beatmapset.py b/app/router/v2/beatmapset.py index 9d2fbae..ebd8ecd 100644 --- a/app/router/v2/beatmapset.py +++ b/app/router/v2/beatmapset.py @@ -1,22 +1,82 @@ from __future__ import annotations -from typing import Literal +import re +from typing import Annotated, Literal +from urllib.parse import parse_qs from app.database import Beatmap, Beatmapset, BeatmapsetResp, FavouriteBeatmapset, User -from app.dependencies.database import get_db +from app.database.beatmapset import SearchBeatmapsetsResp +from app.dependencies.database import engine, get_db from app.dependencies.fetcher import get_fetcher from app.dependencies.user import get_client_user, get_current_user from app.fetcher import Fetcher +from app.models.beatmap import SearchQueryModel from .router import router -from fastapi import Depends, Form, HTTPException, Path, Query, Security +from fastapi import ( + BackgroundTasks, + Depends, + Form, + HTTPException, + Path, + Query, + Request, + Security, +) from fastapi.responses import RedirectResponse from httpx import HTTPError -from sqlmodel import select +from sqlmodel import exists, select from sqlmodel.ext.asyncio.session import AsyncSession +async def _save_to_db(sets: SearchBeatmapsetsResp): + async with AsyncSession(engine) as session: + for s in sets.beatmapsets: + if not ( + await session.exec(select(exists()).where(Beatmapset.id == s.id)) + ).first(): + await Beatmapset.from_resp(session, s) + + +@router.get( + "/beatmapsets/search", + name="搜索谱面集", + tags=["谱面集"], + response_model=SearchBeatmapsetsResp, +) +async def search_beatmapset( + query: Annotated[SearchQueryModel, Query(...)], + request: Request, + background_tasks: BackgroundTasks, + current_user: User = Security(get_current_user, scopes=["public"]), + db: AsyncSession = Depends(get_db), + fetcher: Fetcher = Depends(get_fetcher), +): + params = parse_qs(qs=request.url.query, keep_blank_values=True) + cursor = {} + for k, v in params.items(): + match = re.match(r"cursor\[(\w+)\]", k) + if match: + cursor[match.group(1)] = v[0] if v else None + if ( + "recommended" in query.c + or len(query.r) > 0 + or query.played + or "follows" in query.c + or "mine" in query.s + or "favourites" in query.s + ): + # TODO: search locally + return SearchBeatmapsetsResp(total=0, beatmapsets=[]) + try: + sets = await fetcher.search_beatmapset(query, cursor) + background_tasks.add_task(_save_to_db, sets) + except HTTPError as e: + raise HTTPException(status_code=500, detail=str(e)) + return sets + + @router.get( "/beatmapsets/lookup", tags=["谱面集"], diff --git a/migrations/versions/59c9a0827de0_beatmap_add_indexes.py b/migrations/versions/59c9a0827de0_beatmap_add_indexes.py new file mode 100644 index 0000000..b1a2fbb --- /dev/null +++ b/migrations/versions/59c9a0827de0_beatmap_add_indexes.py @@ -0,0 +1,124 @@ +"""beatmap: add indexes + +Revision ID: 59c9a0827de0 +Revises: 881ac7ca01d5 +Create Date: 2025-08-13 07:07:52.506510 + +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "59c9a0827de0" +down_revision: str | Sequence[str] | None = "f785165a5c0b" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_index( + op.f("ix_beatmaps_beatmap_status"), "beatmaps", ["beatmap_status"], unique=False + ) + op.create_index( + op.f("ix_beatmaps_difficulty_rating"), + "beatmaps", + ["difficulty_rating"], + unique=False, + ) + op.create_index( + op.f("ix_beatmaps_last_updated"), "beatmaps", ["last_updated"], unique=False + ) + op.create_index(op.f("ix_beatmaps_user_id"), "beatmaps", ["user_id"], unique=False) + op.create_index(op.f("ix_beatmaps_version"), "beatmaps", ["version"], unique=False) + op.create_index( + op.f("ix_beatmapsets_beatmap_genre"), + "beatmapsets", + ["beatmap_genre"], + unique=False, + ) + op.create_index( + op.f("ix_beatmapsets_beatmap_language"), + "beatmapsets", + ["beatmap_language"], + unique=False, + ) + op.create_index( + op.f("ix_beatmapsets_beatmap_status"), + "beatmapsets", + ["beatmap_status"], + unique=False, + ) + op.create_index( + op.f("ix_beatmapsets_creator"), "beatmapsets", ["creator"], unique=False + ) + op.create_index( + op.f("ix_beatmapsets_last_updated"), + "beatmapsets", + ["last_updated"], + unique=False, + ) + op.create_index( + op.f("ix_beatmapsets_play_count"), "beatmapsets", ["play_count"], unique=False + ) + op.create_index( + op.f("ix_beatmapsets_ranked_date"), "beatmapsets", ["ranked_date"], unique=False + ) + op.create_index( + op.f("ix_beatmapsets_storyboard"), "beatmapsets", ["storyboard"], unique=False + ) + op.create_index( + op.f("ix_beatmapsets_submitted_date"), + "beatmapsets", + ["submitted_date"], + unique=False, + ) + op.create_index( + op.f("ix_beatmapsets_title"), "beatmapsets", ["title"], unique=False + ) + op.create_index( + op.f("ix_beatmapsets_title_unicode"), + "beatmapsets", + ["title_unicode"], + unique=False, + ) + op.create_index( + op.f("ix_beatmapsets_track_id"), "beatmapsets", ["track_id"], unique=False + ) + op.create_index( + op.f("ix_beatmapsets_user_id"), "beatmapsets", ["user_id"], unique=False + ) + op.create_index( + op.f("ix_beatmapsets_video"), "beatmapsets", ["video"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_beatmapsets_video"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_user_id"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_track_id"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_title_unicode"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_title"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_submitted_date"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_storyboard"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_ranked_date"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_play_count"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_last_updated"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_creator"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_beatmap_status"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_beatmap_language"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmapsets_beatmap_genre"), table_name="beatmapsets") + op.drop_index(op.f("ix_beatmaps_version"), table_name="beatmaps") + op.drop_index(op.f("ix_beatmaps_user_id"), table_name="beatmaps") + op.drop_index(op.f("ix_beatmaps_last_updated"), table_name="beatmaps") + op.drop_index(op.f("ix_beatmaps_difficulty_rating"), table_name="beatmaps") + op.drop_index(op.f("ix_beatmaps_beatmap_status"), table_name="beatmaps") + # ### end Alembic commands ###