use SQL's limit/offset pagination for nextIndex/maxCount requests (#185)

Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need.

Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready.

Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client.

### How this works

Say we have a `GetUserCharacterApi` request:

```json
{
    "userId": 10000,
    "maxCount": 700,
    "nextIndex": 0
}
```

Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query:

```python
select(character)
.where(character.c.user == user_id)
.order_by(character.c.id.asc())
.limit(max_count + 1)
.offset(next_index)
```

The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned:

```python
rows = ...

if len(rows) > max_count:
    # return only max_count rows
    next_index += max_count
else:
    # return everything left
    next_index = -1
```

This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](2274b42358/titles/chuni/base.py (L13)).)

Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/185
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>

titles/mai2/base.py

@@ -1,16 +1,17 @@
-from datetime import datetime, timedelta
-from typing import Any, Dict, List
+import itertools
 import logging
 from base64 import b64decode
-from os import path, stat, remove, mkdir, access, W_OK
-from PIL import ImageFile
-from random import randint
+from datetime import datetime, timedelta
+from os import W_OK, access, mkdir, path
+from typing import Any, Dict, List
 
 import pytz
+
 from core.config import CoreConfig
 from core.utils import Utils
-from .const import Mai2Constants
+
 from .config import Mai2Config
+from .const import Mai2Constants
 from .database import Mai2Data
 
 
@@ -444,23 +445,22 @@ class Mai2Base:
         return {"userId": data["userId"], "userOption": options_dict}
 
     async def handle_get_user_card_api_request(self, data: Dict) -> Dict:
-        user_cards = await self.data.item.get_cards(data["userId"])
-        if user_cards is None:
-            return {"userId": data["userId"], "nextIndex": 0, "userCardList": []}
+        user_id = int(data["userId"])
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
 
-        max_ct = data["maxCount"]
-        next_idx = data["nextIndex"]
-        start_idx = next_idx
-        end_idx = max_ct + start_idx
+        user_cards = await self.data.item.get_cards(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
 
-        if len(user_cards[start_idx:]) > max_ct:
-            next_idx += max_ct
-        else:
-            next_idx = 0
+        if user_cards is None or len(user_cards) == 0:
+            return {"userId": user_id, "nextIndex": 0, "userCardList": []}
 
         card_list = []
-        for card in user_cards:
+
+        for card in user_cards[:max_ct]:
             tmp = card._asdict()
+            
             tmp.pop("id")
             tmp.pop("user")
             tmp["startDate"] = datetime.strftime(
@@ -469,12 +469,18 @@ class Mai2Base:
             tmp["endDate"] = datetime.strftime(
                 tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
             )
+            
             card_list.append(tmp)
 
+        if len(user_cards) > max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
-            "userCardList": card_list[start_idx:end_idx],
+            "userCardList": card_list,
         }
 
     async def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
@@ -536,28 +542,35 @@ class Mai2Base:
         return { "userId": data.get("userId", 0), "userBossData": boss_lst}
 
     async def handle_get_user_item_api_request(self, data: Dict) -> Dict:
-        kind = int(data["nextIndex"] / 10000000000)
-        next_idx = int(data["nextIndex"] % 10000000000)
-        user_item_list = await self.data.item.get_items(data["userId"], kind)
+        user_id: int = data["userId"]
+        kind: int = data["nextIndex"] // 10000000000
+        next_idx: int = data["nextIndex"] % 10000000000
+        max_ct: int = data["maxCount"]
+        rows = await self.data.item.get_items(user_id, kind, limit=max_ct, offset=next_idx)
+
+        if rows is None or len(rows) == 0:
+            return {
+                "userId": user_id,
+                "nextIndex": 0,
+                "itemKind": kind,
+                "userItemList": [],
+            }
 
         items: List[Dict[str, Any]] = []
-        for i in range(next_idx, len(user_item_list)):
-            tmp = user_item_list[i]._asdict()
+
+        for row in rows[:max_ct]:
+            tmp = row._asdict()
             tmp.pop("user")
             tmp.pop("id")
             items.append(tmp)
-            if len(items) >= int(data["maxCount"]):
-                break
 
-        xout = kind * 10000000000 + next_idx + len(items)
-
-        if len(items) < int(data["maxCount"]):
-            next_idx = 0
+        if len(rows) > max_ct:
+            next_idx = kind * 10000000000 + next_idx + max_ct
         else:
-            next_idx = xout
+            next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "itemKind": kind,
             "userItemList": items,
@@ -675,77 +688,90 @@ class Mai2Base:
         return {"length": 0, "userPortraitList": []}
 
     async def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict:
-        friend_season_ranking = await self.data.item.get_friend_season_ranking(data["userId"])
-        if friend_season_ranking is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_friend_season_ranking(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             return {
-                "userId": data["userId"],
+                "userId": user_id,
                 "nextIndex": 0,
                 "userFriendSeasonRankingList": [],
             }
 
         friend_season_ranking_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(friend_season_ranking)):
-            tmp = friend_season_ranking[x]._asdict()
-            tmp.pop("user")
+        for row in rows[:max_ct]:
+            tmp = row._asdict()
+            
             tmp.pop("id")
+            tmp.pop("user")
             tmp["recordDate"] = datetime.strftime(
                 tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
             )
+
             friend_season_ranking_list.append(tmp)
 
-            if len(friend_season_ranking_list) >= max_ct:
-                break
-
-        if len(friend_season_ranking) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userFriendSeasonRankingList": friend_season_ranking_list,
         }
 
     async def handle_get_user_map_api_request(self, data: Dict) -> Dict:
-        maps = await self.data.item.get_maps(data["userId"])
-        if maps is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_maps(
+            user_id, limit=max_ct + 1, offset=next_idx,
+        )
+
+        if rows is None:
             return {
-                "userId": data["userId"],
+                "userId": user_id,
                 "nextIndex": 0,
                 "userMapList": [],
             }
 
         map_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(maps)):
-            tmp = maps[x]._asdict()
+        for row in rows[:max_ct]:
+            tmp = row._asdict()
             tmp.pop("user")
             tmp.pop("id")
             map_list.append(tmp)
 
-            if len(map_list) >= max_ct:
-                break
-
-        if len(maps) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userMapList": map_list,
         }
 
     async def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
-        login_bonuses = await self.data.item.get_login_bonuses(data["userId"])
-        if login_bonuses is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_login_bonuses(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             return {
                 "userId": data["userId"],
                 "nextIndex": 0,
@@ -753,25 +779,20 @@ class Mai2Base:
             }
 
         login_bonus_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(login_bonuses)):
-            tmp = login_bonuses[x]._asdict()
+        for row in rows[:max_ct]:
+            tmp = row._asdict()
             tmp.pop("user")
             tmp.pop("id")
             login_bonus_list.append(tmp)
 
-            if len(login_bonus_list) >= max_ct:
-                break
-
-        if len(login_bonuses) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userLoginBonusList": login_bonus_list,
         }
@@ -805,42 +826,54 @@ class Mai2Base:
         return {"userId": data["userId"], "userGradeStatus": grade_stat, "length": 0, "userGradeList": []}
 
     async def handle_get_user_music_api_request(self, data: Dict) -> Dict:
-        user_id = data.get("userId", 0)        
-        next_index = data.get("nextIndex", 0)
-        max_ct = data.get("maxCount", 50)
-        upper_lim = next_index + max_ct
-        music_detail_list = []
+        user_id: int = data.get("userId", 0)        
+        next_idx: int = data.get("nextIndex", 0)
+        max_ct: int = data.get("maxCount", 50)
 
         if user_id <= 0:
             self.logger.warning("handle_get_user_music_api_request: Could not find userid in data, or userId is 0")
             return {}
         
-        songs = await self.data.score.get_best_scores(user_id, is_dx=False)
-        if songs is None:
+        rows = await self.data.score.get_best_scores(
+            user_id, is_dx=False, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!")
             return {
-            "userId": data["userId"],
-            "nextIndex": 0,
-            "userMusicList": [],
-        }
+                "userId": user_id,
+                "nextIndex": 0,
+                "userMusicList": [],
+            }
 
-        num_user_songs = len(songs)
+        music_details = [row._asdict() for row in rows]
+        returned_count = 0
+        music_list = []
 
-        for x in range(next_index, upper_lim):
-            if num_user_songs <= x:
+        for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]):
+            details: list[dict[Any, Any]] = []
+
+            for d in details_iter:
+                d.pop("id")
+                d.pop("user")
+                
+                details.append(d)
+
+            music_list.append({"userMusicDetailList": details})
+            returned_count += len(details)
+
+            if len(music_list) >= max_ct:
                 break
+        
+        if returned_count < len(rows):
+            next_idx += max_ct
+        else:
+            next_idx = 0
 
-            tmp = songs[x]._asdict()
-            tmp.pop("id")
-            tmp.pop("user")
-            music_detail_list.append(tmp)
-
-        next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim
-        self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
         return {
-            "userId": data["userId"],
-            "nextIndex": next_index,
-            "userMusicList": [{"userMusicDetailList": music_detail_list}],
+            "userId": user_id,
+            "nextIndex": next_idx,
+            "userMusicList": music_list,
         }
 
     async def handle_upload_user_portrait_api_request(self, data: Dict) -> Dict:
@@ -925,30 +958,52 @@ class Mai2Base:
     async def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
         user_id = data.get("userId", 0)
         kind = data.get("kind", 0) # 1 is fav music, 2 is rival user IDs
-        next_index = data.get("nextIndex", 0)
+        next_idx = data.get("nextIndex", 0)
         max_ct = data.get("maxCount", 100) # always 100
         is_all = data.get("isAllFavoriteItem", False) # always false
+
+        empty_resp = {
+            "userId": user_id,
+            "kind": kind,
+            "nextIndex": 0,
+            "userFavoriteItemList": [],
+        }
+
+        if not user_id or kind not in (1, 2):
+            return empty_resp
+
         id_list: List[Dict] = []
 
-        if user_id:
-            if kind == 1:
-                fav_music = await self.data.item.get_fav_music(user_id)
-                if fav_music:
-                    for fav in fav_music:
-                        id_list.append({"orderId": fav["orderId"] or 0, "id": fav["musicId"]})
-                        if len(id_list) >= 100: # Lazy but whatever
-                            break
-            
-            elif kind == 2:
-                rivals = await self.data.profile.get_rivals_game(user_id)
-                if rivals:
-                    for rival in rivals:
-                        id_list.append({"orderId": 0, "id": rival["rival"]})
+        if kind == 1:
+            rows = await self.data.item.get_fav_music(
+                user_id, limit=max_ct + 1, offset=next_idx
+            )
+
+            if rows is None:
+                return empty_resp
+
+            for row in rows[:max_ct]:
+                id_list.append({"orderId": row["orderId"] or 0, "id": row["musicId"]})
+        elif kind == 2:
+            rows = await self.data.profile.get_rivals_game(
+                user_id, limit=max_ct + 1, offset=next_idx
+            )
+
+            if rows is None:
+                return empty_resp
+
+            for row in rows[:max_ct]:
+                id_list.append({"orderId": 0, "id": row["rival"]})
+        
+        if rows is None or len(rows) <= max_ct:
+            next_idx = 0
+        else:
+            next_idx += max_ct
 
         return {
             "userId": user_id,
             "kind": kind,
-            "nextIndex": 0,
+            "nextIndex": next_idx,
             "userFavoriteItemList": id_list,
         }
 
@@ -964,5 +1019,4 @@ class Mai2Base:
         """
         return {"userId": data["userId"], "userRecommendSelectionMusicIdList": []}
     async def handle_get_user_score_ranking_api_request(self, data: Dict) ->Dict:
-
-        return {"userId": data["userId"], "userScoreRanking": []}
+        return {"userId": data["userId"], "userScoreRanking": []}

titles/mai2/dx.py

@@ -1,8 +1,9 @@
-from typing import Any, List, Dict
+import itertools
 from datetime import datetime, timedelta
-import pytz
-import json
 from random import randint
+from typing import Any, Dict, List
+
+import pytz
 
 from core.config import CoreConfig
 from core.utils import Utils
@@ -309,83 +310,112 @@ class Mai2DX(Mai2Base):
         return {"userId": data["userId"], "userOption": options_dict}
 
     async def handle_get_user_card_api_request(self, data: Dict) -> Dict:
-        user_cards = await self.data.item.get_cards(data["userId"])
-        if user_cards is None:
-            return {"userId": data["userId"], "nextIndex": 0, "userCardList": []}
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]        
+        
+        rows = await self.data.item.get_cards(user_id, limit=max_ct + 1, offset=next_idx)
+        
+        if rows is None:
+            return {"userId": user_id, "nextIndex": 0, "userCardList": []}
 
-        max_ct = data["maxCount"]
-        next_idx = data["nextIndex"]
-        start_idx = next_idx
-        end_idx = max_ct + start_idx
+        card_list = []
 
-        if len(user_cards[start_idx:]) > max_ct:
+        for row in rows[:max_ct]:
+            card = row._asdict()
+            card.pop("id")
+            card.pop("user")
+            card["startDate"] = datetime.strftime(
+                card["startDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            card["endDate"] = datetime.strftime(
+                card["endDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            card_list.append(card)
+    
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
-        card_list = []
-        for card in user_cards:
-            tmp = card._asdict()
-            tmp.pop("id")
-            tmp.pop("user")
-            tmp["startDate"] = datetime.strftime(
-                tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
-            )
-            tmp["endDate"] = datetime.strftime(
-                tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
-            )
-            card_list.append(tmp)
-
         return {
             "userId": data["userId"],
             "nextIndex": next_idx,
-            "userCardList": card_list[start_idx:end_idx],
+            "userCardList": card_list,
         }
 
     async def handle_get_user_item_api_request(self, data: Dict) -> Dict:
-        kind = data["nextIndex"] // 10000000000
-        next_idx = data["nextIndex"] % 10000000000
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        kind = next_idx // 10000000000
+        next_idx = next_idx % 10000000000
+
         items: List[Dict[str, Any]] = []
         
         if kind == 4: # presents
-            user_pres_list = await self.data.item.get_presents_by_version_user(self.version, data["userId"])
-            if user_pres_list:
-                self.logger.debug(f"Found {len(user_pres_list)} possible presents")
-                for present in user_pres_list:
-                    if (present['startDate'] and present['startDate'].timestamp() > datetime.now().timestamp()):
-                        self.logger.debug(f"Present {present['id']} distribution hasn't started yet (begins {present['startDate']})")
-                        continue # present period hasn't started yet, move onto the next one
-                    
-                    if (present['endDate'] and present['endDate'].timestamp() < datetime.now().timestamp()):
-                        self.logger.warn(f"Present {present['id']} ended on {present['endDate']} and should be removed")
-                        continue # present period ended, move onto the next one
-                    
-                    test = await self.data.item.get_item(data["userId"], present['itemKind'], present['itemId'])
-                    if not test: # Don't send presents for items the user already has
-                        pres_id = present['itemKind'] * 1000000
-                        pres_id += present['itemId']
-                        items.append({"itemId": pres_id, "itemKind": 4, "stock": present['stock'], "isValid": True})
-                        self.logger.info(f"Give user {data['userId']} {present['stock']}x item {present['itemId']} (kind {present['itemKind']}) as present")
+            rows = await self.data.item.get_presents_by_version_user(
+                version=self.version,
+                user_id=user_id,
+                exclude_owned=True,
+                exclude_not_in_present_period=True,
+                limit=max_ct + 1,
+                offset=next_idx,
+            )
 
+            if rows is None:
+                return {
+                    "userId": user_id,
+                    "nextIndex": 0,
+                    "itemKind": kind,
+                    "userItemList": [],
+                }
+            
+            for row in rows[:max_ct]:
+                self.logger.info(
+                    f"Give user {user_id} {row['stock']}x item {row['itemId']} (kind {row['itemKind']}) as present"
+                )
+
+                items.append(
+                    {
+                        "itemId": row["itemKind"] * 1000000 + row["itemId"],
+                        "itemKind": kind,
+                        "stock": row["stock"],
+                        "isValid": True,
+                    }
+                )
         else:
-            user_item_list = await self.data.item.get_items(data["userId"], kind)
-            for i in range(next_idx, len(user_item_list)):
-                tmp = user_item_list[i]._asdict()
-                tmp.pop("user")
-                tmp.pop("id")
-                items.append(tmp)
-                if len(items) >= int(data["maxCount"]):
-                    break
+            rows = await self.data.item.get_items(
+                user_id=user_id,
+                item_kind=kind,
+                limit=max_ct + 1,
+                offset=next_idx,
+            )
 
-        xout = kind * 10000000000 + next_idx + len(items)
+            if rows is None:
+                return {
+                    "userId": user_id,
+                    "nextIndex": 0,
+                    "itemKind": kind,
+                    "userItemList": [],
+                }
 
-        if len(items) < int(data["maxCount"]):
+            for row in rows[:max_ct]:
+                item = row._asdict()
+                
+                item.pop("id")
+                item.pop("user")
+
+                items.append(item)
+
+        if len(rows) > max_ct:
+            next_idx = kind * 10000000000 + next_idx + max_ct
+        else:
             next_idx = 0
-        else:
-            next_idx = xout
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "itemKind": kind,
             "userItemList": items,
@@ -491,103 +521,115 @@ class Mai2DX(Mai2Base):
         return {"length": 0, "userPortraitList": []}
 
     async def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict:
-        friend_season_ranking = await self.data.item.get_friend_season_ranking(data["userId"])
-        if friend_season_ranking is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_friend_season_ranking(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             return {
-                "userId": data["userId"],
+                "userId": user_id,
                 "nextIndex": 0,
                 "userFriendSeasonRankingList": [],
             }
 
         friend_season_ranking_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(friend_season_ranking)):
-            tmp = friend_season_ranking[x]._asdict()
-            tmp.pop("user")
-            tmp.pop("id")
-            tmp["recordDate"] = datetime.strftime(
-                tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
+        for row in rows[:max_ct]:
+            friend_season_ranking = row._asdict()
+            
+            friend_season_ranking.pop("user")
+            friend_season_ranking.pop("id")
+            friend_season_ranking["recordDate"] = datetime.strftime(
+                friend_season_ranking["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
             )
-            friend_season_ranking_list.append(tmp)
+            
+            friend_season_ranking_list.append(friend_season_ranking)
 
-            if len(friend_season_ranking_list) >= max_ct:
-                break
-
-        if len(friend_season_ranking) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userFriendSeasonRankingList": friend_season_ranking_list,
         }
 
     async def handle_get_user_map_api_request(self, data: Dict) -> Dict:
-        maps = await self.data.item.get_maps(data["userId"])
-        if maps is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_maps(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             return {
-                "userId": data["userId"],
+                "userId": user_id,
                 "nextIndex": 0,
                 "userMapList": [],
             }
 
         map_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(maps)):
-            tmp = maps[x]._asdict()
-            tmp.pop("user")
-            tmp.pop("id")
-            map_list.append(tmp)
+        for row in rows[:max_ct]:
+            map = row._asdict()
+            
+            map.pop("user")
+            map.pop("id")
+            
+            map_list.append(map)
 
-            if len(map_list) >= max_ct:
-                break
-
-        if len(maps) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userMapList": map_list,
         }
 
     async def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
-        login_bonuses = await self.data.item.get_login_bonuses(data["userId"])
-        if login_bonuses is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_login_bonuses(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             return {
-                "userId": data["userId"],
+                "userId": user_id,
                 "nextIndex": 0,
                 "userLoginBonusList": [],
             }
 
         login_bonus_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(login_bonuses)):
-            tmp = login_bonuses[x]._asdict()
-            tmp.pop("user")
-            tmp.pop("id")
-            login_bonus_list.append(tmp)
+        for row in rows[:max_ct]:
+            login_bonus = row._asdict()
+            
+            login_bonus.pop("user")
+            login_bonus.pop("id")
+            
+            login_bonus_list.append(login_bonus)
 
-            if len(login_bonus_list) >= max_ct:
-                break
-
-        if len(login_bonuses) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userLoginBonusList": login_bonus_list,
         }
@@ -619,46 +661,62 @@ class Mai2DX(Mai2Base):
         }
 
     async def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict:
-        user_id = data.get("userId", 0)
-        rival_id = data.get("rivalId", 0)
-        next_index = data.get("nextIndex", 0)
-        max_ct = 100
-        upper_lim = next_index + max_ct
-        rival_music_list: Dict[int, List] = {}
+        user_id: int = data["userId"]
+        rival_id: int = data["rivalId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = 100
+        levels: list[int] = [x["level"] for x in data["userRivalMusicLevelList"]]        
 
-        songs = await self.data.score.get_best_scores(rival_id)
-        if songs is None:
+        rows = await self.data.score.get_best_scores(
+            rival_id,
+            is_dx=True,
+            limit=max_ct + 1,
+            offset=next_idx,
+            levels=levels,
+        )
+
+        if rows is None:
             self.logger.debug("handle_get_user_rival_music_api_request: get_best_scores returned None!")
+
             return {
                 "userId": user_id,
                 "rivalId": rival_id,
                 "nextIndex": 0,
                 "userRivalMusicList": [] # musicId userRivalMusicDetailList -> level achievement deluxscoreMax
             }
+        
+        music_details = [x._asdict() for x in rows]
+        returned_count = 0
+        music_list = []
 
-        num_user_songs = len(songs)
+        for music_id, details_iter in itertools.groupby(music_details, key=lambda x: x["musicId"]):
+            details: list[dict[Any, Any]] = []
 
-        for x in range(next_index, upper_lim):
-            if x >= num_user_songs:
+            for d in details_iter:
+                details.append(
+                    {
+                        "level": d["level"],
+                        "achievement": d["achievement"],
+                        "deluxscoreMax": d["deluxscoreMax"],
+                    }
+                )
+
+            music_list.append({"musicId": music_id, "userRivalMusicDetailList": details})
+            returned_count += len(details)
+
+            if len(music_list) >= max_ct:
                 break
 
-            tmp = songs[x]._asdict()
-            if tmp['musicId'] in rival_music_list:
-                rival_music_list[tmp['musicId']].append([{"level": tmp['level'], 'achievement': tmp['achievement'], 'deluxscoreMax': tmp['deluxscoreMax']}])
-            
-            else:
-                if len(rival_music_list) >= max_ct:
-                    break
-                rival_music_list[tmp['musicId']] = [{"level": tmp['level'], 'achievement': tmp['achievement'], 'deluxscoreMax': tmp['deluxscoreMax']}]
-
-        next_index = 0 if len(rival_music_list) < max_ct or num_user_songs == upper_lim else upper_lim
-        self.logger.info(f"Send rival {rival_id} songs {next_index}-{upper_lim} ({len(rival_music_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
+        if returned_count < len(rows):
+            next_idx += max_ct
+        else:
+            next_idx = 0
 
         return {
             "userId": user_id,
             "rivalId": rival_id,
-            "nextIndex": next_index,
-            "userRivalMusicList": [{"musicId": x, "userRivalMusicDetailList": y} for x, y in rival_music_list.items()]
+            "nextIndex": next_idx,
+            "userRivalMusicList": music_list,
         }
 
     async def handle_get_user_new_item_api_request(self, data: Dict) -> Dict:
@@ -674,42 +732,55 @@ class Mai2DX(Mai2Base):
         }
 
     async def handle_get_user_music_api_request(self, data: Dict) -> Dict:
-        user_id = data.get("userId", 0)        
-        next_index = data.get("nextIndex", 0)
-        max_ct = data.get("maxCount", 50)
-        upper_lim = next_index + max_ct
-        music_detail_list = []
+        user_id: int = data.get("userId", 0)        
+        next_idx: int = data.get("nextIndex", 0)
+        max_ct: int = data.get("maxCount", 50)
 
         if user_id <= 0:
             self.logger.warning("handle_get_user_music_api_request: Could not find userid in data, or userId is 0")
             return {}
         
-        songs = await self.data.score.get_best_scores(user_id)
-        if songs is None:
+        rows = await self.data.score.get_best_scores(
+            user_id, is_dx=True, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!")
+
             return {
-            "userId": data["userId"],
-            "nextIndex": 0,
-            "userMusicList": [],
-        }
+                "userId": user_id,
+                "nextIndex": 0,
+                "userMusicList": [],
+            }
 
-        num_user_songs = len(songs)
+        music_details = [row._asdict() for row in rows]
+        returned_count = 0
+        music_list = []
 
-        for x in range(next_index, upper_lim):
-            if num_user_songs <= x:
+        for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]):
+            details: list[dict[Any, Any]] = []
+
+            for d in details_iter:
+                d.pop("id")
+                d.pop("user")
+                
+                details.append(d)
+
+            music_list.append({"userMusicDetailList": details})
+            returned_count += len(details)
+
+            if len(music_list) >= max_ct:
                 break
+        
+        if returned_count < len(rows):
+            next_idx += max_ct
+        else:
+            next_idx = 0
 
-            tmp = songs[x]._asdict()
-            tmp.pop("id")
-            tmp.pop("user")
-            music_detail_list.append(tmp)
-
-        next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim
-        self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
         return {
-            "userId": data["userId"],
-            "nextIndex": next_index,
-            "userMusicList": [{"userMusicDetailList": music_detail_list}],
+            "userId": user_id,
+            "nextIndex": next_idx,
+            "userMusicList": music_list,
         }
 
     async def handle_user_login_api_request(self, data: Dict) -> Dict:
@@ -812,39 +883,43 @@ class Mai2DX(Mai2Base):
         return {"length": len(selling_card_list), "sellingCardList": selling_card_list}
 
     async def handle_cm_get_user_card_api_request(self, data: Dict) -> Dict:
-        user_cards = await self.data.item.get_cards(data["userId"])
-        if user_cards is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_cards(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             return {"returnCode": 1, "length": 0, "nextIndex": 0, "userCardList": []}
 
-        max_ct = data["maxCount"]
-        next_idx = data["nextIndex"]
-        start_idx = next_idx
-        end_idx = max_ct + start_idx
+        card_list = []
 
-        if len(user_cards[start_idx:]) > max_ct:
+        for row in rows[:max_ct]:
+            card = row._asdict()
+            
+            card.pop("id")
+            card.pop("user")
+            card["startDate"] = datetime.strftime(
+                card["startDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            card["endDate"] = datetime.strftime(
+                card["endDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+
+            card_list.append(card)
+        
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
-        card_list = []
-        for card in user_cards:
-            tmp = card._asdict()
-            tmp.pop("id")
-            tmp.pop("user")
-
-            tmp["startDate"] = datetime.strftime(
-                tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
-            )
-            tmp["endDate"] = datetime.strftime(
-                tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
-            )
-            card_list.append(tmp)
-
         return {
             "returnCode": 1,
-            "length": len(card_list[start_idx:end_idx]),
+            "length": len(card_list),
             "nextIndex": next_idx,
-            "userCardList": card_list[start_idx:end_idx],
+            "userCardList": card_list,
         }
 
     async def handle_cm_get_user_item_api_request(self, data: Dict) -> Dict:

titles/mai2/schema/item.py

@@ -1,15 +1,16 @@
-from core.data.schema import BaseData, metadata
-
 from datetime import datetime
-from typing import Optional, Dict, List
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_, or_
-from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BIGINT, INTEGER
-from sqlalchemy.schema import ForeignKey
-from sqlalchemy.sql import func, select
+from typing import Dict, List, Optional
+
+from sqlalchemy import Column, Table, UniqueConstraint, and_, or_
 from sqlalchemy.dialects.mysql import insert
 from sqlalchemy.engine import Row
+from sqlalchemy.schema import ForeignKey
+from sqlalchemy.sql import func, select
+from sqlalchemy.types import BIGINT, INTEGER, JSON, TIMESTAMP, Boolean, Integer, String
 
-character = Table(
+from core.data.schema import BaseData, metadata
+
+character: Table = Table(
     "mai2_item_character",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -27,7 +28,7 @@ character = Table(
     mysql_charset="utf8mb4",
 )
 
-card = Table(
+card: Table = Table(
     "mai2_item_card",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -46,7 +47,7 @@ card = Table(
     mysql_charset="utf8mb4",
 )
 
-item = Table(
+item: Table = Table(
     "mai2_item_item",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -63,7 +64,7 @@ item = Table(
     mysql_charset="utf8mb4",
 )
 
-map = Table(
+map: Table = Table(
     "mai2_item_map",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -81,7 +82,7 @@ map = Table(
     mysql_charset="utf8mb4",
 )
 
-login_bonus = Table(
+login_bonus: Table = Table(
     "mai2_item_login_bonus",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -98,7 +99,7 @@ login_bonus = Table(
     mysql_charset="utf8mb4",
 )
 
-friend_season_ranking = Table(
+friend_season_ranking: Table = Table(
     "mai2_item_friend_season_ranking",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -134,7 +135,7 @@ favorite = Table(
     mysql_charset="utf8mb4",
 )
 
-fav_music = Table(
+fav_music: Table = Table(
     "mai2_item_favorite_music",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -199,7 +200,7 @@ print_detail = Table(
     mysql_charset="utf8mb4",
 )
 
-present = Table(
+present: Table = Table(
     "mai2_item_present",
     metadata,
     Column('id', BIGINT, primary_key=True, nullable=False),
@@ -239,13 +240,26 @@ class Mai2ItemData(BaseData):
             return None
         return result.lastrowid
 
-    async def get_items(self, user_id: int, item_kind: int = None) -> Optional[List[Row]]:
-        if item_kind is None:
-            sql = item.select(item.c.user == user_id)
-        else:
-            sql = item.select(
-                and_(item.c.user == user_id, item.c.itemKind == item_kind)
-            )
+    async def get_items(
+        self,
+        user_id: int,
+        item_kind: Optional[int] = None,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Optional[List[Row]]:
+        cond = item.c.user == user_id
+
+        if item_kind is not None:
+            cond &= item.c.itemKind == item_kind
+
+        sql = select(item).where(cond)
+
+        if limit is not None or offset is not None:
+            sql = sql.order_by(item.c.id)
+        if limit is not None:
+            sql = sql.limit(limit)
+        if offset is not None:
+            sql = sql.offset(offset)
 
         result = await self.execute(sql)
         if result is None:
@@ -296,8 +310,20 @@ class Mai2ItemData(BaseData):
             return None
         return result.lastrowid
 
-    async def get_login_bonuses(self, user_id: int) -> Optional[List[Row]]:
-        sql = login_bonus.select(login_bonus.c.user == user_id)
+    async def get_login_bonuses(
+        self,
+        user_id: int,
+        limit: Optional[int] = None, 
+        offset: Optional[int] = None,
+    ) -> Optional[List[Row]]:
+        sql = select(login_bonus).where(login_bonus.c.user == user_id)
+
+        if limit is not None or offset is not None:
+            sql = sql.order_by(login_bonus.c.id)
+        if limit is not None:
+            sql = sql.limit(limit)
+        if offset is not None:
+            sql = sql.offset(offset)
 
         result = await self.execute(sql)
         if result is None:
@@ -347,8 +373,20 @@ class Mai2ItemData(BaseData):
             return None
         return result.lastrowid
 
-    async def get_maps(self, user_id: int) -> Optional[List[Row]]:
-        sql = map.select(map.c.user == user_id)
+    async def get_maps(
+        self,
+        user_id: int,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Optional[List[Row]]:
+        sql = select(map).where(map.c.user == user_id)
+
+        if limit is not None or offset is not None:
+            sql = sql.order_by(map.c.id)
+        if limit is not None:
+            sql = sql.limit(limit)
+        if offset is not None:
+            sql = sql.offset(offset)
 
         result = await self.execute(sql)
         if result is None:
@@ -424,8 +462,20 @@ class Mai2ItemData(BaseData):
             return None
         return result.fetchone()
 
-    async def get_friend_season_ranking(self, user_id: int) -> Optional[Row]:
-        sql = friend_season_ranking.select(friend_season_ranking.c.user == user_id)
+    async def get_friend_season_ranking(
+        self,
+        user_id: int,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Optional[List[Row]]:
+        sql = select(friend_season_ranking).where(friend_season_ranking.c.user == user_id)
+
+        if limit is not None or offset is not None:
+            sql = sql.order_by(friend_season_ranking.c.id)
+        if limit is not None:
+            sql = sql.limit(limit)
+        if offset is not None:
+            sql = sql.offset(offset)
 
         result = await self.execute(sql)
         if result is None:
@@ -480,8 +530,23 @@ class Mai2ItemData(BaseData):
             return None
         return result.fetchall()
 
-    async def get_fav_music(self, user_id: int) -> Optional[List[Row]]:
-        result = await self.execute(fav_music.select(fav_music.c.user == user_id))
+    async def get_fav_music(
+        self,
+        user_id: int,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Optional[List[Row]]:
+        sql = select(fav_music).where(fav_music.c.user == user_id)
+
+        if limit is not None or offset is not None:
+            sql = sql.order_by(fav_music.c.id)
+        if limit is not None:
+            sql = sql.limit(limit)
+        if offset is not None:
+            sql = sql.offset(offset)
+
+        result = await self.execute(sql)
+
         if result:
             return result.fetchall()
 
@@ -537,13 +602,24 @@ class Mai2ItemData(BaseData):
             return None
         return result.lastrowid
 
-    async def get_cards(self, user_id: int, kind: int = None) -> Optional[Row]:
-        if kind is None:
-            sql = card.select(card.c.user == user_id)
-        else:
-            sql = card.select(and_(card.c.user == user_id, card.c.cardKind == kind))
+    async def get_cards(
+        self,
+        user_id: int,
+        kind: Optional[int] = None,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Optional[List[Row]]:
+        condition = card.c.user == user_id
 
-        sql = sql.order_by(card.c.startDate.desc())
+        if kind is not None:
+            condition &= card.c.cardKind == kind
+
+        sql = select(card).where(condition).order_by(card.c.startDate.desc(), card.c.id.asc())
+
+        if limit is not None:
+            sql = sql.limit(limit)
+        if offset is not None:
+            sql = sql.offset(offset)
 
         result = await self.execute(sql)
         if result is None:
@@ -634,13 +710,46 @@ class Mai2ItemData(BaseData):
         if result:
             return result.fetchall()
 
-    async def get_presents_by_version_user(self, ver: int = None, user_id: int = None) -> Optional[List[Row]]:
-        result = await self.execute(present.select(
-            and_(
-                or_(present.c.user == user_id, present.c.user == None),
-                or_(present.c.version == ver, present.c.version == None)
+    async def get_presents_by_version_user(
+        self,
+        version: Optional[int] = None,
+        user_id: Optional[int] = None,
+        exclude_owned: bool = False,
+        exclude_not_in_present_period: bool = False,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Optional[List[Row]]:
+        sql = select(present)
+        condition = (
+            ((present.c.user == user_id) | present.c.user.is_(None))
+            & ((present.c.version == version) | present.c.version.is_(None))
+        )
+
+        # Do an anti-join with the mai2_item_item table to exclude any
+        # items the users have already owned.
+        if exclude_owned:
+            sql = sql.join(
+                item,
+                (present.c.itemKind == item.c.itemKind)
+                & (present.c.itemId == item.c.itemId)
             )
-        ))
+            condition &= (item.c.itemKind.is_(None) & item.c.itemId.is_(None))
+
+        if exclude_not_in_present_period:
+            condition &= (present.c.startDate.is_(None) | (present.c.startDate <= func.now()))
+            condition &= (present.c.endDate.is_(None) | (present.c.endDate >= func.now()))
+        
+        sql = sql.where(condition)
+
+        if limit is not None or offset is not None:
+            sql = sql.order_by(present.c.id)
+        if limit is not None:
+            sql = sql.limit(limit)
+        if offset is not None:
+            sql = sql.offset(offset)
+
+        result = await self.execute(sql)
+
         if result:
             return result.fetchall()
     

titles/mai2/schema/profile.py

@@ -1,15 +1,26 @@
-from core.data.schema import BaseData, metadata
-from titles.mai2.const import Mai2Constants
+from datetime import datetime
+from typing import Dict, List, Optional
 from uuid import uuid4
 
-from typing import Optional, Dict, List
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
-from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger, VARCHAR, INTEGER
+from sqlalchemy import Column, Table, UniqueConstraint, and_
+from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.engine import Row
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select
-from sqlalchemy.engine import Row
-from sqlalchemy.dialects.mysql import insert
-from datetime import datetime
+from sqlalchemy.types import (
+    INTEGER,
+    JSON,
+    TIMESTAMP,
+    VARCHAR,
+    BigInteger,
+    Boolean,
+    Integer,
+    SmallInteger,
+    String,
+)
+
+from core.data.schema import BaseData, metadata
+from titles.mai2.const import Mai2Constants
 
 detail = Table(
     "mai2_profile_detail",
@@ -495,7 +506,7 @@ consec_logins = Table(
     mysql_charset="utf8mb4",
 )
 
-rival = Table(
+rival: Table = Table(
     "mai2_user_rival",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -908,8 +919,23 @@ class Mai2ProfileData(BaseData):
         if result:
             return result.fetchall()
 
-    async def get_rivals_game(self, user_id: int) -> Optional[List[Row]]:
-        result = await self.execute(rival.select(and_(rival.c.user == user_id, rival.c.show == True)).limit(3))
+    async def get_rivals_game(
+        self,
+        user_id: int,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+    ) -> Optional[List[Row]]:
+        sql = select(rival).where((rival.c.user == user_id) & rival.c.show.is_(True))
+
+        if limit is not None or offset is not None:
+            sql = sql.order_by(rival.c.id)
+        if limit is not None:
+            sql = sql.limit(limit)
+        if offset is not None:
+            sql = sql.offset(offset)
+
+        result = await self.execute(sql)
+
         if result:
             return result.fetchall()
     

titles/mai2/schema/score.py

@@ -1,15 +1,15 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
-from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger
+
+from sqlalchemy import Column, Table, UniqueConstraint, and_
+from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.engine import Row
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select
-from sqlalchemy.engine import Row
-from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.types import JSON, BigInteger, Boolean, Integer, String
 
 from core.data.schema import BaseData, metadata
-from core.data import cached
 
-best_score = Table(
+best_score: Table = Table(
     "mai2_score_best",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -272,7 +272,7 @@ playlog_old = Table(
     mysql_charset="utf8mb4",
 )
 
-best_score_old = Table(
+best_score_old: Table = Table(
     "maimai_score_best",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
@@ -313,22 +313,55 @@ class Mai2ScoreData(BaseData):
             return None
         return result.lastrowid
 
-    @cached(2)
-    async def get_best_scores(self, user_id: int, song_id: int = None, is_dx: bool = True) -> Optional[List[Row]]:
+    async def get_best_scores(
+        self,
+        user_id: int,
+        song_id: Optional[int] = None,
+        is_dx: bool = True,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        levels: Optional[list[int]] = None,
+    ) -> Optional[List[Row]]:
         if is_dx:
-            sql = best_score.select(
-                and_(
-                    best_score.c.user == user_id,
-                    (best_score.c.musicId == song_id) if song_id is not None else True,
-                )
-            ).order_by(best_score.c.musicId).order_by(best_score.c.level)
+            table = best_score
         else:
-            sql = best_score_old.select(
-                and_(
-                    best_score_old.c.user == user_id,
-                    (best_score_old.c.musicId == song_id) if song_id is not None else True,
-                )
-            ).order_by(best_score.c.musicId).order_by(best_score.c.level)
+            table = best_score_old
+
+        cond = table.c.user == user_id
+
+        if song_id is not None:
+            cond &= table.c.musicId == song_id
+        
+        if levels is not None:
+            cond &= table.c.level.in_(levels)
+        
+        if limit is None and offset is None:
+            sql = (
+                select(table)
+                .where(cond)
+                .order_by(table.c.musicId, table.c.level)
+            )
+        else:
+            subq = (
+                select(table.c.musicId)
+                .distinct()
+                .where(cond)
+                .order_by(table.c.musicId)
+            )
+
+            if limit is not None:
+                subq = subq.limit(limit)
+            if offset is not None:
+                subq = subq.offset(offset)
+            
+            subq = subq.subquery()
+
+            sql = (
+                select(table)
+                .join(subq, table.c.musicId == subq.c.musicId)
+                .where(cond)
+                .order_by(table.c.musicId, table.c.level)
+            )
 
         result = await self.execute(sql)
         if result is None: