use SQL's limit/offset pagination for nextIndex/maxCount requests (#185)

Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need.

Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready.

Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client.

### How this works

Say we have a `GetUserCharacterApi` request:

```json
{
    "userId": 10000,
    "maxCount": 700,
    "nextIndex": 0
}
```

Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query:

```python
select(character)
.where(character.c.user == user_id)
.order_by(character.c.id.asc())
.limit(max_count + 1)
.offset(next_index)
```

The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned:

```python
rows = ...

if len(rows) > max_count:
    # return only max_count rows
    next_index += max_count
else:
    # return everything left
    next_index = -1
```

This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](2274b42358/titles/chuni/base.py (L13)).)

Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/185
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>

titles/mai2/base.py

@@ -1,16 +1,17 @@
-from datetime import datetime, timedelta
-from typing import Any, Dict, List
+import itertools
 import logging
 from base64 import b64decode
-from os import path, stat, remove, mkdir, access, W_OK
-from PIL import ImageFile
-from random import randint
+from datetime import datetime, timedelta
+from os import W_OK, access, mkdir, path
+from typing import Any, Dict, List
 
 import pytz
+
 from core.config import CoreConfig
 from core.utils import Utils
-from .const import Mai2Constants
+
 from .config import Mai2Config
+from .const import Mai2Constants
 from .database import Mai2Data
 
 
@@ -444,23 +445,22 @@ class Mai2Base:
         return {"userId": data["userId"], "userOption": options_dict}
 
     async def handle_get_user_card_api_request(self, data: Dict) -> Dict:
-        user_cards = await self.data.item.get_cards(data["userId"])
-        if user_cards is None:
-            return {"userId": data["userId"], "nextIndex": 0, "userCardList": []}
+        user_id = int(data["userId"])
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
 
-        max_ct = data["maxCount"]
-        next_idx = data["nextIndex"]
-        start_idx = next_idx
-        end_idx = max_ct + start_idx
+        user_cards = await self.data.item.get_cards(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
 
-        if len(user_cards[start_idx:]) > max_ct:
-            next_idx += max_ct
-        else:
-            next_idx = 0
+        if user_cards is None or len(user_cards) == 0:
+            return {"userId": user_id, "nextIndex": 0, "userCardList": []}
 
         card_list = []
-        for card in user_cards:
+
+        for card in user_cards[:max_ct]:
             tmp = card._asdict()
+            
             tmp.pop("id")
             tmp.pop("user")
             tmp["startDate"] = datetime.strftime(
@@ -469,12 +469,18 @@ class Mai2Base:
             tmp["endDate"] = datetime.strftime(
                 tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
             )
+            
             card_list.append(tmp)
 
+        if len(user_cards) > max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
-            "userCardList": card_list[start_idx:end_idx],
+            "userCardList": card_list,
         }
 
     async def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
@@ -536,28 +542,35 @@ class Mai2Base:
         return { "userId": data.get("userId", 0), "userBossData": boss_lst}
 
     async def handle_get_user_item_api_request(self, data: Dict) -> Dict:
-        kind = int(data["nextIndex"] / 10000000000)
-        next_idx = int(data["nextIndex"] % 10000000000)
-        user_item_list = await self.data.item.get_items(data["userId"], kind)
+        user_id: int = data["userId"]
+        kind: int = data["nextIndex"] // 10000000000
+        next_idx: int = data["nextIndex"] % 10000000000
+        max_ct: int = data["maxCount"]
+        rows = await self.data.item.get_items(user_id, kind, limit=max_ct, offset=next_idx)
+
+        if rows is None or len(rows) == 0:
+            return {
+                "userId": user_id,
+                "nextIndex": 0,
+                "itemKind": kind,
+                "userItemList": [],
+            }
 
         items: List[Dict[str, Any]] = []
-        for i in range(next_idx, len(user_item_list)):
-            tmp = user_item_list[i]._asdict()
+
+        for row in rows[:max_ct]:
+            tmp = row._asdict()
             tmp.pop("user")
             tmp.pop("id")
             items.append(tmp)
-            if len(items) >= int(data["maxCount"]):
-                break
 
-        xout = kind * 10000000000 + next_idx + len(items)
-
-        if len(items) < int(data["maxCount"]):
-            next_idx = 0
+        if len(rows) > max_ct:
+            next_idx = kind * 10000000000 + next_idx + max_ct
         else:
-            next_idx = xout
+            next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "itemKind": kind,
             "userItemList": items,
@@ -675,77 +688,90 @@ class Mai2Base:
         return {"length": 0, "userPortraitList": []}
 
     async def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict:
-        friend_season_ranking = await self.data.item.get_friend_season_ranking(data["userId"])
-        if friend_season_ranking is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_friend_season_ranking(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             return {
-                "userId": data["userId"],
+                "userId": user_id,
                 "nextIndex": 0,
                 "userFriendSeasonRankingList": [],
             }
 
         friend_season_ranking_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(friend_season_ranking)):
-            tmp = friend_season_ranking[x]._asdict()
-            tmp.pop("user")
+        for row in rows[:max_ct]:
+            tmp = row._asdict()
+            
             tmp.pop("id")
+            tmp.pop("user")
             tmp["recordDate"] = datetime.strftime(
                 tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
             )
+
             friend_season_ranking_list.append(tmp)
 
-            if len(friend_season_ranking_list) >= max_ct:
-                break
-
-        if len(friend_season_ranking) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userFriendSeasonRankingList": friend_season_ranking_list,
         }
 
     async def handle_get_user_map_api_request(self, data: Dict) -> Dict:
-        maps = await self.data.item.get_maps(data["userId"])
-        if maps is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_maps(
+            user_id, limit=max_ct + 1, offset=next_idx,
+        )
+
+        if rows is None:
             return {
-                "userId": data["userId"],
+                "userId": user_id,
                 "nextIndex": 0,
                 "userMapList": [],
             }
 
         map_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(maps)):
-            tmp = maps[x]._asdict()
+        for row in rows[:max_ct]:
+            tmp = row._asdict()
             tmp.pop("user")
             tmp.pop("id")
             map_list.append(tmp)
 
-            if len(map_list) >= max_ct:
-                break
-
-        if len(maps) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userMapList": map_list,
         }
 
     async def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
-        login_bonuses = await self.data.item.get_login_bonuses(data["userId"])
-        if login_bonuses is None:
+        user_id: int = data["userId"]
+        next_idx: int = data["nextIndex"]
+        max_ct: int = data["maxCount"]
+
+        rows = await self.data.item.get_login_bonuses(
+            user_id, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             return {
                 "userId": data["userId"],
                 "nextIndex": 0,
@@ -753,25 +779,20 @@ class Mai2Base:
             }
 
         login_bonus_list = []
-        next_idx = int(data["nextIndex"])
-        max_ct = int(data["maxCount"])
 
-        for x in range(next_idx, len(login_bonuses)):
-            tmp = login_bonuses[x]._asdict()
+        for row in rows[:max_ct]:
+            tmp = row._asdict()
             tmp.pop("user")
             tmp.pop("id")
             login_bonus_list.append(tmp)
 
-            if len(login_bonus_list) >= max_ct:
-                break
-
-        if len(login_bonuses) >= next_idx + max_ct:
+        if len(rows) > max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"],
+            "userId": user_id,
             "nextIndex": next_idx,
             "userLoginBonusList": login_bonus_list,
         }
@@ -805,42 +826,54 @@ class Mai2Base:
         return {"userId": data["userId"], "userGradeStatus": grade_stat, "length": 0, "userGradeList": []}
 
     async def handle_get_user_music_api_request(self, data: Dict) -> Dict:
-        user_id = data.get("userId", 0)        
-        next_index = data.get("nextIndex", 0)
-        max_ct = data.get("maxCount", 50)
-        upper_lim = next_index + max_ct
-        music_detail_list = []
+        user_id: int = data.get("userId", 0)        
+        next_idx: int = data.get("nextIndex", 0)
+        max_ct: int = data.get("maxCount", 50)
 
         if user_id <= 0:
             self.logger.warning("handle_get_user_music_api_request: Could not find userid in data, or userId is 0")
             return {}
         
-        songs = await self.data.score.get_best_scores(user_id, is_dx=False)
-        if songs is None:
+        rows = await self.data.score.get_best_scores(
+            user_id, is_dx=False, limit=max_ct + 1, offset=next_idx
+        )
+
+        if rows is None:
             self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!")
             return {
-            "userId": data["userId"],
-            "nextIndex": 0,
-            "userMusicList": [],
-        }
+                "userId": user_id,
+                "nextIndex": 0,
+                "userMusicList": [],
+            }
 
-        num_user_songs = len(songs)
+        music_details = [row._asdict() for row in rows]
+        returned_count = 0
+        music_list = []
 
-        for x in range(next_index, upper_lim):
-            if num_user_songs <= x:
+        for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]):
+            details: list[dict[Any, Any]] = []
+
+            for d in details_iter:
+                d.pop("id")
+                d.pop("user")
+                
+                details.append(d)
+
+            music_list.append({"userMusicDetailList": details})
+            returned_count += len(details)
+
+            if len(music_list) >= max_ct:
                 break
+        
+        if returned_count < len(rows):
+            next_idx += max_ct
+        else:
+            next_idx = 0
 
-            tmp = songs[x]._asdict()
-            tmp.pop("id")
-            tmp.pop("user")
-            music_detail_list.append(tmp)
-
-        next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim
-        self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
         return {
-            "userId": data["userId"],
-            "nextIndex": next_index,
-            "userMusicList": [{"userMusicDetailList": music_detail_list}],
+            "userId": user_id,
+            "nextIndex": next_idx,
+            "userMusicList": music_list,
         }
 
     async def handle_upload_user_portrait_api_request(self, data: Dict) -> Dict:
@@ -925,30 +958,52 @@ class Mai2Base:
     async def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
         user_id = data.get("userId", 0)
         kind = data.get("kind", 0) # 1 is fav music, 2 is rival user IDs
-        next_index = data.get("nextIndex", 0)
+        next_idx = data.get("nextIndex", 0)
         max_ct = data.get("maxCount", 100) # always 100
         is_all = data.get("isAllFavoriteItem", False) # always false
+
+        empty_resp = {
+            "userId": user_id,
+            "kind": kind,
+            "nextIndex": 0,
+            "userFavoriteItemList": [],
+        }
+
+        if not user_id or kind not in (1, 2):
+            return empty_resp
+
         id_list: List[Dict] = []
 
-        if user_id:
-            if kind == 1:
-                fav_music = await self.data.item.get_fav_music(user_id)
-                if fav_music:
-                    for fav in fav_music:
-                        id_list.append({"orderId": fav["orderId"] or 0, "id": fav["musicId"]})
-                        if len(id_list) >= 100: # Lazy but whatever
-                            break
-            
-            elif kind == 2:
-                rivals = await self.data.profile.get_rivals_game(user_id)
-                if rivals:
-                    for rival in rivals:
-                        id_list.append({"orderId": 0, "id": rival["rival"]})
+        if kind == 1:
+            rows = await self.data.item.get_fav_music(
+                user_id, limit=max_ct + 1, offset=next_idx
+            )
+
+            if rows is None:
+                return empty_resp
+
+            for row in rows[:max_ct]:
+                id_list.append({"orderId": row["orderId"] or 0, "id": row["musicId"]})
+        elif kind == 2:
+            rows = await self.data.profile.get_rivals_game(
+                user_id, limit=max_ct + 1, offset=next_idx
+            )
+
+            if rows is None:
+                return empty_resp
+
+            for row in rows[:max_ct]:
+                id_list.append({"orderId": 0, "id": row["rival"]})
+        
+        if rows is None or len(rows) <= max_ct:
+            next_idx = 0
+        else:
+            next_idx += max_ct
 
         return {
             "userId": user_id,
             "kind": kind,
-            "nextIndex": 0,
+            "nextIndex": next_idx,
             "userFavoriteItemList": id_list,
         }
 
@@ -964,5 +1019,4 @@ class Mai2Base:
         """
         return {"userId": data["userId"], "userRecommendSelectionMusicIdList": []}
     async def handle_get_user_score_ranking_api_request(self, data: Dict) ->Dict:
-
-        return {"userId": data["userId"], "userScoreRanking": []}
+        return {"userId": data["userId"], "userScoreRanking": []}