use SQL's limit/offset pagination for nextIndex/maxCount requests (#185)

Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need.

Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready.

Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client.

### How this works

Say we have a `GetUserCharacterApi` request:

```json
{
    "userId": 10000,
    "maxCount": 700,
    "nextIndex": 0
}
```

Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query:

```python
select(character)
.where(character.c.user == user_id)
.order_by(character.c.id.asc())
.limit(max_count + 1)
.offset(next_index)
```

The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned:

```python
rows = ...

if len(rows) > max_count:
    # return only max_count rows
    next_index += max_count
else:
    # return everything left
    next_index = -1
```

This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](2274b42358/titles/chuni/base.py (L13)).)

Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/185
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>
This commit is contained in:
beerpsi
2024-11-16 19:10:29 +00:00
committed by Hay1tsme
parent cb009f6e23
commit 58a5177a30
18 changed files with 1410 additions and 713 deletions

View File

@@ -1,8 +1,9 @@
from typing import Any, List, Dict
import itertools
from datetime import datetime, timedelta
import pytz
import json
from random import randint
from typing import Any, Dict, List
import pytz
from core.config import CoreConfig
from core.utils import Utils
@@ -309,83 +310,112 @@ class Mai2DX(Mai2Base):
return {"userId": data["userId"], "userOption": options_dict}
async def handle_get_user_card_api_request(self, data: Dict) -> Dict:
user_cards = await self.data.item.get_cards(data["userId"])
if user_cards is None:
return {"userId": data["userId"], "nextIndex": 0, "userCardList": []}
user_id: int = data["userId"]
next_idx: int = data["nextIndex"]
max_ct: int = data["maxCount"]
rows = await self.data.item.get_cards(user_id, limit=max_ct + 1, offset=next_idx)
if rows is None:
return {"userId": user_id, "nextIndex": 0, "userCardList": []}
max_ct = data["maxCount"]
next_idx = data["nextIndex"]
start_idx = next_idx
end_idx = max_ct + start_idx
card_list = []
if len(user_cards[start_idx:]) > max_ct:
for row in rows[:max_ct]:
card = row._asdict()
card.pop("id")
card.pop("user")
card["startDate"] = datetime.strftime(
card["startDate"], Mai2Constants.DATE_TIME_FORMAT
)
card["endDate"] = datetime.strftime(
card["endDate"], Mai2Constants.DATE_TIME_FORMAT
)
card_list.append(card)
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = 0
card_list = []
for card in user_cards:
tmp = card._asdict()
tmp.pop("id")
tmp.pop("user")
tmp["startDate"] = datetime.strftime(
tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
)
tmp["endDate"] = datetime.strftime(
tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
)
card_list.append(tmp)
return {
"userId": data["userId"],
"nextIndex": next_idx,
"userCardList": card_list[start_idx:end_idx],
"userCardList": card_list,
}
async def handle_get_user_item_api_request(self, data: Dict) -> Dict:
kind = data["nextIndex"] // 10000000000
next_idx = data["nextIndex"] % 10000000000
user_id: int = data["userId"]
next_idx: int = data["nextIndex"]
max_ct: int = data["maxCount"]
kind = next_idx // 10000000000
next_idx = next_idx % 10000000000
items: List[Dict[str, Any]] = []
if kind == 4: # presents
user_pres_list = await self.data.item.get_presents_by_version_user(self.version, data["userId"])
if user_pres_list:
self.logger.debug(f"Found {len(user_pres_list)} possible presents")
for present in user_pres_list:
if (present['startDate'] and present['startDate'].timestamp() > datetime.now().timestamp()):
self.logger.debug(f"Present {present['id']} distribution hasn't started yet (begins {present['startDate']})")
continue # present period hasn't started yet, move onto the next one
if (present['endDate'] and present['endDate'].timestamp() < datetime.now().timestamp()):
self.logger.warn(f"Present {present['id']} ended on {present['endDate']} and should be removed")
continue # present period ended, move onto the next one
test = await self.data.item.get_item(data["userId"], present['itemKind'], present['itemId'])
if not test: # Don't send presents for items the user already has
pres_id = present['itemKind'] * 1000000
pres_id += present['itemId']
items.append({"itemId": pres_id, "itemKind": 4, "stock": present['stock'], "isValid": True})
self.logger.info(f"Give user {data['userId']} {present['stock']}x item {present['itemId']} (kind {present['itemKind']}) as present")
rows = await self.data.item.get_presents_by_version_user(
version=self.version,
user_id=user_id,
exclude_owned=True,
exclude_not_in_present_period=True,
limit=max_ct + 1,
offset=next_idx,
)
if rows is None:
return {
"userId": user_id,
"nextIndex": 0,
"itemKind": kind,
"userItemList": [],
}
for row in rows[:max_ct]:
self.logger.info(
f"Give user {user_id} {row['stock']}x item {row['itemId']} (kind {row['itemKind']}) as present"
)
items.append(
{
"itemId": row["itemKind"] * 1000000 + row["itemId"],
"itemKind": kind,
"stock": row["stock"],
"isValid": True,
}
)
else:
user_item_list = await self.data.item.get_items(data["userId"], kind)
for i in range(next_idx, len(user_item_list)):
tmp = user_item_list[i]._asdict()
tmp.pop("user")
tmp.pop("id")
items.append(tmp)
if len(items) >= int(data["maxCount"]):
break
rows = await self.data.item.get_items(
user_id=user_id,
item_kind=kind,
limit=max_ct + 1,
offset=next_idx,
)
xout = kind * 10000000000 + next_idx + len(items)
if rows is None:
return {
"userId": user_id,
"nextIndex": 0,
"itemKind": kind,
"userItemList": [],
}
if len(items) < int(data["maxCount"]):
for row in rows[:max_ct]:
item = row._asdict()
item.pop("id")
item.pop("user")
items.append(item)
if len(rows) > max_ct:
next_idx = kind * 10000000000 + next_idx + max_ct
else:
next_idx = 0
else:
next_idx = xout
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": next_idx,
"itemKind": kind,
"userItemList": items,
@@ -491,103 +521,115 @@ class Mai2DX(Mai2Base):
return {"length": 0, "userPortraitList": []}
async def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict:
friend_season_ranking = await self.data.item.get_friend_season_ranking(data["userId"])
if friend_season_ranking is None:
user_id: int = data["userId"]
next_idx: int = data["nextIndex"]
max_ct: int = data["maxCount"]
rows = await self.data.item.get_friend_season_ranking(
user_id, limit=max_ct + 1, offset=next_idx
)
if rows is None:
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": 0,
"userFriendSeasonRankingList": [],
}
friend_season_ranking_list = []
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
for x in range(next_idx, len(friend_season_ranking)):
tmp = friend_season_ranking[x]._asdict()
tmp.pop("user")
tmp.pop("id")
tmp["recordDate"] = datetime.strftime(
tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
for row in rows[:max_ct]:
friend_season_ranking = row._asdict()
friend_season_ranking.pop("user")
friend_season_ranking.pop("id")
friend_season_ranking["recordDate"] = datetime.strftime(
friend_season_ranking["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
)
friend_season_ranking_list.append(tmp)
friend_season_ranking_list.append(friend_season_ranking)
if len(friend_season_ranking_list) >= max_ct:
break
if len(friend_season_ranking) >= next_idx + max_ct:
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = 0
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": next_idx,
"userFriendSeasonRankingList": friend_season_ranking_list,
}
async def handle_get_user_map_api_request(self, data: Dict) -> Dict:
maps = await self.data.item.get_maps(data["userId"])
if maps is None:
user_id: int = data["userId"]
next_idx: int = data["nextIndex"]
max_ct: int = data["maxCount"]
rows = await self.data.item.get_maps(
user_id, limit=max_ct + 1, offset=next_idx
)
if rows is None:
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": 0,
"userMapList": [],
}
map_list = []
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
for x in range(next_idx, len(maps)):
tmp = maps[x]._asdict()
tmp.pop("user")
tmp.pop("id")
map_list.append(tmp)
for row in rows[:max_ct]:
map = row._asdict()
map.pop("user")
map.pop("id")
map_list.append(map)
if len(map_list) >= max_ct:
break
if len(maps) >= next_idx + max_ct:
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = 0
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": next_idx,
"userMapList": map_list,
}
async def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
login_bonuses = await self.data.item.get_login_bonuses(data["userId"])
if login_bonuses is None:
user_id: int = data["userId"]
next_idx: int = data["nextIndex"]
max_ct: int = data["maxCount"]
rows = await self.data.item.get_login_bonuses(
user_id, limit=max_ct + 1, offset=next_idx
)
if rows is None:
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": 0,
"userLoginBonusList": [],
}
login_bonus_list = []
next_idx = int(data["nextIndex"])
max_ct = int(data["maxCount"])
for x in range(next_idx, len(login_bonuses)):
tmp = login_bonuses[x]._asdict()
tmp.pop("user")
tmp.pop("id")
login_bonus_list.append(tmp)
for row in rows[:max_ct]:
login_bonus = row._asdict()
login_bonus.pop("user")
login_bonus.pop("id")
login_bonus_list.append(login_bonus)
if len(login_bonus_list) >= max_ct:
break
if len(login_bonuses) >= next_idx + max_ct:
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = 0
return {
"userId": data["userId"],
"userId": user_id,
"nextIndex": next_idx,
"userLoginBonusList": login_bonus_list,
}
@@ -619,46 +661,62 @@ class Mai2DX(Mai2Base):
}
async def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict:
user_id = data.get("userId", 0)
rival_id = data.get("rivalId", 0)
next_index = data.get("nextIndex", 0)
max_ct = 100
upper_lim = next_index + max_ct
rival_music_list: Dict[int, List] = {}
user_id: int = data["userId"]
rival_id: int = data["rivalId"]
next_idx: int = data["nextIndex"]
max_ct: int = 100
levels: list[int] = [x["level"] for x in data["userRivalMusicLevelList"]]
songs = await self.data.score.get_best_scores(rival_id)
if songs is None:
rows = await self.data.score.get_best_scores(
rival_id,
is_dx=True,
limit=max_ct + 1,
offset=next_idx,
levels=levels,
)
if rows is None:
self.logger.debug("handle_get_user_rival_music_api_request: get_best_scores returned None!")
return {
"userId": user_id,
"rivalId": rival_id,
"nextIndex": 0,
"userRivalMusicList": [] # musicId userRivalMusicDetailList -> level achievement deluxscoreMax
}
music_details = [x._asdict() for x in rows]
returned_count = 0
music_list = []
num_user_songs = len(songs)
for music_id, details_iter in itertools.groupby(music_details, key=lambda x: x["musicId"]):
details: list[dict[Any, Any]] = []
for x in range(next_index, upper_lim):
if x >= num_user_songs:
for d in details_iter:
details.append(
{
"level": d["level"],
"achievement": d["achievement"],
"deluxscoreMax": d["deluxscoreMax"],
}
)
music_list.append({"musicId": music_id, "userRivalMusicDetailList": details})
returned_count += len(details)
if len(music_list) >= max_ct:
break
tmp = songs[x]._asdict()
if tmp['musicId'] in rival_music_list:
rival_music_list[tmp['musicId']].append([{"level": tmp['level'], 'achievement': tmp['achievement'], 'deluxscoreMax': tmp['deluxscoreMax']}])
else:
if len(rival_music_list) >= max_ct:
break
rival_music_list[tmp['musicId']] = [{"level": tmp['level'], 'achievement': tmp['achievement'], 'deluxscoreMax': tmp['deluxscoreMax']}]
next_index = 0 if len(rival_music_list) < max_ct or num_user_songs == upper_lim else upper_lim
self.logger.info(f"Send rival {rival_id} songs {next_index}-{upper_lim} ({len(rival_music_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
if returned_count < len(rows):
next_idx += max_ct
else:
next_idx = 0
return {
"userId": user_id,
"rivalId": rival_id,
"nextIndex": next_index,
"userRivalMusicList": [{"musicId": x, "userRivalMusicDetailList": y} for x, y in rival_music_list.items()]
"nextIndex": next_idx,
"userRivalMusicList": music_list,
}
async def handle_get_user_new_item_api_request(self, data: Dict) -> Dict:
@@ -674,42 +732,55 @@ class Mai2DX(Mai2Base):
}
async def handle_get_user_music_api_request(self, data: Dict) -> Dict:
user_id = data.get("userId", 0)
next_index = data.get("nextIndex", 0)
max_ct = data.get("maxCount", 50)
upper_lim = next_index + max_ct
music_detail_list = []
user_id: int = data.get("userId", 0)
next_idx: int = data.get("nextIndex", 0)
max_ct: int = data.get("maxCount", 50)
if user_id <= 0:
self.logger.warning("handle_get_user_music_api_request: Could not find userid in data, or userId is 0")
return {}
songs = await self.data.score.get_best_scores(user_id)
if songs is None:
rows = await self.data.score.get_best_scores(
user_id, is_dx=True, limit=max_ct + 1, offset=next_idx
)
if rows is None:
self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!")
return {
"userId": data["userId"],
"nextIndex": 0,
"userMusicList": [],
}
"userId": user_id,
"nextIndex": 0,
"userMusicList": [],
}
num_user_songs = len(songs)
music_details = [row._asdict() for row in rows]
returned_count = 0
music_list = []
for x in range(next_index, upper_lim):
if num_user_songs <= x:
for _music_id, details_iter in itertools.groupby(music_details, key=lambda d: d["musicId"]):
details: list[dict[Any, Any]] = []
for d in details_iter:
d.pop("id")
d.pop("user")
details.append(d)
music_list.append({"userMusicDetailList": details})
returned_count += len(details)
if len(music_list) >= max_ct:
break
if returned_count < len(rows):
next_idx += max_ct
else:
next_idx = 0
tmp = songs[x]._asdict()
tmp.pop("id")
tmp.pop("user")
music_detail_list.append(tmp)
next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim
self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
return {
"userId": data["userId"],
"nextIndex": next_index,
"userMusicList": [{"userMusicDetailList": music_detail_list}],
"userId": user_id,
"nextIndex": next_idx,
"userMusicList": music_list,
}
async def handle_user_login_api_request(self, data: Dict) -> Dict:
@@ -812,39 +883,43 @@ class Mai2DX(Mai2Base):
return {"length": len(selling_card_list), "sellingCardList": selling_card_list}
async def handle_cm_get_user_card_api_request(self, data: Dict) -> Dict:
user_cards = await self.data.item.get_cards(data["userId"])
if user_cards is None:
user_id: int = data["userId"]
next_idx: int = data["nextIndex"]
max_ct: int = data["maxCount"]
rows = await self.data.item.get_cards(
user_id, limit=max_ct + 1, offset=next_idx
)
if rows is None:
return {"returnCode": 1, "length": 0, "nextIndex": 0, "userCardList": []}
max_ct = data["maxCount"]
next_idx = data["nextIndex"]
start_idx = next_idx
end_idx = max_ct + start_idx
card_list = []
if len(user_cards[start_idx:]) > max_ct:
for row in rows[:max_ct]:
card = row._asdict()
card.pop("id")
card.pop("user")
card["startDate"] = datetime.strftime(
card["startDate"], Mai2Constants.DATE_TIME_FORMAT
)
card["endDate"] = datetime.strftime(
card["endDate"], Mai2Constants.DATE_TIME_FORMAT
)
card_list.append(card)
if len(rows) > max_ct:
next_idx += max_ct
else:
next_idx = 0
card_list = []
for card in user_cards:
tmp = card._asdict()
tmp.pop("id")
tmp.pop("user")
tmp["startDate"] = datetime.strftime(
tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
)
tmp["endDate"] = datetime.strftime(
tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
)
card_list.append(tmp)
return {
"returnCode": 1,
"length": len(card_list[start_idx:end_idx]),
"length": len(card_list),
"nextIndex": next_idx,
"userCardList": card_list[start_idx:end_idx],
"userCardList": card_list,
}
async def handle_cm_get_user_item_api_request(self, data: Dict) -> Dict: