Bennett/artemis
1
0
Fork 0
mirror of https://gitea.tendokyu.moe/Hay1tsme/artemis.git synced 2026-02-14 11:47:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

use SQL's limit/offset pagination for nextIndex/maxCount requests (#185)

Browse Source 
Instead of retrieving the entire list of items/characters/scores/etc. at once (and even store them in memory), use SQL's `LIMIT ... OFFSET ...` pagination so we only take what we need.

Currently only CHUNITHM uses this, but this will also affect maimai DX and O.N.G.E.K.I. once the PR is ready.

Also snuck in a fix for CHUNITHM/maimai DX's `GetUserRivalMusicApi` to respect the `userRivalMusicLevelList` sent by the client.

### How this works

Say we have a `GetUserCharacterApi` request:

```json
{
    "userId": 10000,
    "maxCount": 700,
    "nextIndex": 0
}
```

Instead of getting the entire character list from the database (which can be very large if the user force unlocked everything), add limit/offset to the query:

```python
select(character)
.where(character.c.user == user_id)
.order_by(character.c.id.asc())
.limit(max_count + 1)
.offset(next_index)
```

The query takes `maxCount + 1` items from the database to determine if there is more items than can be returned:

```python
rows = ...

if len(rows) > max_count:
    # return only max_count rows
    next_index += max_count
else:
    # return everything left
    next_index = -1
```

This has the benefit of not needing to load everything into memory (and also having to store server state, as seen in the [`SCORE_BUFFER` list](2274b42358/titles/chuni/base.py (L13)).)

Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/185
Co-authored-by: beerpsi <beerpsi@duck.com>
Co-committed-by: beerpsi <beerpsi@duck.com>
This commit is contained in:
beerpsi
2024-11-16 19:10:29 +00:00
committed by Hay1tsme
parent cb009f6e23
commit 58a5177a30
18 changed files with 1410 additions and 713 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
titles/mai2/schema/item.py
View File

@@ -1,15 +1,16 @@
from core.data.schema import BaseData, metadata
from datetime import datetime
from typing import Optional, Dict, List
from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_, or_
from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BIGINT, INTEGER
from sqlalchemy.schema import ForeignKey
from sqlalchemy.sql import func, select
from typing import Dict, List, Optional
from sqlalchemy import Column, Table, UniqueConstraint, and_, or_
from sqlalchemy.dialects.mysql import insert
from sqlalchemy.engine import Row
from sqlalchemy.schema import ForeignKey
from sqlalchemy.sql import func, select
from sqlalchemy.types import BIGINT, INTEGER, JSON, TIMESTAMP, Boolean, Integer, String
character = Table(
from core.data.schema import BaseData, metadata
character: Table = Table(
"mai2_item_character",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -27,7 +28,7 @@ character = Table(
mysql_charset="utf8mb4",
)
card = Table(
card: Table = Table(
"mai2_item_card",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -46,7 +47,7 @@ card = Table(
mysql_charset="utf8mb4",
)
item = Table(
item: Table = Table(
"mai2_item_item",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -63,7 +64,7 @@ item = Table(
mysql_charset="utf8mb4",
)
map = Table(
map: Table = Table(
"mai2_item_map",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -81,7 +82,7 @@ map = Table(
mysql_charset="utf8mb4",
)
login_bonus = Table(
login_bonus: Table = Table(
"mai2_item_login_bonus",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -98,7 +99,7 @@ login_bonus = Table(
mysql_charset="utf8mb4",
)
friend_season_ranking = Table(
friend_season_ranking: Table = Table(
"mai2_item_friend_season_ranking",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -134,7 +135,7 @@ favorite = Table(
mysql_charset="utf8mb4",
)
fav_music = Table(
fav_music: Table = Table(
"mai2_item_favorite_music",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -199,7 +200,7 @@ print_detail = Table(
mysql_charset="utf8mb4",
)
present = Table(
present: Table = Table(
"mai2_item_present",
metadata,
Column('id', BIGINT, primary_key=True, nullable=False),
@@ -239,13 +240,26 @@ class Mai2ItemData(BaseData):
return None
return result.lastrowid
async def get_items(self, user_id: int, item_kind: int = None) -> Optional[List[Row]]:
if item_kind is None:
sql = item.select(item.c.user == user_id)
else:
sql = item.select(
and_(item.c.user == user_id, item.c.itemKind == item_kind)
)
async def get_items(
self,
user_id: int,
item_kind: Optional[int] = None,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Optional[List[Row]]:
cond = item.c.user == user_id
if item_kind is not None:
cond &= item.c.itemKind == item_kind
sql = select(item).where(cond)
if limit is not None or offset is not None:
sql = sql.order_by(item.c.id)
if limit is not None:
sql = sql.limit(limit)
if offset is not None:
sql = sql.offset(offset)
result = await self.execute(sql)
if result is None:
@@ -296,8 +310,20 @@ class Mai2ItemData(BaseData):
return None
return result.lastrowid
async def get_login_bonuses(self, user_id: int) -> Optional[List[Row]]:
sql = login_bonus.select(login_bonus.c.user == user_id)
async def get_login_bonuses(
self,
user_id: int,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Optional[List[Row]]:
sql = select(login_bonus).where(login_bonus.c.user == user_id)
if limit is not None or offset is not None:
sql = sql.order_by(login_bonus.c.id)
if limit is not None:
sql = sql.limit(limit)
if offset is not None:
sql = sql.offset(offset)
result = await self.execute(sql)
if result is None:
@@ -347,8 +373,20 @@ class Mai2ItemData(BaseData):
return None
return result.lastrowid
async def get_maps(self, user_id: int) -> Optional[List[Row]]:
sql = map.select(map.c.user == user_id)
async def get_maps(
self,
user_id: int,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Optional[List[Row]]:
sql = select(map).where(map.c.user == user_id)
if limit is not None or offset is not None:
sql = sql.order_by(map.c.id)
if limit is not None:
sql = sql.limit(limit)
if offset is not None:
sql = sql.offset(offset)
result = await self.execute(sql)
if result is None:
@@ -424,8 +462,20 @@ class Mai2ItemData(BaseData):
return None
return result.fetchone()
async def get_friend_season_ranking(self, user_id: int) -> Optional[Row]:
sql = friend_season_ranking.select(friend_season_ranking.c.user == user_id)
async def get_friend_season_ranking(
self,
user_id: int,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Optional[List[Row]]:
sql = select(friend_season_ranking).where(friend_season_ranking.c.user == user_id)
if limit is not None or offset is not None:
sql = sql.order_by(friend_season_ranking.c.id)
if limit is not None:
sql = sql.limit(limit)
if offset is not None:
sql = sql.offset(offset)
result = await self.execute(sql)
if result is None:
@@ -480,8 +530,23 @@ class Mai2ItemData(BaseData):
return None
return result.fetchall()
async def get_fav_music(self, user_id: int) -> Optional[List[Row]]:
result = await self.execute(fav_music.select(fav_music.c.user == user_id))
async def get_fav_music(
self,
user_id: int,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Optional[List[Row]]:
sql = select(fav_music).where(fav_music.c.user == user_id)
if limit is not None or offset is not None:
sql = sql.order_by(fav_music.c.id)
if limit is not None:
sql = sql.limit(limit)
if offset is not None:
sql = sql.offset(offset)
result = await self.execute(sql)
if result:
return result.fetchall()
@@ -537,13 +602,24 @@ class Mai2ItemData(BaseData):
return None
return result.lastrowid
async def get_cards(self, user_id: int, kind: int = None) -> Optional[Row]:
if kind is None:
sql = card.select(card.c.user == user_id)
else:
sql = card.select(and_(card.c.user == user_id, card.c.cardKind == kind))
async def get_cards(
self,
user_id: int,
kind: Optional[int] = None,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Optional[List[Row]]:
condition = card.c.user == user_id
sql = sql.order_by(card.c.startDate.desc())
if kind is not None:
condition &= card.c.cardKind == kind
sql = select(card).where(condition).order_by(card.c.startDate.desc(), card.c.id.asc())
if limit is not None:
sql = sql.limit(limit)
if offset is not None:
sql = sql.offset(offset)
result = await self.execute(sql)
if result is None:
@@ -634,13 +710,46 @@ class Mai2ItemData(BaseData):
if result:
return result.fetchall()
async def get_presents_by_version_user(self, ver: int = None, user_id: int = None) -> Optional[List[Row]]:
result = await self.execute(present.select(
and_(
or_(present.c.user == user_id, present.c.user == None),
or_(present.c.version == ver, present.c.version == None)
async def get_presents_by_version_user(
self,
version: Optional[int] = None,
user_id: Optional[int] = None,
exclude_owned: bool = False,
exclude_not_in_present_period: bool = False,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Optional[List[Row]]:
sql = select(present)
condition = (
((present.c.user == user_id) | present.c.user.is_(None))
& ((present.c.version == version) | present.c.version.is_(None))
)
# Do an anti-join with the mai2_item_item table to exclude any
# items the users have already owned.
if exclude_owned:
sql = sql.join(
item,
(present.c.itemKind == item.c.itemKind)
& (present.c.itemId == item.c.itemId)
)
))
condition &= (item.c.itemKind.is_(None) & item.c.itemId.is_(None))
if exclude_not_in_present_period:
condition &= (present.c.startDate.is_(None) | (present.c.startDate <= func.now()))
condition &= (present.c.endDate.is_(None) | (present.c.endDate >= func.now()))
sql = sql.where(condition)
if limit is not None or offset is not None:
sql = sql.order_by(present.c.id)
if limit is not None:
sql = sql.limit(limit)
if offset is not None:
sql = sql.offset(offset)
result = await self.execute(sql)
if result:
return result.fetchall()

48
titles/mai2/schema/profile.py
View File

@@ -1,15 +1,26 @@
from core.data.schema import BaseData, metadata
from titles.mai2.const import Mai2Constants
from datetime import datetime
from typing import Dict, List, Optional
from uuid import uuid4
from typing import Optional, Dict, List
from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger, SmallInteger, VARCHAR, INTEGER
from sqlalchemy import Column, Table, UniqueConstraint, and_
from sqlalchemy.dialects.mysql import insert
from sqlalchemy.engine import Row
from sqlalchemy.schema import ForeignKey
from sqlalchemy.sql import func, select
from sqlalchemy.engine import Row
from sqlalchemy.dialects.mysql import insert
from datetime import datetime
from sqlalchemy.types import (
INTEGER,
JSON,
TIMESTAMP,
VARCHAR,
BigInteger,
Boolean,
Integer,
SmallInteger,
String,
)
from core.data.schema import BaseData, metadata
from titles.mai2.const import Mai2Constants
detail = Table(
"mai2_profile_detail",
@@ -495,7 +506,7 @@ consec_logins = Table(
mysql_charset="utf8mb4",
)
rival = Table(
rival: Table = Table(
"mai2_user_rival",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -908,8 +919,23 @@ class Mai2ProfileData(BaseData):
if result:
return result.fetchall()
async def get_rivals_game(self, user_id: int) -> Optional[List[Row]]:
result = await self.execute(rival.select(and_(rival.c.user == user_id, rival.c.show == True)).limit(3))
async def get_rivals_game(
self,
user_id: int,
limit: Optional[int] = None,
offset: Optional[int] = None,
) -> Optional[List[Row]]:
sql = select(rival).where((rival.c.user == user_id) & rival.c.show.is_(True))
if limit is not None or offset is not None:
sql = sql.order_by(rival.c.id)
if limit is not None:
sql = sql.limit(limit)
if offset is not None:
sql = sql.offset(offset)
result = await self.execute(sql)
if result:
return result.fetchall()

75
titles/mai2/schema/score.py
View File

@@ -1,15 +1,15 @@
from typing import Dict, List, Optional
from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger
from sqlalchemy import Column, Table, UniqueConstraint, and_
from sqlalchemy.dialects.mysql import insert
from sqlalchemy.engine import Row
from sqlalchemy.schema import ForeignKey
from sqlalchemy.sql import func, select
from sqlalchemy.engine import Row
from sqlalchemy.dialects.mysql import insert
from sqlalchemy.types import JSON, BigInteger, Boolean, Integer, String
from core.data.schema import BaseData, metadata
from core.data import cached
best_score = Table(
best_score: Table = Table(
"mai2_score_best",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -272,7 +272,7 @@ playlog_old = Table(
mysql_charset="utf8mb4",
)
best_score_old = Table(
best_score_old: Table = Table(
"maimai_score_best",
metadata,
Column("id", Integer, primary_key=True, nullable=False),
@@ -313,22 +313,55 @@ class Mai2ScoreData(BaseData):
return None
return result.lastrowid
@cached(2)
async def get_best_scores(self, user_id: int, song_id: int = None, is_dx: bool = True) -> Optional[List[Row]]:
async def get_best_scores(
self,
user_id: int,
song_id: Optional[int] = None,
is_dx: bool = True,
limit: Optional[int] = None,
offset: Optional[int] = None,
levels: Optional[list[int]] = None,
) -> Optional[List[Row]]:
if is_dx:
sql = best_score.select(
and_(
best_score.c.user == user_id,
(best_score.c.musicId == song_id) if song_id is not None else True,
)
).order_by(best_score.c.musicId).order_by(best_score.c.level)
table = best_score
else:
sql = best_score_old.select(
and_(
best_score_old.c.user == user_id,
(best_score_old.c.musicId == song_id) if song_id is not None else True,
)
).order_by(best_score.c.musicId).order_by(best_score.c.level)
table = best_score_old
cond = table.c.user == user_id
if song_id is not None:
cond &= table.c.musicId == song_id
if levels is not None:
cond &= table.c.level.in_(levels)
if limit is None and offset is None:
sql = (
select(table)
.where(cond)
.order_by(table.c.musicId, table.c.level)
)
else:
subq = (
select(table.c.musicId)
.distinct()
.where(cond)
.order_by(table.c.musicId)
)
if limit is not None:
subq = subq.limit(limit)
if offset is not None:
subq = subq.offset(offset)
subq = subq.subquery()
sql = (
select(table)
.join(subq, table.c.musicId == subq.c.musicId)
.where(cond)
.order_by(table.c.musicId, table.c.level)
)
result = await self.execute(sql)
if result is None: