From 126546a9384afc31d4c7dfcdf0dc7227e36edb7b Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Fri, 9 Feb 2024 22:33:54 -0500 Subject: [PATCH 1/4] [O] Fix crash due to duplicate keys in maimai2_user_item --- .../db/migration/mariadb/V250__maimai2_item_integrity.sql | 4 ++++ .../db/migration/mysql/V250__maimai2_item_integrity.sql | 4 ++++ .../db/migration/sqlite/V250__maimai2_item_integrity.sql | 1 + 3 files changed, 9 insertions(+) create mode 100644 src/main/resources/db/migration/mariadb/V250__maimai2_item_integrity.sql create mode 100644 src/main/resources/db/migration/mysql/V250__maimai2_item_integrity.sql create mode 100644 src/main/resources/db/migration/sqlite/V250__maimai2_item_integrity.sql diff --git a/src/main/resources/db/migration/mariadb/V250__maimai2_item_integrity.sql b/src/main/resources/db/migration/mariadb/V250__maimai2_item_integrity.sql new file mode 100644 index 00000000..6fe387b4 --- /dev/null +++ b/src/main/resources/db/migration/mariadb/V250__maimai2_item_integrity.sql @@ -0,0 +1,4 @@ +# Fixes crashes due to duplicate keys in maimai2_user_item +alter table maimai2_user_item + add constraint maimai2_user_item_pk + unique (item_kind, item_id, user_id); diff --git a/src/main/resources/db/migration/mysql/V250__maimai2_item_integrity.sql b/src/main/resources/db/migration/mysql/V250__maimai2_item_integrity.sql new file mode 100644 index 00000000..6fe387b4 --- /dev/null +++ b/src/main/resources/db/migration/mysql/V250__maimai2_item_integrity.sql @@ -0,0 +1,4 @@ +# Fixes crashes due to duplicate keys in maimai2_user_item +alter table maimai2_user_item + add constraint maimai2_user_item_pk + unique (item_kind, item_id, user_id); diff --git a/src/main/resources/db/migration/sqlite/V250__maimai2_item_integrity.sql b/src/main/resources/db/migration/sqlite/V250__maimai2_item_integrity.sql new file mode 100644 index 00000000..324c57e5 --- /dev/null +++ b/src/main/resources/db/migration/sqlite/V250__maimai2_item_integrity.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX maimai2_user_item_pk ON maimai2_user_item(item_kind, item_id, user_id); From 5d9693c419fd1a105620dc0d5daa9a190529b3b0 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Fri, 9 Feb 2024 22:35:13 -0500 Subject: [PATCH 2/4] [U] Combine music jsons --- tools/data_convert.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tools/data_convert.py b/tools/data_convert.py index d11bf795..b2b88f77 100644 --- a/tools/data_convert.py +++ b/tools/data_convert.py @@ -1,5 +1,6 @@ import argparse +import os import shutil from pathlib import Path @@ -45,6 +46,32 @@ def convert_one(file: Path): write(target, orjson.dumps(xml)) +def combine_music(): + # Read all music json files + music_files = list(dst.rglob('music/*.json')) + print(f'> Found {len(music_files)} music files') + jsons = [orjson.loads(f.read_text()) for f in music_files] + + # Combine all music + combined = {d['name']['id']: { + 'name': d['name']['str'], + 'ver': int(d['version']), + 'composer': d['artistName']['str'], + 'genre': d['genreName']['str'], + 'bpm': int(d['bpm']), + 'lock': f"{d['lockType']} {d['subLockType']}", + 'notes': [{ + 'lv': int(n['level']) + (int(n['levelDecimal']) / 10), + 'designer': n['notesDesigner']['str'], + 'lv_id': n['musicLevelID'], + 'notes': int(n['maxNotes']), + } for n in d['notesData']['Notes'] if n['isEnable'] != 'false'] + } for d in jsons} + + # Write combined music + write(dst / '00/all-music.json', orjson.dumps(combined)) + + if __name__ == '__main__': agupa = argparse.ArgumentParser() agupa.add_argument('source', type=str, help='Package/Sinmai_Data/StreamingAssets directory') @@ -54,6 +81,24 @@ if __name__ == '__main__': src = Path(args.source) dst = Path(args.destination) + # Special post-convert command to relocate stuff + if args.source == 'post-convert': + ori = dst + dst = dst.parent + + # In assetbundle/dir, move each XXX_{id}_XXX.png to assetbundle/dir/{id}.png + for d in os.listdir(dst / 'assetbundle'): + d = dst / 'assetbundle' / d + if not d.is_dir(): + continue + + print(f'Relocating {d}') + for file in d.rglob('*.png'): + id = ''.join(filter(str.isdigit, file.stem)) + shutil.move(file, d / f'{id}.png') + + exit(0) + # Assert that A000 exists in the source directory assert (src / 'A000').exists(), f'{src}/A000 does not exist' @@ -69,5 +114,10 @@ if __name__ == '__main__': # Multithreaded map pmap(convert_one, files, desc='Converting', unit='file', chunksize=50) + print('> Finished converting') + + # Convert all music + print('Combining music') + combine_music() From f46c8a03d9891b38fd04ea36de6b2b7d14d139e9 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Fri, 9 Feb 2024 22:36:26 -0500 Subject: [PATCH 3/4] [+] Character convert --- tools/maimai_convert.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tools/maimai_convert.py b/tools/maimai_convert.py index bae4ec4e..01ae48f3 100644 --- a/tools/maimai_convert.py +++ b/tools/maimai_convert.py @@ -66,6 +66,38 @@ def add_migration(f_name: str, mysql: str): (migration_path / 'sqlite' / f_name).write_text(';\n'.join(sqlite) + ';\n') +class Character(NamedTuple): + id: int # CharaData.name.id + name: str # CharaData.name.str + color_id: int # CharaData.color.id + color_name: str # CharaData.color.str + genre_id: int # CharaData.genre.id + genre_name: str # CharaData.genre.str + is_copyright: bool # CharaData.isCopyright + disable: bool # CharaData.disable + + +def parse_bool(s: str) -> bool: + if s == 'true' or s == '1': + return True + if s == 'false' or s == '0': + return False + raise ValueError(f'Invalid boolean value: {s}') + + +def parse_character(d: dict) -> Character: + return Character( + id=int(d['CharaData']['name']['id']), + name=d['CharaData']['name']['str'], + color_id=int(d['CharaData']['color']['id']), + color_name=d['CharaData']['color']['str'], + genre_id=int(d['CharaData']['genre']['id']), + genre_name=d['CharaData']['genre']['str'], + is_copyright=parse_bool(d['CharaData']['isCopyright']), + disable=parse_bool(d['CharaData']['disable']) + ) + + if __name__ == '__main__': agupa = argparse.ArgumentParser(description='Convert maimai data to csv') agupa.add_argument('path', type=Path, help='Path to A000 data folder') @@ -77,6 +109,8 @@ if __name__ == '__main__': events = read_list('event', '*/Event.xml', parse_event) + characters = read_list('chara', '*/Chara.xml', parse_character) + # Write incremental sql # ids = [int(v.split(",")[0]) for v in (Path(__file__).parent / 'maimai2_game_event.csv').read_text().splitlines()] # new_events = [e for e in events if e.id not in ids] @@ -108,4 +142,3 @@ CREATE TABLE `maimai2_game_ticket` ( sql += ";\n" add_migration(f"V{last_sql_version + 1}__maimai2_tickets.sql", sql) - From 106bded9b6c1ddda5918aa9d6e3e7fbcffec4907 Mon Sep 17 00:00:00 2001 From: Azalea Date: Fri, 9 Feb 2024 19:39:09 -0800 Subject: [PATCH 4/4] [F] Fix test failure --- .github/workflows/gradle.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 55e668ec..b0d54599 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -3,6 +3,7 @@ name: Gradle Build on: pull_request: branches: [ master ] + workflow_dispatch: jobs: build: @@ -21,4 +22,6 @@ jobs: uses: gradle/gradle-build-action@v2 - name: Build with Gradle - run: ./gradlew build + run: | + mkdir data + ./gradlew build