diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 55e668ec..b0d54599 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -3,6 +3,7 @@ name: Gradle Build on: pull_request: branches: [ master ] + workflow_dispatch: jobs: build: @@ -21,4 +22,6 @@ jobs: uses: gradle/gradle-build-action@v2 - name: Build with Gradle - run: ./gradlew build + run: | + mkdir data + ./gradlew build diff --git a/src/main/resources/db/migration/mariadb/V250__maimai2_item_integrity.sql b/src/main/resources/db/migration/mariadb/V250__maimai2_item_integrity.sql new file mode 100644 index 00000000..6fe387b4 --- /dev/null +++ b/src/main/resources/db/migration/mariadb/V250__maimai2_item_integrity.sql @@ -0,0 +1,4 @@ +# Fixes crashes due to duplicate keys in maimai2_user_item +alter table maimai2_user_item + add constraint maimai2_user_item_pk + unique (item_kind, item_id, user_id); diff --git a/src/main/resources/db/migration/mysql/V250__maimai2_item_integrity.sql b/src/main/resources/db/migration/mysql/V250__maimai2_item_integrity.sql new file mode 100644 index 00000000..6fe387b4 --- /dev/null +++ b/src/main/resources/db/migration/mysql/V250__maimai2_item_integrity.sql @@ -0,0 +1,4 @@ +# Fixes crashes due to duplicate keys in maimai2_user_item +alter table maimai2_user_item + add constraint maimai2_user_item_pk + unique (item_kind, item_id, user_id); diff --git a/src/main/resources/db/migration/sqlite/V250__maimai2_item_integrity.sql b/src/main/resources/db/migration/sqlite/V250__maimai2_item_integrity.sql new file mode 100644 index 00000000..324c57e5 --- /dev/null +++ b/src/main/resources/db/migration/sqlite/V250__maimai2_item_integrity.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX maimai2_user_item_pk ON maimai2_user_item(item_kind, item_id, user_id); diff --git a/tools/data_convert.py b/tools/data_convert.py index d11bf795..b2b88f77 100644 --- a/tools/data_convert.py +++ b/tools/data_convert.py @@ -1,5 +1,6 @@ import argparse +import os import shutil from pathlib import Path @@ -45,6 +46,32 @@ def convert_one(file: Path): write(target, orjson.dumps(xml)) +def combine_music(): + # Read all music json files + music_files = list(dst.rglob('music/*.json')) + print(f'> Found {len(music_files)} music files') + jsons = [orjson.loads(f.read_text()) for f in music_files] + + # Combine all music + combined = {d['name']['id']: { + 'name': d['name']['str'], + 'ver': int(d['version']), + 'composer': d['artistName']['str'], + 'genre': d['genreName']['str'], + 'bpm': int(d['bpm']), + 'lock': f"{d['lockType']} {d['subLockType']}", + 'notes': [{ + 'lv': int(n['level']) + (int(n['levelDecimal']) / 10), + 'designer': n['notesDesigner']['str'], + 'lv_id': n['musicLevelID'], + 'notes': int(n['maxNotes']), + } for n in d['notesData']['Notes'] if n['isEnable'] != 'false'] + } for d in jsons} + + # Write combined music + write(dst / '00/all-music.json', orjson.dumps(combined)) + + if __name__ == '__main__': agupa = argparse.ArgumentParser() agupa.add_argument('source', type=str, help='Package/Sinmai_Data/StreamingAssets directory') @@ -54,6 +81,24 @@ if __name__ == '__main__': src = Path(args.source) dst = Path(args.destination) + # Special post-convert command to relocate stuff + if args.source == 'post-convert': + ori = dst + dst = dst.parent + + # In assetbundle/dir, move each XXX_{id}_XXX.png to assetbundle/dir/{id}.png + for d in os.listdir(dst / 'assetbundle'): + d = dst / 'assetbundle' / d + if not d.is_dir(): + continue + + print(f'Relocating {d}') + for file in d.rglob('*.png'): + id = ''.join(filter(str.isdigit, file.stem)) + shutil.move(file, d / f'{id}.png') + + exit(0) + # Assert that A000 exists in the source directory assert (src / 'A000').exists(), f'{src}/A000 does not exist' @@ -69,5 +114,10 @@ if __name__ == '__main__': # Multithreaded map pmap(convert_one, files, desc='Converting', unit='file', chunksize=50) + print('> Finished converting') + + # Convert all music + print('Combining music') + combine_music() diff --git a/tools/maimai_convert.py b/tools/maimai_convert.py index bae4ec4e..01ae48f3 100644 --- a/tools/maimai_convert.py +++ b/tools/maimai_convert.py @@ -66,6 +66,38 @@ def add_migration(f_name: str, mysql: str): (migration_path / 'sqlite' / f_name).write_text(';\n'.join(sqlite) + ';\n') +class Character(NamedTuple): + id: int # CharaData.name.id + name: str # CharaData.name.str + color_id: int # CharaData.color.id + color_name: str # CharaData.color.str + genre_id: int # CharaData.genre.id + genre_name: str # CharaData.genre.str + is_copyright: bool # CharaData.isCopyright + disable: bool # CharaData.disable + + +def parse_bool(s: str) -> bool: + if s == 'true' or s == '1': + return True + if s == 'false' or s == '0': + return False + raise ValueError(f'Invalid boolean value: {s}') + + +def parse_character(d: dict) -> Character: + return Character( + id=int(d['CharaData']['name']['id']), + name=d['CharaData']['name']['str'], + color_id=int(d['CharaData']['color']['id']), + color_name=d['CharaData']['color']['str'], + genre_id=int(d['CharaData']['genre']['id']), + genre_name=d['CharaData']['genre']['str'], + is_copyright=parse_bool(d['CharaData']['isCopyright']), + disable=parse_bool(d['CharaData']['disable']) + ) + + if __name__ == '__main__': agupa = argparse.ArgumentParser(description='Convert maimai data to csv') agupa.add_argument('path', type=Path, help='Path to A000 data folder') @@ -77,6 +109,8 @@ if __name__ == '__main__': events = read_list('event', '*/Event.xml', parse_event) + characters = read_list('chara', '*/Chara.xml', parse_character) + # Write incremental sql # ids = [int(v.split(",")[0]) for v in (Path(__file__).parent / 'maimai2_game_event.csv').read_text().splitlines()] # new_events = [e for e in events if e.id not in ids] @@ -108,4 +142,3 @@ CREATE TABLE `maimai2_game_ticket` ( sql += ";\n" add_migration(f"V{last_sql_version + 1}__maimai2_tickets.sql", sql) -