Merge branch 'develop' into diva_handler_classes

titles/chuni/__init__.py

@@ -7,4 +7,4 @@ index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
 game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW]
-current_schema_version = 3
+current_schema_version = 4

titles/chuni/base.py

@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
 from time import strftime
 
 import pytz
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 from core.config import CoreConfig
 from titles.chuni.const import ChuniConstants
@@ -44,13 +44,15 @@ class ChuniBase:
             # check if a user already has some pogress and if not add the
             # login bonus entry
             user_login_bonus = self.data.item.get_login_bonus(
-                user_id, self.version, preset["id"]
+                user_id, self.version, preset["presetId"]
             )
             if user_login_bonus is None:
-                self.data.item.put_login_bonus(user_id, self.version, preset["id"])
+                self.data.item.put_login_bonus(
+                    user_id, self.version, preset["presetId"]
+                )
                 # yeah i'm lazy
                 user_login_bonus = self.data.item.get_login_bonus(
-                    user_id, self.version, preset["id"]
+                    user_id, self.version, preset["presetId"]
                 )
 
             # skip the login bonus entirely if its already finished
@@ -66,13 +68,13 @@ class ChuniBase:
                 last_update_date = datetime.now()
 
                 all_login_boni = self.data.static.get_login_bonus(
-                    self.version, preset["id"]
+                    self.version, preset["presetId"]
                 )
 
                 # skip the current bonus preset if no boni were found
                 if all_login_boni is None or len(all_login_boni) < 1:
-                    self.logger.warn(
-                        f"No bonus entries found for bonus preset {preset['id']}"
+                    self.logger.warning(
+                        f"No bonus entries found for bonus preset {preset['presetId']}"
                     )
                     continue
 
@@ -83,14 +85,14 @@ class ChuniBase:
                 if bonus_count > max_needed_days:
                     # assume that all login preset ids under 3000 needs to be
                     # looped, like 30 and 40 are looped, 40 does not work?
-                    if preset["id"] < 3000:
+                    if preset["presetId"] < 3000:
                         bonus_count = 1
                     else:
                         is_finished = True
 
                 # grab the item for the corresponding day
                 login_item = self.data.static.get_login_bonus_by_required_days(
-                    self.version, preset["id"], bonus_count
+                    self.version, preset["presetId"], bonus_count
                 )
                 if login_item is not None:
                     # now add the present to the database so the
@@ -108,7 +110,7 @@ class ChuniBase:
                 self.data.item.put_login_bonus(
                     user_id,
                     self.version,
-                    preset["id"],
+                    preset["presetId"],
                     bonusCount=bonus_count,
                     lastUpdateDate=last_update_date,
                     isWatched=False,
@@ -147,7 +149,7 @@ class ChuniBase:
         game_events = self.data.static.get_enabled_events(self.version)
 
         if game_events is None or len(game_events) == 0:
-            self.logger.warn("No enabled events, did you run the reader?")
+            self.logger.warning("No enabled events, did you run the reader?")
             return {
                 "type": data["type"],
                 "length": 0,
@@ -156,12 +158,18 @@ class ChuniBase:
 
         event_list = []
         for evt_row in game_events:
-            tmp = {}
-            tmp["id"] = evt_row["eventId"]
-            tmp["type"] = evt_row["type"]
-            tmp["startDate"] = "2017-12-05 07:00:00.0"
-            tmp["endDate"] = "2099-12-31 00:00:00.0"
-            event_list.append(tmp)
+            event_list.append(
+                {
+                    "id": evt_row["eventId"],
+                    "type": evt_row["type"],
+                    # actually use the startDate from the import so it
+                    # properly shows all the events when new ones are imported
+                    "startDate": datetime.strftime(
+                        evt_row["startDate"], "%Y-%m-%d %H:%M:%S"
+                    ),
+                    "endDate": "2099-12-31 00:00:00",
+                }
+            )
 
         return {
             "type": data["type"],
@@ -228,29 +236,36 @@ class ChuniBase:
     def handle_get_user_character_api_request(self, data: Dict) -> Dict:
         characters = self.data.item.get_characters(data["userId"])
         if characters is None:
-            return {}
-        next_idx = -1
+            return {
+                "userId": data["userId"],
+                "length": 0,
+                "nextIndex": -1,
+                "userCharacterList": [],
+            }
 
-        characterList = []
-        for x in range(int(data["nextIndex"]), len(characters)):
+        character_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(characters)):
             tmp = characters[x]._asdict()
             tmp.pop("user")
             tmp.pop("id")
-            characterList.append(tmp)
+            character_list.append(tmp)
 
-            if len(characterList) >= int(data["maxCount"]):
+            if len(character_list) >= max_ct:
                 break
 
-        if len(characterList) >= int(data["maxCount"]) and len(characters) > int(
-            data["maxCount"]
-        ) + int(data["nextIndex"]):
-            next_idx = int(data["maxCount"]) + int(data["nextIndex"]) + 1
+        if len(characters) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = -1
 
         return {
             "userId": data["userId"],
-            "length": len(characterList),
+            "length": len(character_list),
             "nextIndex": next_idx,
-            "userCharacterList": characterList,
+            "userCharacterList": character_list,
         }
 
     def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
@@ -292,8 +307,8 @@ class ChuniBase:
             if len(user_course_list) >= max_ct:
                 break
 
-        if len(user_course_list) >= max_ct:
-            next_idx = next_idx + max_ct
+        if len(user_course_list) >= next_idx + max_ct:
+            next_idx += max_ct
         else:
             next_idx = -1
 
@@ -347,12 +362,23 @@ class ChuniBase:
         }
 
     def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
+        user_fav_item_list = []
+
+        # still needs to be implemented on WebUI
+        # 1: Music, 3: Character
+        fav_list = self.data.item.get_all_favorites(
+            data["userId"], self.version, fav_kind=int(data["kind"])
+        )
+        if fav_list is not None:
+            for fav in fav_list:
+                user_fav_item_list.append({"id": fav["favId"]})
+
         return {
             "userId": data["userId"],
-            "length": 0,
+            "length": len(user_fav_item_list),
             "kind": data["kind"],
             "nextIndex": -1,
-            "userFavoriteItemList": [],
+            "userFavoriteItemList": user_fav_item_list,
         }
 
     def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
@@ -375,7 +401,7 @@ class ChuniBase:
                 "userItemList": [],
             }
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(next_idx, len(user_item_list)):
             tmp = user_item_list[i]._asdict()
             tmp.pop("user")
@@ -387,13 +413,13 @@ class ChuniBase:
         xout = kind * 10000000000 + next_idx + len(items)
 
         if len(items) < int(data["maxCount"]):
-            nextIndex = 0
+            next_idx = 0
         else:
-            nextIndex = xout
+            next_idx = xout
 
         return {
             "userId": data["userId"],
-            "nextIndex": nextIndex,
+            "nextIndex": next_idx,
             "itemKind": kind,
             "length": len(items),
             "userItemList": items,
@@ -452,6 +478,7 @@ class ChuniBase:
                 "nextIndex": -1,
                 "userMusicList": [],  # 240
             }
+
         song_list = []
         next_idx = int(data["nextIndex"])
         max_ct = int(data["maxCount"])
@@ -474,10 +501,10 @@ class ChuniBase:
             if len(song_list) >= max_ct:
                 break
 
-        if len(song_list) >= max_ct:
+        if len(song_list) >= next_idx + max_ct:
             next_idx += max_ct
         else:
-            next_idx = 0
+            next_idx = -1
 
         return {
             "userId": data["userId"],
@@ -617,18 +644,21 @@ class ChuniBase:
                 upsert["userData"][0]["userName"] = self.read_wtf8(
                     upsert["userData"][0]["userName"]
                 )
-            except:
+            except Exception:
                 pass
 
             self.data.profile.put_profile_data(
                 user_id, self.version, upsert["userData"][0]
             )
+
         if "userDataEx" in upsert:
             self.data.profile.put_profile_data_ex(
                 user_id, self.version, upsert["userDataEx"][0]
             )
+
         if "userGameOption" in upsert:
             self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
+
         if "userGameOptionEx" in upsert:
             self.data.profile.put_profile_option_ex(
                 user_id, upsert["userGameOptionEx"][0]
@@ -672,6 +702,10 @@ class ChuniBase:
 
         if "userPlaylogList" in upsert:
             for playlog in upsert["userPlaylogList"]:
+                # convert the player names to utf-8
+                playlog["playedUserName1"] = self.read_wtf8(playlog["playedUserName1"])
+                playlog["playedUserName2"] = self.read_wtf8(playlog["playedUserName2"])
+                playlog["playedUserName3"] = self.read_wtf8(playlog["playedUserName3"])
                 self.data.score.put_playlog(user_id, playlog)
 
         if "userTeamPoint" in upsert:

titles/chuni/const.py

@@ -17,21 +17,23 @@ class ChuniConstants:
     VER_CHUNITHM_PARADISE = 10
     VER_CHUNITHM_NEW = 11
     VER_CHUNITHM_NEW_PLUS = 12
+    VER_CHUNITHM_SUN = 13
 
     VERSION_NAMES = [
-        "Chunithm",
-        "Chunithm+",
-        "Chunithm Air",
-        "Chunithm Air+",
-        "Chunithm Star",
-        "Chunithm Star+",
-        "Chunithm Amazon",
-        "Chunithm Amazon+",
-        "Chunithm Crystal",
-        "Chunithm Crystal+",
-        "Chunithm Paradise",
-        "Chunithm New!!",
-        "Chunithm New!!+",
+        "CHUNITHM",
+        "CHUNITHM PLUS",
+        "CHUNITHM AIR",
+        "CHUNITHM AIR PLUS",
+        "CHUNITHM STAR",
+        "CHUNITHM STAR  PLUS",
+        "CHUNITHM AMAZON",
+        "CHUNITHM AMAZON PLUS",
+        "CHUNITHM CRYSTAL",
+        "CHUNITHM CRYSTAL PLUS",
+        "CHUNITHM PARADISE",
+        "CHUNITHM NEW!!",
+        "CHUNITHM NEW PLUS!!",
+        "CHUNITHM SUN"
     ]
 
     @classmethod

titles/chuni/index.py

@@ -29,6 +29,7 @@ from titles.chuni.crystalplus import ChuniCrystalPlus
 from titles.chuni.paradise import ChuniParadise
 from titles.chuni.new import ChuniNew
 from titles.chuni.newplus import ChuniNewPlus
+from titles.chuni.sun import ChuniSun
 
 
 class ChuniServlet:
@@ -55,6 +56,7 @@ class ChuniServlet:
             ChuniParadise,
             ChuniNew,
             ChuniNewPlus,
+            ChuniSun,
         ]
 
         self.logger = logging.getLogger("chuni")
@@ -96,15 +98,18 @@ class ChuniServlet:
             ]
             for method in method_list:
                 method_fixed = inflection.camelize(method)[6:-7]
+                # number of iterations was changed to 70 in SUN
+                iter_count = 70 if version >= ChuniConstants.VER_CHUNITHM_SUN else 44
                 hash = PBKDF2(
                     method_fixed,
                     bytes.fromhex(keys[2]),
                     128,
-                    count=44,
+                    count=iter_count,
                     hmac_hash_module=SHA1,
                 )
 
-                self.hash_table[version][hash.hex()] = method_fixed
+                hashed_name = hash.hex()[:32] # truncate unused bytes like the game does
+                self.hash_table[version][hashed_name] = method_fixed
 
                 self.logger.debug(
                     f"Hashed v{version} method {method_fixed} with {bytes.fromhex(keys[2])} to get {hash.hex()}"
@@ -145,30 +150,32 @@ class ChuniServlet:
 
         if version < 105:  # 1.0
             internal_ver = ChuniConstants.VER_CHUNITHM
-        elif version >= 105 and version < 110:  # Plus
+        elif version >= 105 and version < 110:  # PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_PLUS
-        elif version >= 110 and version < 115:  # Air
+        elif version >= 110 and version < 115:  # AIR
             internal_ver = ChuniConstants.VER_CHUNITHM_AIR
-        elif version >= 115 and version < 120:  # Air Plus
+        elif version >= 115 and version < 120:  # AIR PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_AIR_PLUS
-        elif version >= 120 and version < 125:  # Star
+        elif version >= 120 and version < 125:  # STAR
             internal_ver = ChuniConstants.VER_CHUNITHM_STAR
-        elif version >= 125 and version < 130:  # Star Plus
+        elif version >= 125 and version < 130:  # STAR PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_STAR_PLUS
-        elif version >= 130 and version < 135:  # Amazon
+        elif version >= 130 and version < 135:  # AMAZON
             internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON
-        elif version >= 135 and version < 140:  # Amazon Plus
+        elif version >= 135 and version < 140:  # AMAZON PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON_PLUS
-        elif version >= 140 and version < 145:  # Crystal
+        elif version >= 140 and version < 145:  # CRYSTAL
             internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL
-        elif version >= 145 and version < 150:  # Crystal Plus
+        elif version >= 145 and version < 150:  # CRYSTAL PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
-        elif version >= 150 and version < 200:  # Paradise
+        elif version >= 150 and version < 200:  # PARADISE
             internal_ver = ChuniConstants.VER_CHUNITHM_PARADISE
-        elif version >= 200 and version < 205:  # New
+        elif version >= 200 and version < 205:  # NEW!!
             internal_ver = ChuniConstants.VER_CHUNITHM_NEW
-        elif version >= 205 and version < 210:  # New Plus
+        elif version >= 205 and version < 210:  # NEW PLUS!!
             internal_ver = ChuniConstants.VER_CHUNITHM_NEW_PLUS
+        elif version >= 210:  # SUN
+            internal_ver = ChuniConstants.VER_CHUNITHM_SUN
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
             # If we get a 32 character long hex string, it's a hash and we're

titles/chuni/new.py

@@ -23,41 +23,44 @@ class ChuniNew(ChuniBase):
         self.version = ChuniConstants.VER_CHUNITHM_NEW
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        # use UTC time and convert it to JST time by adding +9
+        # matching therefore starts one hour before and lasts for 8 hours
         match_start = datetime.strftime(
-            datetime.now() - timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=8), self.date_time_format
         )
         match_end = datetime.strftime(
-            datetime.now() + timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=16), self.date_time_format
         )
         reboot_start = datetime.strftime(
-            datetime.now() - timedelta(hours=11), self.date_time_format
+            datetime.utcnow() + timedelta(hours=6), self.date_time_format
         )
         reboot_end = datetime.strftime(
-            datetime.now() - timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=7), self.date_time_format
         )
         return {
             "gameSetting": {
-                "isMaintenance": "false",
+                "isMaintenance": False,
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,
-                "isBackgroundDistribute": "false",
+                "isBackgroundDistribute": False,
                 "maxCountCharacter": 300,
                 "maxCountItem": 300,
                 "maxCountMusic": 300,
                 "matchStartTime": match_start,
                 "matchEndTime": match_end,
-                "matchTimeLimit": 99,
+                "matchTimeLimit": 60,
                 "matchErrorLimit": 9999,
                 "romVersion": self.game_cfg.version.version(self.version)["rom"],
                 "dataVersion": self.game_cfg.version.version(self.version)["data"],
                 "matchingUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
                 "matchingUriX": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
+                # might be really important for online battle to connect the cabs via UDP port 50201
                 "udpHolePunchUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
                 "reflectorUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
             },
-            "isDumpUpload": "false",
-            "isAou": "false",
+            "isDumpUpload": False,
+            "isAou": False,
         }
 
     def handle_remove_token_api_request(self, data: Dict) -> Dict:
@@ -468,3 +471,162 @@ class ChuniNew(ChuniBase):
             self.data.item.put_user_print_state(user_id, id=order_id, hasCompleted=True)
 
         return {"returnCode": "1", "apiName": "CMUpsertUserPrintCancelApi"}
+
+    def handle_ping_request(self, data: Dict) -> Dict:
+        # matchmaking ping request
+        return {"returnCode": "1"}
+
+    def handle_begin_matching_api_request(self, data: Dict) -> Dict:
+        room_id = 1
+        # check if there is a free matching room
+        matching_room = self.data.item.get_oldest_free_matching(self.version)
+
+        if matching_room is None:
+            # grab the latest roomId and add 1 for the new room
+            newest_matching = self.data.item.get_newest_matching(self.version)
+            if newest_matching is not None:
+                room_id = newest_matching["roomId"] + 1
+
+            # fix userName WTF8
+            new_member = data["matchingMemberInfo"]
+            new_member["userName"] = self.read_wtf8(new_member["userName"])
+
+            # create the new room with room_id and the current user id (host)
+            # user id is required for the countdown later on
+            self.data.item.put_matching(
+                self.version, room_id, [new_member], user_id=new_member["userId"]
+            )
+
+            # get the newly created matching room
+            matching_room = self.data.item.get_matching(self.version, room_id)
+        else:
+            # a room already exists, so just add the new member to it
+            matching_member_list = matching_room["matchingMemberInfoList"]
+            # fix userName WTF8
+            new_member = data["matchingMemberInfo"]
+            new_member["userName"] = self.read_wtf8(new_member["userName"])
+            matching_member_list.append(new_member)
+
+            # add the updated room to the database, make sure to set isFull correctly!
+            self.data.item.put_matching(
+                self.version,
+                matching_room["roomId"],
+                matching_member_list,
+                user_id=matching_room["user"],
+                is_full=True if len(matching_member_list) >= 4 else False,
+            )
+
+        matching_wait = {
+            "isFinish": False,
+            "restMSec": matching_room["restMSec"],  # in sec
+            "pollingInterval": 1,  # in sec
+            "matchingMemberInfoList": matching_room["matchingMemberInfoList"],
+        }
+
+        return {"roomId": 1, "matchingWaitState": matching_wait}
+
+    def handle_end_matching_api_request(self, data: Dict) -> Dict:
+        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        members = matching_room["matchingMemberInfoList"]
+
+        # only set the host user to role 1 every other to 0?
+        role_list = [
+            {"role": 1} if m["userId"] == matching_room["user"] else {"role": 0}
+            for m in members
+        ]
+
+        self.data.item.put_matching(
+            self.version,
+            matching_room["roomId"],
+            members,
+            user_id=matching_room["user"],
+            rest_sec=0,  # make sure to always set 0
+            is_full=True,  # and full, so no one can join
+        )
+
+        return {
+            "matchingResult": 1,  # needs to be 1 for successful matching
+            "matchingMemberInfoList": members,
+            # no idea, maybe to differentiate between CPUs and real players?
+            "matchingMemberRoleList": role_list,
+            # TCP/UDP connection?
+            "reflectorUri": f"{self.core_cfg.title.hostname}",
+        }
+
+    def handle_remove_matching_member_api_request(self, data: Dict) -> Dict:
+        # get all matching rooms, because Chuni only returns the userId
+        # not the actual roomId
+        matching_rooms = self.data.item.get_all_matchings(self.version)
+        if matching_rooms is None:
+            return {"returnCode": "1"}
+
+        for room in matching_rooms:
+            old_members = room["matchingMemberInfoList"]
+            new_members = [m for m in old_members if m["userId"] != data["userId"]]
+
+            # if nothing changed go to the next room
+            if len(old_members) == len(new_members):
+                continue
+
+            # if the last user got removed, delete the matching room
+            if len(new_members) <= 0:
+                self.data.item.delete_matching(self.version, room["roomId"])
+            else:
+                # remove the user from the room
+                self.data.item.put_matching(
+                    self.version,
+                    room["roomId"],
+                    new_members,
+                    user_id=room["user"],
+                    rest_sec=room["restMSec"],
+                )
+
+        return {"returnCode": "1"}
+
+    def handle_get_matching_state_api_request(self, data: Dict) -> Dict:
+        polling_interval = 1
+        # get the current active room
+        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        members = matching_room["matchingMemberInfoList"]
+        rest_sec = matching_room["restMSec"]
+
+        # grab the current member
+        current_member = data["matchingMemberInfo"]
+
+        # only the host user can decrease the countdown
+        if matching_room["user"] == int(current_member["userId"]):
+            # cap the restMSec to 0
+            if rest_sec > 0:
+                rest_sec -= polling_interval
+            else:
+                rest_sec = 0
+
+        # update the members in order to recieve messages
+        for i, member in enumerate(members):
+            if member["userId"] == current_member["userId"]:
+                # replace the old user data with the current user data,
+                # also parse WTF-8 everytime
+                current_member["userName"] = self.read_wtf8(current_member["userName"])
+                members[i] = current_member
+
+        self.data.item.put_matching(
+            self.version,
+            data["roomId"],
+            members,
+            rest_sec=rest_sec,
+            user_id=matching_room["user"],
+        )
+
+        # only add the other members to the list
+        diff_members = [m for m in members if m["userId"] != current_member["userId"]]
+
+        matching_wait = {
+            # makes no difference? Always use False?
+            "isFinish": True if rest_sec == 0 else False,
+            "restMSec": rest_sec,
+            "pollingInterval": polling_interval,
+            # the current user needs to be the first one?
+            "matchingMemberInfoList": [current_member] + diff_members,
+        }
+
+        return {"matchingWaitState": matching_wait}

titles/chuni/newplus.py

@@ -36,6 +36,6 @@ class ChuniNewPlus(ChuniNew):
     def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
         user_data = super().handle_cm_get_user_preview_api_request(data)
 
-        # hardcode lastDataVersion for CardMaker 1.35
+        # hardcode lastDataVersion for CardMaker 1.35 A028
         user_data["lastDataVersion"] = "2.05.00"
         return user_data

titles/chuni/read.py

@@ -67,7 +67,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted login bonus preset {id}")
                     else:
-                        self.logger.warn(f"Failed to insert login bonus preset {id}")
+                        self.logger.warning(f"Failed to insert login bonus preset {id}")
 
                     for bonus in xml_root.find("infos").findall("LoginBonusDataInfo"):
                         for name in bonus.findall("loginBonusName"):
@@ -113,7 +113,7 @@ class ChuniReader(BaseReader):
                                 if result is not None:
                                     self.logger.info(f"Inserted login bonus {bonus_id}")
                                 else:
-                                    self.logger.warn(
+                                    self.logger.warning(
                                         f"Failed to insert login bonus {bonus_id}"
                                     )
 
@@ -138,7 +138,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted event {id}")
                     else:
-                        self.logger.warn(f"Failed to insert event {id}")
+                        self.logger.warning(f"Failed to insert event {id}")
 
     def read_music(self, music_dir: str) -> None:
         for root, dirs, files in walk(music_dir):
@@ -169,8 +169,10 @@ class ChuniReader(BaseReader):
                             fumen_path = MusicFumenData.find("file").find("path")
 
                             if fumen_path is not None:
-                                chart_id = MusicFumenData.find("type").find("id").text
-                                if chart_id == "4":
+                                chart_type = MusicFumenData.find("type")
+                                chart_id = chart_type.find("id").text
+                                chart_diff = chart_type.find("str").text
+                                if chart_diff == "WorldsEnd" and (chart_id == "4" or chart_id == "5"): # 4 in SDBT, 5 in SDHD
                                     level = float(xml_root.find("starDifType").text)
                                     we_chara = (
                                         xml_root.find("worldsEndTagName")
@@ -200,7 +202,7 @@ class ChuniReader(BaseReader):
                                         f"Inserted music {song_id} chart {chart_id}"
                                     )
                                 else:
-                                    self.logger.warn(
+                                    self.logger.warning(
                                         f"Failed to insert music {song_id} chart {chart_id}"
                                     )
 
@@ -232,7 +234,7 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted charge {id}")
                     else:
-                        self.logger.warn(f"Failed to insert charge {id}")
+                        self.logger.warning(f"Failed to insert charge {id}")
 
     def read_avatar(self, avatar_dir: str) -> None:
         for root, dirs, files in walk(avatar_dir):
@@ -259,4 +261,4 @@ class ChuniReader(BaseReader):
                     if result is not None:
                         self.logger.info(f"Inserted avatarAccessory {id}")
                     else:
-                        self.logger.warn(f"Failed to insert avatarAccessory {id}")
+                        self.logger.warning(f"Failed to insert avatarAccessory {id}")

titles/chuni/schema/item.py

@@ -1,5 +1,12 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy import (
+    Table,
+    Column,
+    UniqueConstraint,
+    PrimaryKeyConstraint,
+    and_,
+    delete,
+)
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.schema import ForeignKey
@@ -203,8 +210,141 @@ login_bonus = Table(
     mysql_charset="utf8mb4",
 )
 
+favorite = Table(
+    "chuni_item_favorite",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("favId", Integer, nullable=False),
+    Column("favKind", Integer, nullable=False, server_default="1"),
+    UniqueConstraint("version", "user", "favId", name="chuni_item_favorite_uk"),
+    mysql_charset="utf8mb4",
+)
+
+matching = Table(
+    "chuni_item_matching",
+    metadata,
+    Column("roomId", Integer, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("restMSec", Integer, nullable=False, server_default="60"),
+    Column("isFull", Boolean, nullable=False, server_default="0"),
+    PrimaryKeyConstraint("roomId", "version", name="chuni_item_matching_pk"),
+    Column("matchingMemberInfoList", JSON, nullable=False),
+    mysql_charset="utf8mb4",
+)
+
 
 class ChuniItemData(BaseData):
+    def get_oldest_free_matching(self, version: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version,
+                matching.c.isFull == False
+            )
+        ).order_by(matching.c.roomId.asc())
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_newest_matching(self, version: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version
+            )
+        ).order_by(matching.c.roomId.desc())
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_all_matchings(self, version: int) -> Optional[List[Row]]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_matching(self, version: int, room_id: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(matching.c.version == version, matching.c.roomId == room_id)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_matching(
+        self,
+        version: int,
+        room_id: int,
+        matching_member_info_list: List,
+        user_id: int = None,
+        rest_sec: int = 60,
+        is_full: bool = False
+    ) -> Optional[int]:
+        sql = insert(matching).values(
+            roomId=room_id,
+            version=version,
+            restMSec=rest_sec,
+            user=user_id,
+            isFull=is_full,
+            matchingMemberInfoList=matching_member_info_list,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            restMSec=rest_sec, matchingMemberInfoList=matching_member_info_list
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def delete_matching(self, version: int, room_id: int):
+        sql = delete(matching).where(
+            and_(matching.c.roomId == room_id, matching.c.version == version)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def get_all_favorites(
+        self, user_id: int, version: int, fav_kind: int = 1
+    ) -> Optional[List[Row]]:
+        sql = favorite.select(
+            and_(
+                favorite.c.version == version,
+                favorite.c.user == user_id,
+                favorite.c.favKind == fav_kind,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def put_login_bonus(
         self, user_id: int, version: int, preset_id: int, **login_bonus_data
     ) -> Optional[int]:
@@ -390,7 +530,7 @@ class ChuniItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
+            self.logger.warning(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -432,7 +572,7 @@ class ChuniItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_user_print_state: Failed to insert! aime_id: {aime_id}"
             )
             return None
@@ -449,7 +589,7 @@ class ChuniItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_user_print_detail: Failed to insert! aime_id: {aime_id}"
             )
             return None

titles/chuni/schema/profile.py

@@ -89,8 +89,6 @@ profile = Table(
         Integer,
         ForeignKey("chuni_profile_team.id", ondelete="SET NULL", onupdate="SET NULL"),
     ),
-    Column("avatarBack", Integer, server_default="0"),
-    Column("avatarFace", Integer, server_default="0"),
     Column("eliteRankPoint", Integer, server_default="0"),
     Column("stockedGridCount", Integer, server_default="0"),
     Column("netBattleLoseCount", Integer, server_default="0"),
@@ -98,10 +96,8 @@ profile = Table(
     Column("netBattle4thCount", Integer, server_default="0"),
     Column("overPowerRate", Integer, server_default="0"),
     Column("battleRewardStatus", Integer, server_default="0"),
-    Column("avatarPoint", Integer, server_default="0"),
     Column("netBattle1stCount", Integer, server_default="0"),
     Column("charaIllustId", Integer, server_default="0"),
-    Column("avatarItem", Integer, server_default="0"),
     Column("userNameEx", String(8), server_default=""),
     Column("netBattleWinCount", Integer, server_default="0"),
     Column("netBattleCorrection", Integer, server_default="0"),
@@ -112,7 +108,6 @@ profile = Table(
     Column("netBattle3rdCount", Integer, server_default="0"),
     Column("netBattleConsecutiveWinCount", Integer, server_default="0"),
     Column("overPowerLowerRank", Integer, server_default="0"),
-    Column("avatarWear", Integer, server_default="0"),
     Column("classEmblemBase", Integer, server_default="0"),
     Column("battleRankPoint", Integer, server_default="0"),
     Column("netBattle2ndCount", Integer, server_default="0"),
@@ -120,13 +115,19 @@ profile = Table(
     Column("skillId", Integer, server_default="0"),
     Column("lastCountryCode", String(5), server_default="JPN"),
     Column("isNetBattleHost", Boolean, server_default="0"),
-    Column("avatarFront", Integer, server_default="0"),
-    Column("avatarSkin", Integer, server_default="0"),
     Column("battleRewardCount", Integer, server_default="0"),
     Column("battleRewardIndex", Integer, server_default="0"),
     Column("netBattlePlayCount", Integer, server_default="0"),
     Column("exMapLoopCount", Integer, server_default="0"),
     Column("netBattleEndState", Integer, server_default="0"),
+    Column("rankUpChallengeResults", JSON),
+    Column("avatarBack", Integer, server_default="0"),
+    Column("avatarFace", Integer, server_default="0"),
+    Column("avatarPoint", Integer, server_default="0"),
+    Column("avatarItem", Integer, server_default="0"),
+    Column("avatarWear", Integer, server_default="0"),
+    Column("avatarFront", Integer, server_default="0"),
+    Column("avatarSkin", Integer, server_default="0"),
     Column("avatarHead", Integer, server_default="0"),
     UniqueConstraint("user", "version", name="chuni_profile_profile_uk"),
     mysql_charset="utf8mb4",
@@ -409,7 +410,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_profile_data: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_profile_data: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -417,8 +418,8 @@ class ChuniProfileData(BaseData):
         sql = (
             select([profile, option])
             .join(option, profile.c.user == option.c.user)
-            .filter(and_(profile.c.user == aime_id, profile.c.version == version))
-        )
+            .filter(and_(profile.c.user == aime_id, profile.c.version <= version))
+        ).order_by(profile.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -429,9 +430,9 @@ class ChuniProfileData(BaseData):
         sql = select(profile).where(
             and_(
                 profile.c.user == aime_id,
-                profile.c.version == version,
+                profile.c.version <= version,
             )
-        )
+        ).order_by(profile.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -451,7 +452,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_data_ex: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -461,9 +462,9 @@ class ChuniProfileData(BaseData):
         sql = select(profile_ex).where(
             and_(
                 profile_ex.c.user == aime_id,
-                profile_ex.c.version == version,
+                profile_ex.c.version <= version,
             )
-        )
+        ).order_by(profile_ex.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -478,7 +479,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_option: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -502,7 +503,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_option_ex: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -526,7 +527,7 @@ class ChuniProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_recent_rating: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -551,7 +552,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_activity: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -577,7 +578,7 @@ class ChuniProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_charge: Failed to update! aime_id: {aime_id}"
             )
             return None

titles/chuni/schema/score.py

@@ -134,7 +134,9 @@ playlog = Table(
     Column("charaIllustId", Integer),
     Column("romVersion", String(255)),
     Column("judgeHeaven", Integer),
-    mysql_charset="utf8mb4",
+    Column("regionId", Integer),
+    Column("machineType", Integer),
+    mysql_charset="utf8mb4"
 )
 
 

titles/chuni/schema/static.py

@@ -1,11 +1,19 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy import (
+    ForeignKeyConstraint,
+    Table,
+    Column,
+    UniqueConstraint,
+    PrimaryKeyConstraint,
+    and_,
+)
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.engine import Row
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select
 from sqlalchemy.dialects.mysql import insert
+from datetime import datetime
 
 from core.data.schema import BaseData, metadata
 
@@ -17,6 +25,7 @@ events = Table(
     Column("eventId", Integer),
     Column("type", Integer),
     Column("name", String(255)),
+    Column("startDate", TIMESTAMP, server_default=func.now()),
     Column("enabled", Boolean, server_default="1"),
     UniqueConstraint("version", "eventId", name="chuni_static_events_uk"),
     mysql_charset="utf8mb4",
@@ -125,11 +134,13 @@ gacha_cards = Table(
 login_bonus_preset = Table(
     "chuni_static_login_bonus_preset",
     metadata,
-    Column("id", Integer, primary_key=True, nullable=False),
+    Column("presetId", Integer, nullable=False),
     Column("version", Integer, nullable=False),
     Column("presetName", String(255), nullable=False),
     Column("isEnabled", Boolean, server_default="1"),
-    UniqueConstraint("version", "id", name="chuni_static_login_bonus_preset_uk"),
+    PrimaryKeyConstraint(
+        "presetId", "version", name="chuni_static_login_bonus_preset_pk"
+    ),
     mysql_charset="utf8mb4",
 )
 
@@ -138,15 +149,7 @@ login_bonus = Table(
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
     Column("version", Integer, nullable=False),
-    Column(
-        "presetId",
-        ForeignKey(
-            "chuni_static_login_bonus_preset.id",
-            ondelete="cascade",
-            onupdate="cascade",
-        ),
-        nullable=False,
-    ),
+    Column("presetId", Integer, nullable=False),
     Column("loginBonusId", Integer, nullable=False),
     Column("loginBonusName", String(255), nullable=False),
     Column("presentId", Integer, nullable=False),
@@ -157,6 +160,16 @@ login_bonus = Table(
     UniqueConstraint(
         "version", "presetId", "loginBonusId", name="chuni_static_login_bonus_uk"
     ),
+    ForeignKeyConstraint(
+        ["presetId", "version"],
+        [
+            "chuni_static_login_bonus_preset.presetId",
+            "chuni_static_login_bonus_preset.version",
+        ],
+        onupdate="CASCADE",
+        ondelete="CASCADE",
+        name="chuni_static_login_bonus_ibfk_1",
+    ),
     mysql_charset="utf8mb4",
 )
 
@@ -236,7 +249,7 @@ class ChuniStaticData(BaseData):
         self, version: int, preset_id: int, preset_name: str, is_enabled: bool
     ) -> Optional[int]:
         sql = insert(login_bonus_preset).values(
-            id=preset_id,
+            presetId=preset_id,
             version=version,
             presetName=preset_name,
             isEnabled=is_enabled,
@@ -289,14 +302,14 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(sql)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"update_event: failed to update event! version: {version}, event_id: {event_id}, enabled: {enabled}"
             )
             return None
 
         event = self.get_event(version, event_id)
         if event is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"update_event: failed to fetch event {event_id} after updating"
             )
             return None
@@ -416,6 +429,14 @@ class ChuniStaticData(BaseData):
             return None
         return result.fetchall()
 
+    def get_music(self, version: int) -> Optional[List[Row]]:
+        sql = music.select(music.c.version <= version)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def get_music_chart(
         self, version: int, song_id: int, chart_id: int
     ) -> Optional[List[Row]]:
@@ -485,7 +506,7 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert gacha! gacha_id {gacha_id}")
+            self.logger.warning(f"Failed to insert gacha! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
@@ -520,7 +541,7 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert gacha card! gacha_id {gacha_id}")
+            self.logger.warning(f"Failed to insert gacha card! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
@@ -556,7 +577,7 @@ class ChuniStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert card! card_id {card_id}")
+            self.logger.warning(f"Failed to insert card! card_id {card_id}")
             return None
         return result.lastrowid
 

titles/chuni/sun.py

@@ -0,0 +1,37 @@
+from typing import Dict, Any
+
+from core.config import CoreConfig
+from titles.chuni.newplus import ChuniNewPlus
+from titles.chuni.const import ChuniConstants
+from titles.chuni.config import ChuniConfig
+
+
+class ChuniSun(ChuniNewPlus):
+    def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
+        super().__init__(core_cfg, game_cfg)
+        self.version = ChuniConstants.VER_CHUNITHM_SUN
+
+    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = super().handle_get_game_setting_api_request(data)
+        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)["rom"]
+        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)["data"]
+        ret["gameSetting"][
+            "matchingUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "matchingUriX"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "udpHolePunchUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "reflectorUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        return ret
+
+    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        user_data = super().handle_cm_get_user_preview_api_request(data)
+
+        # hardcode lastDataVersion for CardMaker 1.35 A032
+        user_data["lastDataVersion"] = "2.10.00"
+        return user_data

titles/cm/base.py

@@ -23,19 +23,40 @@ class CardMakerBase:
         self.game = CardMakerConstants.GAME_CODE
         self.version = CardMakerConstants.VER_CARD_MAKER
 
+    @staticmethod
+    def _parse_int_ver(version: str) -> str:
+        return version.replace(".", "")[:3]
+
     def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
         if self.core_cfg.server.is_develop:
             uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
         else:
             uri = f"http://{self.core_cfg.title.hostname}"
 
-        # CHUNITHM = 0, maimai = 1, ONGEKI = 2
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "length": 3,
             "gameConnectList": [
-                {"modelKind": 0, "type": 1, "titleUri": f"{uri}/SDHD/200/"},
-                {"modelKind": 1, "type": 1, "titleUri": f"{uri}/SDEZ/120/"},
-                {"modelKind": 2, "type": 1, "titleUri": f"{uri}/SDDT/130/"},
+                # CHUNITHM
+                {
+                    "modelKind": 0,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDHD/{self._parse_int_ver(games_ver['chuni'])}/",
+                },
+                # maimai DX
+                {
+                    "modelKind": 1,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDEZ/{self._parse_int_ver(games_ver['maimai'])}/",
+                },
+                # ONGEKI
+                {
+                    "modelKind": 2,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDDT/{self._parse_int_ver(games_ver['ongeki'])}/",
+                },
             ],
         }
 
@@ -47,12 +68,15 @@ class CardMakerBase:
             datetime.now() + timedelta(hours=4), self.date_time_format
         )
 
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "gameSetting": {
                 "dataVersion": "1.30.00",
-                "ongekiCmVersion": "1.30.01",
-                "chuniCmVersion": "2.00.00",
-                "maimaiCmVersion": "1.20.00",
+                "ongekiCmVersion": games_ver["ongeki"],
+                "chuniCmVersion": games_ver["chuni"],
+                "maimaiCmVersion": games_ver["maimai"],
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,

titles/cm/cm135.py

@@ -1,8 +1,4 @@
-from datetime import date, datetime, timedelta
-from typing import Any, Dict, List
-import json
-import logging
-from enum import Enum
+from typing import Dict
 
 from core.config import CoreConfig
 from core.data.cache import cached
@@ -16,23 +12,7 @@ class CardMaker135(CardMakerBase):
         super().__init__(core_cfg, game_cfg)
         self.version = CardMakerConstants.VER_CARD_MAKER_135
 
-    def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_connect_api_request(data)
-        if self.core_cfg.server.is_develop:
-            uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
-        else:
-            uri = f"http://{self.core_cfg.title.hostname}"
-
-        ret["gameConnectList"][0]["titleUri"] = f"{uri}/SDHD/205/"
-        ret["gameConnectList"][1]["titleUri"] = f"{uri}/SDEZ/125/"
-        ret["gameConnectList"][2]["titleUri"] = f"{uri}/SDDT/135/"
-
-        return ret
-
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
-        ret["gameSetting"]["ongekiCmVersion"] = "1.35.03"
-        ret["gameSetting"]["chuniCmVersion"] = "2.05.00"
-        ret["gameSetting"]["maimaiCmVersion"] = "1.25.00"
         return ret

titles/cm/config.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from core.config import CoreConfig
 
 
@@ -20,6 +21,32 @@ class CardMakerServerConfig:
         )
 
 
+class CardMakerVersionConfig:
+    def __init__(self, parent_config: "CardMakerConfig") -> None:
+        self.__config = parent_config
+
+    def version(self, version: int) -> Dict:
+        """
+        in the form of:
+        1: {"ongeki": 1.30.01, "chuni": 2.00.00, "maimai": 1.20.00}
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "cardmaker", "version", default={
+                0: {    
+                    "ongeki": "1.30.01",
+                    "chuni": "2.00.00",
+                    "maimai": "1.20.00"
+                },
+                1: {
+                    "ongeki": "1.35.03",
+                    "chuni": "2.10.00",
+                    "maimai": "1.30.00"
+                }
+            }
+        )[version]
+
+
 class CardMakerConfig(dict):
     def __init__(self) -> None:
         self.server = CardMakerServerConfig(self)
+        self.version = CardMakerVersionConfig(self)

titles/cm/const.py

@@ -6,7 +6,7 @@ class CardMakerConstants:
     VER_CARD_MAKER = 0
     VER_CARD_MAKER_135 = 1
 
-    VERSION_NAMES = ("Card Maker 1.34", "Card Maker 1.35")
+    VERSION_NAMES = ("Card Maker 1.30", "Card Maker 1.35")
 
     @classmethod
     def game_ver_to_string(cls, ver: int):

titles/cm/index.py

@@ -30,7 +30,7 @@ class CardMakerServlet:
 
         self.versions = [
             CardMakerBase(core_cfg, self.game_cfg),
-            CardMaker135(core_cfg, self.game_cfg),
+            CardMaker135(core_cfg, self.game_cfg)
         ]
 
         self.logger = logging.getLogger("cardmaker")
@@ -85,11 +85,9 @@ class CardMakerServlet:
         endpoint = url_split[len(url_split) - 1]
         client_ip = Utils.get_ip_addr(request)
 
-        print(f"version: {version}")
-
         if version >= 130 and version < 135:  # Card Maker
             internal_ver = CardMakerConstants.VER_CARD_MAKER
-        elif version >= 135 and version < 136:  # Card Maker 1.35
+        elif version >= 135 and version < 140:  # Card Maker 1.35
             internal_ver = CardMakerConstants.VER_CARD_MAKER_135
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
@@ -124,11 +122,12 @@ class CardMakerServlet:
 
         except Exception as e:
             self.logger.error(f"Error handling v{version} method {endpoint} - {e}")
+            raise
             return zlib.compress(b'{"stat": "0"}')
 
         if resp is None:
             resp = {"returnCode": 1}
 
-        self.logger.info(f"Response {resp}")
+        self.logger.debug(f"Response {resp}")
 
         return zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))

titles/cm/read.py

@@ -68,7 +68,7 @@ class CardMakerReader(BaseReader):
                     read_csv = getattr(CardMakerReader, func)
                     read_csv(self, f"{self.bin_dir}/MU3/{file}")
                 else:
-                    self.logger.warn(
+                    self.logger.warning(
                         f"Couldn't find {file} file in {self.bin_dir}, skipping"
                     )
 
@@ -89,8 +89,7 @@ class CardMakerReader(BaseReader):
         version_ids = {
             "v2_00": ChuniConstants.VER_CHUNITHM_NEW,
             "v2_05": ChuniConstants.VER_CHUNITHM_NEW_PLUS,
-            # Chunithm SUN, ignore for now
-            "v2_10": ChuniConstants.VER_CHUNITHM_NEW_PLUS + 1,
+            "v2_10": ChuniConstants.VER_CHUNITHM_SUN,
         }
 
         for root, dirs, files in os.walk(base_dir):
@@ -138,8 +137,7 @@ class CardMakerReader(BaseReader):
         version_ids = {
             "v2_00": ChuniConstants.VER_CHUNITHM_NEW,
             "v2_05": ChuniConstants.VER_CHUNITHM_NEW_PLUS,
-            # Chunithm SUN, ignore for now
-            "v2_10": ChuniConstants.VER_CHUNITHM_NEW_PLUS + 1,
+            "v2_10": ChuniConstants.VER_CHUNITHM_SUN,
         }
 
         for root, dirs, files in os.walk(base_dir):
@@ -226,6 +224,12 @@ class CardMakerReader(BaseReader):
                             True if troot.find("disable").text == "false" else False
                         )
 
+                        # check if a date is part of the name and disable the
+                        # card if it is
+                        enabled = (
+                            False if re.search(r"\d{2}/\d{2}/\d{2}", name) else enabled
+                        )
+
                         self.mai2_data.static.put_card(
                             version, card_id, name, enabled=enabled
                         )

titles/cxb/base.py

@@ -2,7 +2,7 @@ import logging
 import json
 from decimal import Decimal
 from base64 import b64encode
-from typing import Any, Dict
+from typing import Any, Dict, List
 from hashlib import md5
 from datetime import datetime
 
@@ -11,6 +11,7 @@ from titles.cxb.config import CxbConfig
 from titles.cxb.const import CxbConstants
 from titles.cxb.database import CxbData
 
+from threading import Thread
 
 class CxbBase:
     def __init__(self, cfg: CoreConfig, game_cfg: CxbConfig) -> None:
@@ -51,9 +52,154 @@ class CxbBase:
             self.logger.info(f"Login user {data['login']['authid']}")
             return {"token": data["login"]["authid"], "uid": data["login"]["authid"]}
 
-        self.logger.warn(f"User {data['login']['authid']} does not have a profile")
+        self.logger.warning(f"User {data['login']['authid']} does not have a profile")
         return {}
 
+    def task_generateCoupon(index, data1):
+        # Coupons
+        for i in range(500, 510):
+            index.append(str(i))
+            couponid = int(i) - 500
+            dataValue = [
+                {
+                    "couponId": str(couponid),
+                    "couponNum": "1",
+                    "couponLog": [],
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateShopListTitle(index, data1):
+        # ShopList_Title
+        for i in range(200000, 201451):
+            index.append(str(i))
+            shopid = int(i) - 200000
+            dataValue = [
+                {
+                    "shopId": shopid,
+                    "shopState": "2",
+                    "isDisable": "t",
+                    "isDeleted": "f",
+                    "isSpecialFlag": "f",
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateShopListIcon(index, data1):
+        # ShopList_Icon
+        for i in range(202000, 202264):
+            index.append(str(i))
+            shopid = int(i) - 200000
+            dataValue = [
+                {
+                    "shopId": shopid,
+                    "shopState": "2",
+                    "isDisable": "t",
+                    "isDeleted": "f",
+                    "isSpecialFlag": "f",
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateStories(index, data1):
+        # Stories
+        for i in range(900000, 900003):
+            index.append(str(i))
+            storyid = int(i) - 900000
+            dataValue = [
+                {
+                    "storyId": storyid,
+                    "unlockState1": ["t"] * 10,
+                    "unlockState2": ["t"] * 10,
+                    "unlockState3": ["t"] * 10,
+                    "unlockState4": ["t"] * 10,
+                    "unlockState5": ["t"] * 10,
+                    "unlockState6": ["t"] * 10,
+                    "unlockState7": ["t"] * 10,
+                    "unlockState8": ["t"] * 10,
+                    "unlockState9": ["t"] * 10,
+                    "unlockState10": ["t"] * 10,
+                    "unlockState11": ["t"] * 10,
+                    "unlockState12": ["t"] * 10,
+                    "unlockState13": ["t"] * 10,
+                    "unlockState14": ["t"] * 10,
+                    "unlockState15": ["t"] * 10,
+                    "unlockState16": ["t"] * 10,
+                }
+            ]
+            data1.append(
+                b64encode(
+                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
+                ).decode("utf-8")
+            )
+
+    def task_generateScoreData(song, index, data1):
+        song_data = song["data"]
+        songCode = []
+
+        songCode.append(
+            {
+                "mcode": song_data["mcode"],
+                "musicState": song_data["musicState"],
+                "playCount": song_data["playCount"],
+                "totalScore": song_data["totalScore"],
+                "highScore": song_data["highScore"],
+                "everHighScore": song_data["everHighScore"]
+                if "everHighScore" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "clearRate": song_data["clearRate"],
+                "rankPoint": song_data["rankPoint"],
+                "normalCR": song_data["normalCR"]
+                if "normalCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "survivalCR": song_data["survivalCR"]
+                if "survivalCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "ultimateCR": song_data["ultimateCR"]
+                if "ultimateCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "nohopeCR": song_data["nohopeCR"]
+                if "nohopeCR" in song_data
+                else ["0", "0", "0", "0", "0"],
+                "combo": song_data["combo"],
+                "coupleUserId": song_data["coupleUserId"],
+                "difficulty": song_data["difficulty"],
+                "isFullCombo": song_data["isFullCombo"],
+                "clearGaugeType": song_data["clearGaugeType"],
+                "fieldType": song_data["fieldType"],
+                "gameType": song_data["gameType"],
+                "grade": song_data["grade"],
+                "unlockState": song_data["unlockState"],
+                "extraState": song_data["extraState"],
+            }
+        )
+        index.append(song_data["index"])
+        data1.append(
+            b64encode(
+                bytes(json.dumps(songCode[0], separators=(",", ":")), "utf-8")
+            ).decode("utf-8")
+        )
+
+    def task_generateIndexData(versionindex):
+        try:
+            v_profile = self.data.profile.get_profile_index(0, uid, self.version)
+            v_profile_data = v_profile["data"]
+            versionindex.append(int(v_profile_data["appVersion"]))
+        except Exception:
+            versionindex.append("10400")
+
     def handle_action_loadrange_request(self, data: Dict) -> Dict:
         range_start = data["loadrange"]["range"][0]
         range_end = data["loadrange"]["range"][1]
@@ -107,146 +253,29 @@ class CxbBase:
         900000 = Stories
         """
 
-        # Coupons
-        for i in range(500, 510):
-            index.append(str(i))
-            couponid = int(i) - 500
-            dataValue = [
-                {
-                    "couponId": str(couponid),
-                    "couponNum": "1",
-                    "couponLog": [],
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
+        # Async threads to generate the response
+        thread_Coupon = Thread(target=CxbBase.task_generateCoupon(index, data1))
+        thread_ShopListTitle = Thread(target=CxbBase.task_generateShopListTitle(index, data1))
+        thread_ShopListIcon = Thread(target=CxbBase.task_generateShopListIcon(index, data1))
+        thread_Stories = Thread(target=CxbBase.task_generateStories(index, data1))
 
-        # ShopList_Title
-        for i in range(200000, 201451):
-            index.append(str(i))
-            shopid = int(i) - 200000
-            dataValue = [
-                {
-                    "shopId": shopid,
-                    "shopState": "2",
-                    "isDisable": "t",
-                    "isDeleted": "f",
-                    "isSpecialFlag": "f",
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
+        thread_Coupon.start()
+        thread_ShopListTitle.start()
+        thread_ShopListIcon.start()
+        thread_Stories.start()
 
-        # ShopList_Icon
-        for i in range(202000, 202264):
-            index.append(str(i))
-            shopid = int(i) - 200000
-            dataValue = [
-                {
-                    "shopId": shopid,
-                    "shopState": "2",
-                    "isDisable": "t",
-                    "isDeleted": "f",
-                    "isSpecialFlag": "f",
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
-
-        # Stories
-        for i in range(900000, 900003):
-            index.append(str(i))
-            storyid = int(i) - 900000
-            dataValue = [
-                {
-                    "storyId": storyid,
-                    "unlockState1": ["t"] * 10,
-                    "unlockState2": ["t"] * 10,
-                    "unlockState3": ["t"] * 10,
-                    "unlockState4": ["t"] * 10,
-                    "unlockState5": ["t"] * 10,
-                    "unlockState6": ["t"] * 10,
-                    "unlockState7": ["t"] * 10,
-                    "unlockState8": ["t"] * 10,
-                    "unlockState9": ["t"] * 10,
-                    "unlockState10": ["t"] * 10,
-                    "unlockState11": ["t"] * 10,
-                    "unlockState12": ["t"] * 10,
-                    "unlockState13": ["t"] * 10,
-                    "unlockState14": ["t"] * 10,
-                    "unlockState15": ["t"] * 10,
-                    "unlockState16": ["t"] * 10,
-                }
-            ]
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(dataValue[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
+        thread_Coupon.join()
+        thread_ShopListTitle.join()
+        thread_ShopListIcon.join()
+        thread_Stories.join()
 
         for song in songs:
-            song_data = song["data"]
-            songCode = []
-
-            songCode.append(
-                {
-                    "mcode": song_data["mcode"],
-                    "musicState": song_data["musicState"],
-                    "playCount": song_data["playCount"],
-                    "totalScore": song_data["totalScore"],
-                    "highScore": song_data["highScore"],
-                    "everHighScore": song_data["everHighScore"]
-                    if "everHighScore" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "clearRate": song_data["clearRate"],
-                    "rankPoint": song_data["rankPoint"],
-                    "normalCR": song_data["normalCR"]
-                    if "normalCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "survivalCR": song_data["survivalCR"]
-                    if "survivalCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "ultimateCR": song_data["ultimateCR"]
-                    if "ultimateCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "nohopeCR": song_data["nohopeCR"]
-                    if "nohopeCR" in song_data
-                    else ["0", "0", "0", "0", "0"],
-                    "combo": song_data["combo"],
-                    "coupleUserId": song_data["coupleUserId"],
-                    "difficulty": song_data["difficulty"],
-                    "isFullCombo": song_data["isFullCombo"],
-                    "clearGaugeType": song_data["clearGaugeType"],
-                    "fieldType": song_data["fieldType"],
-                    "gameType": song_data["gameType"],
-                    "grade": song_data["grade"],
-                    "unlockState": song_data["unlockState"],
-                    "extraState": song_data["extraState"],
-                }
-            )
-            index.append(song_data["index"])
-            data1.append(
-                b64encode(
-                    bytes(json.dumps(songCode[0], separators=(",", ":")), "utf-8")
-                ).decode("utf-8")
-            )
+            thread_ScoreData = Thread(target=CxbBase.task_generateScoreData(song, index, data1))
+            thread_ScoreData.start()
 
         for v in index:
-            try:
-                v_profile = self.data.profile.get_profile_index(0, uid, self.version)
-                v_profile_data = v_profile["data"]
-                versionindex.append(int(v_profile_data["appVersion"]))
-            except:
-                versionindex.append("10400")
+            thread_IndexData = Thread(target=CxbBase.task_generateIndexData(versionindex))
+            thread_IndexData.start()
 
         return {"index": index, "data": data1, "version": versionindex}
 
@@ -257,7 +286,7 @@ class CxbBase:
             # REV Omnimix Version Fetcher
             gameversion = data["saveindex"]["data"][0][2]
             self.logger.warning(f"Game Version is {gameversion}")
-        except:
+        except Exception:
             pass
 
         if "10205" in gameversion:
@@ -319,7 +348,7 @@ class CxbBase:
         # Sunrise
         try:
             profileIndex = save_data["index"].index("0")
-        except:
+        except Exception:
             return {"data": ""}  # Maybe
 
         profile = json.loads(save_data["data"][profileIndex])
@@ -416,7 +445,7 @@ class CxbBase:
         self.logger.info(f"Get best rankings for {uid}")
         p = self.data.score.get_best_rankings(uid)
 
-        rankList: list[Dict[str, Any]] = []
+        rankList: List[Dict[str, Any]] = []
 
         for rank in p:
             if rank["song_id"] is not None:
@@ -467,7 +496,7 @@ class CxbBase:
                         score=int(rid["sc"][0]),
                         clear=rid["clear"],
                     )
-                except:
+                except Exception:
                     self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
@@ -485,7 +514,7 @@ class CxbBase:
                         score=int(rid["sc"][0]),
                         clear=0,
                     )
-                except:
+                except Exception:
                     self.data.score.put_ranking(
                         user_id=uid,
                         rev_id=int(rid["rid"]),
@@ -544,4 +573,4 @@ class CxbBase:
 
     def handle_action_stampreq_request(self, data: Dict) -> Dict:
         self.logger.info(data)
-        return {"stampreq": ""}
+        return {"stampreq": ""}

titles/cxb/index.py

@@ -103,7 +103,7 @@ class CxbServlet(resource.Resource):
             else:
                 self.logger.info(f"Ready on port {self.game_cfg.server.port}")
 
-    def render_POST(self, request: Request):
+    def render_POST(self, request: Request, version: int, endpoint: str):
         version = 0
         internal_ver = 0
         func_to_find = ""
@@ -123,13 +123,13 @@ class CxbServlet(resource.Resource):
                 )
 
             except Exception as f:
-                self.logger.warn(
+                self.logger.warning(
                     f"Error decoding json: {e} / {f} - {req_url} - {req_bytes}"
                 )
                 return b""
 
         if req_json == {}:
-            self.logger.warn(f"Empty json request to {req_url}")
+            self.logger.warning(f"Empty json request to {req_url}")
             return b""
 
         cmd = url_split[len(url_split) - 1]
@@ -140,7 +140,7 @@ class CxbServlet(resource.Resource):
                 not type(req_json["dldate"]) is dict
                 or "filetype" not in req_json["dldate"]
             ):
-                self.logger.warn(f"Malformed dldate request: {req_url} {req_json}")
+                self.logger.warning(f"Malformed dldate request: {req_url} {req_json}")
                 return b""
 
             filetype = req_json["dldate"]["filetype"]

titles/cxb/read.py

@@ -33,7 +33,7 @@ class CxbReader(BaseReader):
         pull_bin_ram = True
 
         if not path.exists(f"{self.bin_dir}"):
-            self.logger.warn(f"Couldn't find csv file in {self.bin_dir}, skipping")
+            self.logger.warning(f"Couldn't find csv file in {self.bin_dir}, skipping")
             pull_bin_ram = False
 
         if pull_bin_ram:
@@ -123,5 +123,5 @@ class CxbReader(BaseReader):
                             genre,
                             int(row["easy"].replace("Easy ", "").replace("N/A", "0")),
                         )
-        except:
-            self.logger.warn(f"Couldn't read csv file in {self.bin_dir}, skipping")
+        except Exception:
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")

titles/diva/base.py

@@ -2,7 +2,8 @@ from datetime import datetime
 from typing import Any, List, Dict
 import logging
 import json
-from urllib import parse
+import urllib
+from threading import Thread
 
 from core.config import CoreConfig
 from titles.diva.config import DivaConfig
@@ -661,50 +662,66 @@ class DivaBase:
 
         return pv_result
 
-    def handle_get_pv_pd_request(self, data: bytes) -> str:
-        song_id = data["pd_pv_id_lst"].split(",")
-        pv = ""
+    def task_generateScoreData(self, data: Dict, pd_by_pv_id, song):
+        
+        if int(song) > 0:
+            # the request do not send a edition so just perform a query best score and ranking for each edition.
+            # 0=ORIGINAL, 1=EXTRA
+            pd_db_song_0 = self.data.score.get_best_user_score(
+                data["pd_id"], int(song), data["difficulty"], edition=0
+            )
+            pd_db_song_1 = self.data.score.get_best_user_score(
+                data["pd_id"], int(song), data["difficulty"], edition=1
+            )
 
-        for song in song_id:
-            if int(song) > 0:
-                # the request do not send a edition so just perform a query best score and ranking for each edition.
-                # 0=ORIGINAL, 1=EXTRA
-                pd_db_song_0 = self.data.score.get_best_user_score(
+            pd_db_ranking_0, pd_db_ranking_1 = None, None
+            if pd_db_song_0:
+                pd_db_ranking_0 = self.data.score.get_global_ranking(
                     data["pd_id"], int(song), data["difficulty"], edition=0
                 )
-                pd_db_song_1 = self.data.score.get_best_user_score(
+
+            if pd_db_song_1:
+                pd_db_ranking_1 = self.data.score.get_global_ranking(
                     data["pd_id"], int(song), data["difficulty"], edition=1
                 )
 
-                pd_db_ranking_0, pd_db_ranking_1 = None, None
-                if pd_db_song_0:
-                    pd_db_ranking_0 = self.data.score.get_global_ranking(
-                        data["pd_id"], int(song), data["difficulty"], edition=0
-                    )
+            pd_db_customize = self.data.pv_customize.get_pv_customize(
+                data["pd_id"], int(song)
+            )
 
-                if pd_db_song_1:
-                    pd_db_ranking_1 = self.data.score.get_global_ranking(
-                        data["pd_id"], int(song), data["difficulty"], edition=1
-                    )
+            # generate the pv_result string with the ORIGINAL edition and the EXTRA edition appended
+            pv_result = self._get_pv_pd_result(
+                int(song), pd_db_song_0, pd_db_ranking_0, pd_db_customize, edition=0
+            )
+            pv_result += "," + self._get_pv_pd_result(
+                int(song), pd_db_song_1, pd_db_ranking_1, pd_db_customize, edition=1
+            )
 
-                pd_db_customize = self.data.pv_customize.get_pv_customize(
-                    data["pd_id"], int(song)
-                )
+            self.logger.debug(f"pv_result = {pv_result}")
+            pd_by_pv_id.append(urllib.parse.quote(pv_result))
+        else:
+            pd_by_pv_id.append(urllib.parse.quote(f"{song}***"))
+        pd_by_pv_id.append(",")
 
-                # generate the pv_result string with the ORIGINAL edition and the EXTRA edition appended
-                pv_result = self._get_pv_pd_result(
-                    int(song), pd_db_song_0, pd_db_ranking_0, pd_db_customize, edition=0
-                )
-                pv_result += "," + self._get_pv_pd_result(
-                    int(song), pd_db_song_1, pd_db_ranking_1, pd_db_customize, edition=1
-                )
+    def handle_get_pv_pd_request(self, data: Dict) -> Dict:
+        song_id = data["pd_pv_id_lst"].split(",")
+        pv = ""
 
-                self.logger.debug(f"pv_result = {pv_result}")
+        threads = []
+        pd_by_pv_id = []
 
-                pv += parse.quote(pv_result)
-            else:
-                pv += parse.quote(f"{song}***")
-            pv += ","
+        for song in song_id:
+            thread_ScoreData = Thread(target=self.task_generateScoreData(data, pd_by_pv_id, song))
+            threads.append(thread_ScoreData)
+
+        for x in threads:
+            x.start()
+
+        for x in threads:
+            x.join()
+
+        for x in pd_by_pv_id:
+            pv += x
 
         response = ""
         response += f"&pd_by_pv_id={pv[:-1]}"

titles/diva/read.py

@@ -34,18 +34,18 @@ class DivaReader(BaseReader):
         pull_opt_rom = True
 
         if not path.exists(f"{self.bin_dir}/ram"):
-            self.logger.warn(f"Couldn't find ram folder in {self.bin_dir}, skipping")
+            self.logger.warning(f"Couldn't find ram folder in {self.bin_dir}, skipping")
             pull_bin_ram = False
 
         if not path.exists(f"{self.bin_dir}/rom"):
-            self.logger.warn(f"Couldn't find rom folder in {self.bin_dir}, skipping")
+            self.logger.warning(f"Couldn't find rom folder in {self.bin_dir}, skipping")
             pull_bin_rom = False
 
         if self.opt_dir is not None:
             opt_dirs = self.get_data_directories(self.opt_dir)
         else:
             pull_opt_rom = False
-            self.logger.warn("No option directory specified, skipping")
+            self.logger.warning("No option directory specified, skipping")
 
         if pull_bin_ram:
             self.read_ram(f"{self.bin_dir}/ram")
@@ -139,7 +139,7 @@ class DivaReader(BaseReader):
                                 else:
                                     continue
         else:
-            self.logger.warn(f"Databank folder not found in {ram_root_dir}, skipping")
+            self.logger.warning(f"Databank folder not found in {ram_root_dir}, skipping")
 
     def read_rom(self, rom_root_dir: str) -> None:
         self.logger.info(f"Read ROM from {rom_root_dir}")
@@ -150,7 +150,7 @@ class DivaReader(BaseReader):
         elif path.exists(f"{rom_root_dir}/pv_db.txt"):
             file_path = f"{rom_root_dir}/pv_db.txt"
         else:
-            self.logger.warn(
+            self.logger.warning(
                 f"Cannot find pv_db.txt or mdata_pv_db.txt in {rom_root_dir}, skipping"
             )
             return

titles/idz/userdb.py

@@ -83,7 +83,13 @@ class IDZUserDBProtocol(Protocol):
     def dataReceived(self, data: bytes) -> None:
         self.logger.debug(f"Receive data {data.hex()}")
         crypt = AES.new(self.static_key, AES.MODE_ECB)
-        data_dec = crypt.decrypt(data)
+        
+        try:
+            data_dec = crypt.decrypt(data)
+        
+        except Exception as e:
+            self.logger.error(f"Failed to decrypt UserDB request from {self.transport.getPeer().host} because {e} - {data.hex()}")
+        
         self.logger.debug(f"Decrypt data {data_dec.hex()}")
 
         magic = struct.unpack_from("<I", data_dec, 0)[0]
@@ -108,7 +114,7 @@ class IDZUserDBProtocol(Protocol):
             elif self.version == 230:
                 self.version_internal = IDZConstants.VER_IDZ_230
             else:
-                self.logger.warn(f"Bad version v{self.version}")
+                self.logger.warning(f"Bad version v{self.version}")
                 self.version = None
                 self.version_internal = None
 
@@ -136,7 +142,7 @@ class IDZUserDBProtocol(Protocol):
             self.version_internal
         ].get(cmd, None)
         if handler_cls is None:
-            self.logger.warn(f"No handler for v{self.version} {hex(cmd)} cmd")
+            self.logger.warning(f"No handler for v{self.version} {hex(cmd)} cmd")
             handler_cls = IDZHandlerBase
 
         handler = handler_cls(self.core_config, self.game_config, self.version_internal)

titles/mai2/__init__.py

@@ -6,5 +6,14 @@ from titles.mai2.read import Mai2Reader
 index = Mai2Servlet
 database = Mai2Data
 reader = Mai2Reader
-game_codes = [Mai2Constants.GAME_CODE]
-current_schema_version = 4
+game_codes = [
+    Mai2Constants.GAME_CODE_DX,
+    Mai2Constants.GAME_CODE_FINALE,
+    Mai2Constants.GAME_CODE_MILK,
+    Mai2Constants.GAME_CODE_MURASAKI,
+    Mai2Constants.GAME_CODE_PINK,
+    Mai2Constants.GAME_CODE_ORANGE,
+    Mai2Constants.GAME_CODE_GREEN,
+    Mai2Constants.GAME_CODE,
+]
+current_schema_version = 7

titles/mai2/base.py

@@ -1,6 +1,9 @@
-from datetime import datetime, date, timedelta
-from typing import Any, Dict
+from datetime import datetime
+from typing import Any, Dict, List
 import logging
+from base64 import b64decode
+from os import path, stat, remove
+from PIL import ImageFile
 
 from core.config import CoreConfig
 from titles.mai2.const import Mai2Constants
@@ -12,33 +15,35 @@ class Mai2Base:
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         self.core_config = cfg
         self.game_config = game_cfg
-        self.game = Mai2Constants.GAME_CODE
-        self.version = Mai2Constants.VER_MAIMAI_DX
+        self.version = Mai2Constants.VER_MAIMAI
         self.data = Mai2Data(cfg)
         self.logger = logging.getLogger("mai2")
+        self.can_deliver = False
+        self.can_usbdl = False
+        self.old_server = ""
+        
+        if self.core_config.server.is_develop and self.core_config.title.port > 0:
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEY/197/"
+        
+        else:
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEY/197/"
 
     def handle_get_game_setting_api_request(self, data: Dict):
-        reboot_start = date.strftime(
-            datetime.now() + timedelta(hours=3), Mai2Constants.DATE_TIME_FORMAT
-        )
-        reboot_end = date.strftime(
-            datetime.now() + timedelta(hours=4), Mai2Constants.DATE_TIME_FORMAT
-        )
-        return {
+        return {            
+            "isDevelop": False,
+            "isAouAccession": False,
             "gameSetting": {
-                "isMaintenance": "false",
-                "requestInterval": 10,
-                "rebootStartTime": reboot_start,
-                "rebootEndTime": reboot_end,
-                "movieUploadLimit": 10000,
-                "movieStatus": 0,
-                "movieServerUri": "",
-                "deliverServerUri": "",
-                "oldServerUri": "",
-                "usbDlServerUri": "",
-                "rebootInterval": 0,
+                "isMaintenance": False,
+                "requestInterval": 1800,
+                "rebootStartTime": "2020-01-01 07:00:00.0",
+                "rebootEndTime": "2020-01-01 07:59:59.0",
+                "movieUploadLimit": 100,
+                "movieStatus": 1,
+                "movieServerUri": self.old_server + "api/movie" if self.game_config.uploads.movies else "movie",
+                "deliverServerUri": self.old_server + "deliver/" if self.can_deliver and self.game_config.deliver.enable else "",
+                "oldServerUri": self.old_server + "old",
+                "usbDlServerUri": self.old_server + "usbdl/" if self.can_deliver and self.game_config.deliver.udbdl_enable else "",
             },
-            "isAouAccession": "true",
         }
 
     def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
@@ -51,8 +56,8 @@ class Mai2Base:
     def handle_get_game_event_api_request(self, data: Dict) -> Dict:
         events = self.data.static.get_enabled_events(self.version)
         events_lst = []
-        if events is None:
-            self.logger.warn("No enabled events, did you run the reader?")
+        if events is None or not events:
+            self.logger.warning("No enabled events, did you run the reader?")
             return {"type": data["type"], "length": 0, "gameEventList": []}
 
         for event in events:
@@ -87,7 +92,7 @@ class Mai2Base:
         for i, charge in enumerate(game_charge_list):
             charge_list.append(
                 {
-                    "orderId": i,
+                    "orderId": i + 1,
                     "chargeId": charge["ticketId"],
                     "price": charge["price"],
                     "startDate": "2017-12-05 07:00:00.0",
@@ -110,43 +115,35 @@ class Mai2Base:
         return {"returnCode": 1, "apiName": "UpsertClientTestmodeApi"}
 
     def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_detail(data["userId"], self.version)
-        o = self.data.profile.get_profile_option(data["userId"], self.version)
-        if p is None or o is None:
+        p = self.data.profile.get_profile_detail(data["userId"], self.version, False)
+        w = self.data.profile.get_web_option(data["userId"], self.version)
+        if p is None or w is None:
             return {}  # Register
         profile = p._asdict()
-        option = o._asdict()
+        web_opt = w._asdict()
 
         return {
             "userId": data["userId"],
             "userName": profile["userName"],
             "isLogin": False,
-            "lastGameId": profile["lastGameId"],
             "lastDataVersion": profile["lastDataVersion"],
-            "lastRomVersion": profile["lastRomVersion"],
-            "lastLoginDate": profile["lastLoginDate"],
+            "lastLoginDate": profile["lastPlayDate"],
             "lastPlayDate": profile["lastPlayDate"],
             "playerRating": profile["playerRating"],
-            "nameplateId": 0,  # Unused
-            "iconId": profile["iconId"],
-            "trophyId": 0,  # Unused
-            "partnerId": profile["partnerId"],
+            "nameplateId": profile["nameplateId"],
             "frameId": profile["frameId"],
-            "dispRate": option[
-                "dispRate"
-            ],  # 0: all/begin, 1: disprate, 2: dispDan, 3: hide, 4: end
-            "totalAwake": profile["totalAwake"],
-            "isNetMember": profile["isNetMember"],
-            "dailyBonusDate": profile["dailyBonusDate"],
-            "headPhoneVolume": option["headPhoneVolume"],
-            "isInherit": False,  # Not sure what this is or does??
-            "banState": profile["banState"]
-            if profile["banState"] is not None
-            else 0,  # New with uni+
+            "iconId": profile["iconId"],
+            "trophyId": profile["trophyId"],
+            "dispRate": web_opt["dispRate"],  # 0: all, 1: dispRate, 2: dispDan, 3: hide
+            "dispRank": web_opt["dispRank"],
+            "dispHomeRanker": web_opt["dispHomeRanker"],
+            "dispTotalLv": web_opt["dispTotalLv"],
+            "totalLv": profile["totalLv"],
         }
 
     def handle_user_login_api_request(self, data: Dict) -> Dict:
         profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        consec = self.data.profile.get_consec_login(data["userId"], self.version)
 
         if profile is not None:
             lastLoginDate = profile["lastLoginDate"]
@@ -157,13 +154,32 @@ class Mai2Base:
         else:
             loginCt = 0
             lastLoginDate = "2017-12-05 07:00:00.0"
+        
+        if consec is None or not consec:
+            consec_ct = 1
+        
+        else:
+            lastlogindate_ = datetime.strptime(profile["lastLoginDate"], "%Y-%m-%d %H:%M:%S.%f").timestamp()
+            today_midnight = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp()
+            yesterday_midnight = today_midnight - 86400
+
+            if lastlogindate_ < today_midnight:
+                consec_ct = consec['logins'] + 1                
+                self.data.profile.add_consec_login(data["userId"], self.version)
+            
+            elif lastlogindate_ < yesterday_midnight:
+                consec_ct = 1
+                self.data.profile.reset_consec_login(data["userId"], self.version)
+
+            else:
+                consec_ct = consec['logins']
+        
 
         return {
             "returnCode": 1,
             "lastLoginDate": lastLoginDate,
             "loginCount": loginCt,
-            "consecutiveLoginCount": 0,  # We don't really have a way to track this...
-            "loginId": loginCt,  # Used with the playlog!
+            "consecutiveLoginCount": consec_ct, # Number of consecutive days we've logged in.
         }
 
     def handle_upload_user_playlog_api_request(self, data: Dict) -> Dict:
@@ -195,11 +211,34 @@ class Mai2Base:
         upsert = data["upsertUserAll"]
 
         if "userData" in upsert and len(upsert["userData"]) > 0:
-            upsert["userData"][0]["isNetMember"] = 1
             upsert["userData"][0].pop("accessCode")
+            upsert["userData"][0].pop("userId")
+
             self.data.profile.put_profile_detail(
-                user_id, self.version, upsert["userData"][0]
+                user_id, self.version, upsert["userData"][0], False
             )
+        
+        if "userWebOption" in upsert and len(upsert["userWebOption"]) > 0:            
+            upsert["userWebOption"][0]["isNetMember"] = True
+            self.data.profile.put_web_option(
+                user_id, self.version, upsert["userWebOption"][0]
+            )
+
+        if "userGradeStatusList" in upsert and len(upsert["userGradeStatusList"]) > 0:
+            self.data.profile.put_grade_status(
+                user_id, upsert["userGradeStatusList"][0]
+            )
+
+        if "userBossList" in upsert and len(upsert["userBossList"]) > 0:
+            self.data.profile.put_boss_list(
+                user_id, upsert["userBossList"][0]
+            )
+
+        if "userPlaylogList" in upsert and len(upsert["userPlaylogList"]) > 0:
+            for playlog in upsert["userPlaylogList"]:
+                self.data.score.put_playlog(
+                    user_id, playlog, False
+                )
 
         if "userExtend" in upsert and len(upsert["userExtend"]) > 0:
             self.data.profile.put_profile_extend(
@@ -208,11 +247,15 @@ class Mai2Base:
 
         if "userGhost" in upsert:
             for ghost in upsert["userGhost"]:
-                self.data.profile.put_profile_extend(user_id, self.version, ghost)
+                self.data.profile.put_profile_ghost(user_id, self.version, ghost)
+
+        if "userRecentRatingList" in upsert:
+            self.data.profile.put_recent_rating(user_id, upsert["userRecentRatingList"])
 
         if "userOption" in upsert and len(upsert["userOption"]) > 0:
+            upsert["userOption"][0].pop("userId")
             self.data.profile.put_profile_option(
-                user_id, self.version, upsert["userOption"][0]
+                user_id, self.version, upsert["userOption"][0], False
             )
 
         if "userRatingList" in upsert and len(upsert["userRatingList"]) > 0:
@@ -221,9 +264,8 @@ class Mai2Base:
             )
 
         if "userActivityList" in upsert and len(upsert["userActivityList"]) > 0:
-            for k, v in upsert["userActivityList"][0].items():
-                for act in v:
-                    self.data.profile.put_profile_activity(user_id, act)
+            for act in upsert["userActivityList"]:
+                self.data.profile.put_profile_activity(user_id, act)
 
         if "userChargeList" in upsert and len(upsert["userChargeList"]) > 0:
             for charge in upsert["userChargeList"]:
@@ -243,12 +285,9 @@ class Mai2Base:
 
         if "userCharacterList" in upsert and len(upsert["userCharacterList"]) > 0:
             for char in upsert["userCharacterList"]:
-                self.data.item.put_character(
+                self.data.item.put_character_(
                     user_id,
-                    char["characterId"],
-                    char["level"],
-                    char["awakening"],
-                    char["useCount"],
+                    char
                 )
 
         if "userItemList" in upsert and len(upsert["userItemList"]) > 0:
@@ -258,7 +297,7 @@ class Mai2Base:
                     int(item["itemKind"]),
                     item["itemId"],
                     item["stock"],
-                    item["isValid"],
+                    True
                 )
 
         if "userLoginBonusList" in upsert and len(upsert["userLoginBonusList"]) > 0:
@@ -284,7 +323,7 @@ class Mai2Base:
 
         if "userMusicDetailList" in upsert and len(upsert["userMusicDetailList"]) > 0:
             for music in upsert["userMusicDetailList"]:
-                self.data.score.put_best_score(user_id, music)
+                self.data.score.put_best_score(user_id, music, False)
 
         if "userCourseList" in upsert and len(upsert["userCourseList"]) > 0:
             for course in upsert["userCourseList"]:
@@ -312,7 +351,7 @@ class Mai2Base:
         return {"returnCode": 1}
 
     def handle_get_user_data_api_request(self, data: Dict) -> Dict:
-        profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        profile = self.data.profile.get_profile_detail(data["userId"], self.version, False)
         if profile is None:
             return
 
@@ -336,7 +375,7 @@ class Mai2Base:
         return {"userId": data["userId"], "userExtend": extend_dict}
 
     def handle_get_user_option_api_request(self, data: Dict) -> Dict:
-        options = self.data.profile.get_profile_option(data["userId"], self.version)
+        options = self.data.profile.get_profile_option(data["userId"], self.version, False)
         if options is None:
             return
 
@@ -406,12 +445,31 @@ class Mai2Base:
             "userChargeList": user_charge_list,
         }
 
+    def handle_get_user_present_api_request(self, data: Dict) -> Dict:
+        return { "userId": data.get("userId", 0), "length": 0, "userPresentList": []}
+    
+    def handle_get_transfer_friend_api_request(self, data: Dict) -> Dict:
+        return {}
+
+    def handle_get_user_present_event_api_request(self, data: Dict) -> Dict:
+        return { "userId": data.get("userId", 0), "length": 0, "userPresentEventList": []}
+    
+    def handle_get_user_boss_api_request(self, data: Dict) -> Dict:
+        b = self.data.profile.get_boss_list(data["userId"])
+        if b is None:
+            return { "userId": data.get("userId", 0), "userBossData": {}}
+        boss_lst = b._asdict()
+        boss_lst.pop("id")
+        boss_lst.pop("user")
+
+        return { "userId": data.get("userId", 0), "userBossData": boss_lst}
+
     def handle_get_user_item_api_request(self, data: Dict) -> Dict:
         kind = int(data["nextIndex"] / 10000000000)
         next_idx = int(data["nextIndex"] % 10000000000)
         user_item_list = self.data.item.get_items(data["userId"], kind)
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(next_idx, len(user_item_list)):
             tmp = user_item_list[i]._asdict()
             tmp.pop("user")
@@ -442,6 +500,8 @@ class Mai2Base:
             tmp = chara._asdict()
             tmp.pop("id")
             tmp.pop("user")
+            tmp.pop("awakening")
+            tmp.pop("useCount")
             chara_list.append(tmp)
 
         return {"userId": data["userId"], "userCharacterList": chara_list}
@@ -475,6 +535,16 @@ class Mai2Base:
 
         return {"userId": data["userId"], "userGhost": ghost_dict}
 
+    def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
+        rating = self.data.profile.get_recent_rating(data["userId"])
+        if rating is None:
+            return
+        
+        r = rating._asdict()
+        lst = r.get("userRecentRatingList", [])
+        
+        return {"userId": data["userId"], "length": len(lst), "userRecentRatingList": lst}
+
     def handle_get_user_rating_api_request(self, data: Dict) -> Dict:
         rating = self.data.profile.get_profile_rating(data["userId"], self.version)
         if rating is None:
@@ -637,25 +707,158 @@ class Mai2Base:
 
     def handle_get_user_region_api_request(self, data: Dict) -> Dict:
         return {"userId": data["userId"], "length": 0, "userRegionList": []}
+    
+    def handle_get_user_web_option_api_request(self, data: Dict) -> Dict:
+        w = self.data.profile.get_web_option(data["userId"], self.version)
+        if  w is None:
+            return {"userId": data["userId"], "userWebOption": {}}
+        
+        web_opt = w._asdict()        
+        web_opt.pop("id")
+        web_opt.pop("user")
+        web_opt.pop("version")
+
+        return {"userId": data["userId"], "userWebOption": web_opt}
+
+    def handle_get_user_survival_api_request(self, data: Dict) -> Dict:
+        return {"userId": data["userId"], "length": 0, "userSurvivalList": []}
+
+    def handle_get_user_grade_api_request(self, data: Dict) -> Dict:
+        g = self.data.profile.get_grade_status(data["userId"])
+        if g is None:
+            return {"userId": data["userId"], "userGradeStatus": {}, "length": 0, "userGradeList": []}
+        grade_stat = g._asdict()
+        grade_stat.pop("id")
+        grade_stat.pop("user")
+
+        return {"userId": data["userId"], "userGradeStatus": grade_stat, "length": 0, "userGradeList": []}
 
     def handle_get_user_music_api_request(self, data: Dict) -> Dict:
-        songs = self.data.score.get_best_scores(data["userId"])
+        user_id = data.get("userId", 0)        
+        next_index = data.get("nextIndex", 0)
+        max_ct = data.get("maxCount", 50)
+        upper_lim = next_index + max_ct
         music_detail_list = []
-        next_index = 0
 
-        if songs is not None:
-            for song in songs:
-                tmp = song._asdict()
-                tmp.pop("id")
-                tmp.pop("user")
-                music_detail_list.append(tmp)
+        if user_id <= 0:
+            self.logger.warning("handle_get_user_music_api_request: Could not find userid in data, or userId is 0")
+            return {}
+        
+        songs = self.data.score.get_best_scores(user_id, is_dx=False)
+        if songs is None:
+            self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!")
+            return {
+            "userId": data["userId"],
+            "nextIndex": 0,
+            "userMusicList": [],
+        }
 
-                if len(music_detail_list) == data["maxCount"]:
-                    next_index = data["maxCount"] + data["nextIndex"]
-                    break
+        num_user_songs = len(songs)
 
+        for x in range(next_index, upper_lim):
+            if num_user_songs <= x:
+                break
+
+            tmp = songs[x]._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            music_detail_list.append(tmp)
+
+        next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim
+        self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
         return {
             "userId": data["userId"],
             "nextIndex": next_index,
             "userMusicList": [{"userMusicDetailList": music_detail_list}],
         }
+
+    def handle_upload_user_portrait_api_request(self, data: Dict) -> Dict:
+        self.logger.debug(data)
+
+    def handle_upload_user_photo_api_request(self, data: Dict) -> Dict:
+        if not self.game_config.uploads.photos or not self.game_config.uploads.photos_dir:
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+
+        photo = data.get("userPhoto", {})
+
+        if photo is None or not photo:
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        order_id = int(photo.get("orderId", -1))
+        user_id = int(photo.get("userId", -1))
+        div_num = int(photo.get("divNumber", -1))
+        div_len = int(photo.get("divLength", -1))
+        div_data = photo.get("divData", "")
+        playlog_id = int(photo.get("playlogId", -1))
+        track_num = int(photo.get("trackNo", -1))
+        upload_date = photo.get("uploadDate", "")
+
+        if order_id < 0 or user_id <= 0 or div_num < 0 or div_len <= 0 or not div_data or playlog_id < 0 or track_num <= 0 or not upload_date:
+            self.logger.warning(f"Malformed photo upload request")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        if order_id == 0 and div_num > 0:
+            self.logger.warning(f"Failed to set orderId properly (still 0 after first chunk)")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+
+        if div_num == 0 and order_id > 0:
+            self.logger.warning(f"First chuck re-send, Ignore")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        if div_num >= div_len:
+            self.logger.warning(f"Sent extra chunks ({div_num} >= {div_len})")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+
+        if div_len >= 100:
+            self.logger.warning(f"Photo too large ({div_len} * 10240 = {div_len * 10240} bytes)")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        ret_code = order_id + 1
+        photo_chunk = b64decode(div_data)
+
+        if len(photo_chunk) > 10240 or (len(photo_chunk) < 10240 and div_num + 1 != div_len):
+            self.logger.warning(f"Incorrect data size after decoding (Expected 10240, got {len(photo_chunk)})")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        out_name = f"{self.game_config.uploads.photos_dir}/{user_id}_{playlog_id}_{track_num}"
+
+        if not path.exists(f"{out_name}.bin") and div_num != 0:
+            self.logger.warning(f"Out of order photo upload (div_num {div_num})")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+    
+        if path.exists(f"{out_name}.bin") and div_num == 0:
+            self.logger.warning(f"Duplicate file upload")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        elif path.exists(f"{out_name}.bin"):
+            fstats = stat(f"{out_name}.bin")
+            if fstats.st_size != 10240 * div_num:
+                self.logger.warning(f"Out of order photo upload (trying to upload div {div_num}, expected div {fstats.st_size / 10240} for file sized {fstats.st_size} bytes)")
+                return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+        
+        try:
+            with open(f"{out_name}.bin", "ab") as f:
+                f.write(photo_chunk)
+        
+        except Exception:
+            self.logger.error(f"Failed writing to {out_name}.bin")
+            return {'returnCode': 0, 'apiName': 'UploadUserPhotoApi'}
+
+        if div_num + 1 == div_len and path.exists(f"{out_name}.bin"):
+            try:
+                p = ImageFile.Parser()
+                with open(f"{out_name}.bin", "rb") as f:
+                    p.feed(f.read())
+                
+                im = p.close()
+                im.save(f"{out_name}.jpeg")
+            except Exception:
+                self.logger.error(f"File {out_name}.bin failed image validation")
+            
+            try:
+                remove(f"{out_name}.bin")
+            
+            except Exception:
+                self.logger.error(f"Failed to delete {out_name}.bin, please remove it manually")
+
+        return {'returnCode': ret_code, 'apiName': 'UploadUserPhotoApi'}

titles/mai2/config.py

@@ -19,7 +19,59 @@ class Mai2ServerConfig:
             )
         )
 
+class Mai2DeliverConfig:
+    def __init__(self, parent: "Mai2Config") -> None:
+        self.__config = parent
+
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "deliver", "enable", default=False
+        )
+
+    @property
+    def udbdl_enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "deliver", "udbdl_enable", default=False
+        )
+    
+    @property
+    def content_folder(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "deliver", "content_folder", default=""
+        )
+
+class Mai2UploadsConfig:
+    def __init__(self, parent: "Mai2Config") -> None:
+        self.__config = parent
+    
+    @property
+    def photos(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "photos", default=False
+        )
+
+    @property
+    def photos_dir(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "photos_dir", default=""
+        )
+
+    @property
+    def movies(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "movies", default=False
+        )
+
+    @property
+    def movies_dir(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "mai2", "uploads", "movies_dir", default=""
+        )
+
 
 class Mai2Config(dict):
     def __init__(self) -> None:
         self.server = Mai2ServerConfig(self)
+        self.deliver = Mai2DeliverConfig(self)
+        self.uploads = Mai2UploadsConfig(self)

titles/mai2/const.py

@@ -20,26 +20,60 @@ class Mai2Constants:
 
     DATE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
 
-    GAME_CODE = "SDEZ"
+    GAME_CODE = "SBXL"    
+    GAME_CODE_GREEN = "SBZF"    
+    GAME_CODE_ORANGE = "SDBM"    
+    GAME_CODE_PINK = "SDCQ"    
+    GAME_CODE_MURASAKI = "SDDK"    
+    GAME_CODE_MILK = "SDDZ"    
+    GAME_CODE_FINALE = "SDEY"    
+    GAME_CODE_DX = "SDEZ"
 
     CONFIG_NAME = "mai2.yaml"
 
-    VER_MAIMAI_DX = 0
-    VER_MAIMAI_DX_PLUS = 1
-    VER_MAIMAI_DX_SPLASH = 2
-    VER_MAIMAI_DX_SPLASH_PLUS = 3
-    VER_MAIMAI_DX_UNIVERSE = 4
-    VER_MAIMAI_DX_UNIVERSE_PLUS = 5
-    VER_MAIMAI_DX_FESTIVAL = 6
+    VER_MAIMAI = 0
+    VER_MAIMAI_PLUS = 1
+    VER_MAIMAI_GREEN = 2
+    VER_MAIMAI_GREEN_PLUS = 3
+    VER_MAIMAI_ORANGE = 4
+    VER_MAIMAI_ORANGE_PLUS = 5
+    VER_MAIMAI_PINK = 6
+    VER_MAIMAI_PINK_PLUS = 7
+    VER_MAIMAI_MURASAKI = 8
+    VER_MAIMAI_MURASAKI_PLUS = 9
+    VER_MAIMAI_MILK = 10
+    VER_MAIMAI_MILK_PLUS = 11
+    VER_MAIMAI_FINALE = 12
+
+    VER_MAIMAI_DX = 13
+    VER_MAIMAI_DX_PLUS = 14
+    VER_MAIMAI_DX_SPLASH = 15
+    VER_MAIMAI_DX_SPLASH_PLUS = 16
+    VER_MAIMAI_DX_UNIVERSE = 17
+    VER_MAIMAI_DX_UNIVERSE_PLUS = 18
+    VER_MAIMAI_DX_FESTIVAL = 19
 
     VERSION_STRING = (
+        "maimai",
+        "maimai PLUS",
+        "maimai GreeN",
+        "maimai GreeN PLUS",
+        "maimai ORANGE",
+        "maimai ORANGE PLUS",
+        "maimai PiNK",
+        "maimai PiNK PLUS",
+        "maimai MURASAKi",
+        "maimai MURASAKi PLUS",
+        "maimai MiLK",
+        "maimai MiLK PLUS",
+        "maimai FiNALE",        
         "maimai DX",
         "maimai DX PLUS",
         "maimai DX Splash",
         "maimai DX Splash PLUS",
-        "maimai DX Universe",
-        "maimai DX Universe PLUS",
-        "maimai DX Festival",
+        "maimai DX UNiVERSE",
+        "maimai DX UNiVERSE PLUS",
+        "maimai DX FESTiVAL",
     )
 
     @classmethod

titles/mai2/dx.py

@@ -0,0 +1,591 @@
+from typing import Any, List, Dict
+from datetime import datetime, timedelta
+import pytz
+import json
+from random import randint
+
+from core.config import CoreConfig
+from titles.mai2.base import Mai2Base
+from titles.mai2.config import Mai2Config
+from titles.mai2.const import Mai2Constants
+
+
+class Mai2DX(Mai2Base):
+    def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
+        super().__init__(cfg, game_cfg)
+        self.version = Mai2Constants.VER_MAIMAI_DX
+        
+        if self.core_config.server.is_develop and self.core_config.title.port > 0:
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEZ/100/"
+        
+        else:
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEZ/100/"
+    
+    def handle_get_game_setting_api_request(self, data: Dict):
+        return {
+            "gameSetting": {
+                "isMaintenance": False,
+                "requestInterval": 1800,
+                "rebootStartTime": "2020-01-01 07:00:00.0",
+                "rebootEndTime": "2020-01-01 07:59:59.0",
+                "movieUploadLimit": 100,
+                "movieStatus": 1,
+                "movieServerUri": self.old_server + "movie/",
+                "deliverServerUri": self.old_server + "deliver/" if self.can_deliver and self.game_config.deliver.enable else "",
+                "oldServerUri": self.old_server + "old",
+                "usbDlServerUri": self.old_server + "usbdl/" if self.can_deliver and self.game_config.deliver.udbdl_enable else "",
+                "rebootInterval": 0,
+            },
+            "isAouAccession": False,
+        }
+
+    def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
+        p = self.data.profile.get_profile_detail(data["userId"], self.version)
+        o = self.data.profile.get_profile_option(data["userId"], self.version)
+        if p is None or o is None:
+            return {}  # Register
+        profile = p._asdict()
+        option = o._asdict()
+
+        return {
+            "userId": data["userId"],
+            "userName": profile["userName"],
+            "isLogin": False,
+            "lastGameId": profile["lastGameId"],
+            "lastDataVersion": profile["lastDataVersion"],
+            "lastRomVersion": profile["lastRomVersion"],
+            "lastLoginDate": profile["lastLoginDate"],
+            "lastPlayDate": profile["lastPlayDate"],
+            "playerRating": profile["playerRating"],
+            "nameplateId": 0,  # Unused
+            "iconId": profile["iconId"],
+            "trophyId": 0,  # Unused
+            "partnerId": profile["partnerId"],
+            "frameId": profile["frameId"],
+            "dispRate": option[
+                "dispRate"
+            ],  # 0: all/begin, 1: disprate, 2: dispDan, 3: hide, 4: end
+            "totalAwake": profile["totalAwake"],
+            "isNetMember": profile["isNetMember"],
+            "dailyBonusDate": profile["dailyBonusDate"],
+            "headPhoneVolume": option["headPhoneVolume"],
+            "isInherit": False,  # Not sure what this is or does??
+            "banState": profile["banState"]
+            if profile["banState"] is not None
+            else 0,  # New with uni+
+        }
+
+    def handle_upload_user_playlog_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        playlog = data["userPlaylog"]
+
+        self.data.score.put_playlog(user_id, playlog)
+
+        return {"returnCode": 1, "apiName": "UploadUserPlaylogApi"}
+
+    def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        charge = data["userCharge"]
+
+        # remove the ".0" from the date string, festival only?
+        charge["purchaseDate"] = charge["purchaseDate"].replace(".0", "")
+        self.data.item.put_charge(
+            user_id,
+            charge["chargeId"],
+            charge["stock"],
+            datetime.strptime(charge["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT),
+            datetime.strptime(charge["validDate"], Mai2Constants.DATE_TIME_FORMAT),
+        )
+
+        return {"returnCode": 1, "apiName": "UpsertUserChargelogApi"}
+
+    def handle_upsert_user_all_api_request(self, data: Dict) -> Dict:
+        user_id = data["userId"]
+        upsert = data["upsertUserAll"]
+
+        if "userData" in upsert and len(upsert["userData"]) > 0:
+            upsert["userData"][0]["isNetMember"] = 1
+            upsert["userData"][0].pop("accessCode")
+            self.data.profile.put_profile_detail(
+                user_id, self.version, upsert["userData"][0]
+            )
+
+        if "userExtend" in upsert and len(upsert["userExtend"]) > 0:
+            self.data.profile.put_profile_extend(
+                user_id, self.version, upsert["userExtend"][0]
+            )
+
+        if "userGhost" in upsert:
+            for ghost in upsert["userGhost"]:
+                self.data.profile.put_profile_ghost(user_id, self.version, ghost)
+
+        if "userOption" in upsert and len(upsert["userOption"]) > 0:
+            self.data.profile.put_profile_option(
+                user_id, self.version, upsert["userOption"][0]
+            )
+
+        if "userRatingList" in upsert and len(upsert["userRatingList"]) > 0:
+            self.data.profile.put_profile_rating(
+                user_id, self.version, upsert["userRatingList"][0]
+            )
+
+        if "userActivityList" in upsert and len(upsert["userActivityList"]) > 0:
+            for k, v in upsert["userActivityList"][0].items():
+                for act in v:
+                    self.data.profile.put_profile_activity(user_id, act)
+
+        if "userChargeList" in upsert and len(upsert["userChargeList"]) > 0:
+            for charge in upsert["userChargeList"]:
+                # remove the ".0" from the date string, festival only?
+                charge["purchaseDate"] = charge["purchaseDate"].replace(".0", "")
+                self.data.item.put_charge(
+                    user_id,
+                    charge["chargeId"],
+                    charge["stock"],
+                    datetime.strptime(
+                        charge["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT
+                    ),
+                    datetime.strptime(
+                        charge["validDate"], Mai2Constants.DATE_TIME_FORMAT
+                    ),
+                )
+
+        if "userCharacterList" in upsert and len(upsert["userCharacterList"]) > 0:
+            for char in upsert["userCharacterList"]:
+                self.data.item.put_character(
+                    user_id,
+                    char["characterId"],
+                    char["level"],
+                    char["awakening"],
+                    char["useCount"],
+                )
+
+        if "userItemList" in upsert and len(upsert["userItemList"]) > 0:
+            for item in upsert["userItemList"]:
+                self.data.item.put_item(
+                    user_id,
+                    int(item["itemKind"]),
+                    item["itemId"],
+                    item["stock"],
+                    item["isValid"],
+                )
+
+        if "userLoginBonusList" in upsert and len(upsert["userLoginBonusList"]) > 0:
+            for login_bonus in upsert["userLoginBonusList"]:
+                self.data.item.put_login_bonus(
+                    user_id,
+                    login_bonus["bonusId"],
+                    login_bonus["point"],
+                    login_bonus["isCurrent"],
+                    login_bonus["isComplete"],
+                )
+
+        if "userMapList" in upsert and len(upsert["userMapList"]) > 0:
+            for map in upsert["userMapList"]:
+                self.data.item.put_map(
+                    user_id,
+                    map["mapId"],
+                    map["distance"],
+                    map["isLock"],
+                    map["isClear"],
+                    map["isComplete"],
+                )
+
+        if "userMusicDetailList" in upsert and len(upsert["userMusicDetailList"]) > 0:
+            for music in upsert["userMusicDetailList"]:
+                self.data.score.put_best_score(user_id, music)
+
+        if "userCourseList" in upsert and len(upsert["userCourseList"]) > 0:
+            for course in upsert["userCourseList"]:
+                self.data.score.put_course(user_id, course)
+
+        if "userFavoriteList" in upsert and len(upsert["userFavoriteList"]) > 0:
+            for fav in upsert["userFavoriteList"]:
+                self.data.item.put_favorite(user_id, fav["kind"], fav["itemIdList"])
+
+        if (
+            "userFriendSeasonRankingList" in upsert
+            and len(upsert["userFriendSeasonRankingList"]) > 0
+        ):
+            for fsr in upsert["userFriendSeasonRankingList"]:
+                fsr["recordDate"] = (
+                    datetime.strptime(
+                        fsr["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
+                    ),
+                )
+                self.data.item.put_friend_season_ranking(user_id, fsr)
+
+        return {"returnCode": 1, "apiName": "UpsertUserAllApi"}
+
+    def handle_get_user_data_api_request(self, data: Dict) -> Dict:
+        profile = self.data.profile.get_profile_detail(data["userId"], self.version)
+        if profile is None:
+            return
+
+        profile_dict = profile._asdict()
+        profile_dict.pop("id")
+        profile_dict.pop("user")
+        profile_dict.pop("version")
+
+        return {"userId": data["userId"], "userData": profile_dict}
+
+    def handle_get_user_extend_api_request(self, data: Dict) -> Dict:
+        extend = self.data.profile.get_profile_extend(data["userId"], self.version)
+        if extend is None:
+            return
+
+        extend_dict = extend._asdict()
+        extend_dict.pop("id")
+        extend_dict.pop("user")
+        extend_dict.pop("version")
+
+        return {"userId": data["userId"], "userExtend": extend_dict}
+
+    def handle_get_user_option_api_request(self, data: Dict) -> Dict:
+        options = self.data.profile.get_profile_option(data["userId"], self.version)
+        if options is None:
+            return
+
+        options_dict = options._asdict()
+        options_dict.pop("id")
+        options_dict.pop("user")
+        options_dict.pop("version")
+
+        return {"userId": data["userId"], "userOption": options_dict}
+
+    def handle_get_user_card_api_request(self, data: Dict) -> Dict:
+        user_cards = self.data.item.get_cards(data["userId"])
+        if user_cards is None:
+            return {"userId": data["userId"], "nextIndex": 0, "userCardList": []}
+
+        max_ct = data["maxCount"]
+        next_idx = data["nextIndex"]
+        start_idx = next_idx
+        end_idx = max_ct + start_idx
+
+        if len(user_cards[start_idx:]) > max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        card_list = []
+        for card in user_cards:
+            tmp = card._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            tmp["startDate"] = datetime.strftime(
+                tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["endDate"] = datetime.strftime(
+                tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            card_list.append(tmp)
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userCardList": card_list[start_idx:end_idx],
+        }
+
+    def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
+        user_charges = self.data.item.get_charges(data["userId"])
+        if user_charges is None:
+            return {"userId": data["userId"], "length": 0, "userChargeList": []}
+
+        user_charge_list = []
+        for charge in user_charges:
+            tmp = charge._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            tmp["purchaseDate"] = datetime.strftime(
+                tmp["purchaseDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["validDate"] = datetime.strftime(
+                tmp["validDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+
+            user_charge_list.append(tmp)
+
+        return {
+            "userId": data["userId"],
+            "length": len(user_charge_list),
+            "userChargeList": user_charge_list,
+        }
+
+    def handle_get_user_item_api_request(self, data: Dict) -> Dict:
+        kind = int(data["nextIndex"] / 10000000000)
+        next_idx = int(data["nextIndex"] % 10000000000)
+        user_item_list = self.data.item.get_items(data["userId"], kind)
+
+        items: List[Dict[str, Any]] = []
+        for i in range(next_idx, len(user_item_list)):
+            tmp = user_item_list[i]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            items.append(tmp)
+            if len(items) >= int(data["maxCount"]):
+                break
+
+        xout = kind * 10000000000 + next_idx + len(items)
+
+        if len(items) < int(data["maxCount"]):
+            next_idx = 0
+        else:
+            next_idx = xout
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "itemKind": kind,
+            "userItemList": items,
+        }
+
+    def handle_get_user_character_api_request(self, data: Dict) -> Dict:
+        characters = self.data.item.get_characters(data["userId"])
+
+        chara_list = []
+        for chara in characters:
+            tmp = chara._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            chara_list.append(tmp)
+
+        return {"userId": data["userId"], "userCharacterList": chara_list}
+
+    def handle_get_user_favorite_api_request(self, data: Dict) -> Dict:
+        favorites = self.data.item.get_favorites(data["userId"], data["itemKind"])
+        if favorites is None:
+            return
+
+        userFavs = []
+        for fav in favorites:
+            userFavs.append(
+                {
+                    "userId": data["userId"],
+                    "itemKind": fav["itemKind"],
+                    "itemIdList": fav["itemIdList"],
+                }
+            )
+
+        return {"userId": data["userId"], "userFavoriteData": userFavs}
+
+    def handle_get_user_ghost_api_request(self, data: Dict) -> Dict:
+        ghost = self.data.profile.get_profile_ghost(data["userId"], self.version)
+        if ghost is None:
+            return
+
+        ghost_dict = ghost._asdict()
+        ghost_dict.pop("user")
+        ghost_dict.pop("id")
+        ghost_dict.pop("version_int")
+
+        return {"userId": data["userId"], "userGhost": ghost_dict}
+
+    def handle_get_user_rating_api_request(self, data: Dict) -> Dict:
+        rating = self.data.profile.get_profile_rating(data["userId"], self.version)
+        if rating is None:
+            return
+
+        rating_dict = rating._asdict()
+        rating_dict.pop("user")
+        rating_dict.pop("id")
+        rating_dict.pop("version")
+
+        return {"userId": data["userId"], "userRating": rating_dict}
+
+    def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
+        """
+        kind 1 is playlist, kind 2 is music list
+        """
+        playlist = self.data.profile.get_profile_activity(data["userId"], 1)
+        musiclist = self.data.profile.get_profile_activity(data["userId"], 2)
+        if playlist is None or musiclist is None:
+            return
+
+        plst = []
+        mlst = []
+
+        for play in playlist:
+            tmp = play._asdict()
+            tmp["id"] = tmp["activityId"]
+            tmp.pop("activityId")
+            tmp.pop("user")
+            plst.append(tmp)
+
+        for music in musiclist:
+            tmp = music._asdict()
+            tmp["id"] = tmp["activityId"]
+            tmp.pop("activityId")
+            tmp.pop("user")
+            mlst.append(tmp)
+
+        return {"userActivity": {"playList": plst, "musicList": mlst}}
+
+    def handle_get_user_course_api_request(self, data: Dict) -> Dict:
+        user_courses = self.data.score.get_courses(data["userId"])
+        if user_courses is None:
+            return {"userId": data["userId"], "nextIndex": 0, "userCourseList": []}
+
+        course_list = []
+        for course in user_courses:
+            tmp = course._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            course_list.append(tmp)
+
+        return {"userId": data["userId"], "nextIndex": 0, "userCourseList": course_list}
+
+    def handle_get_user_portrait_api_request(self, data: Dict) -> Dict:
+        # No support for custom pfps
+        return {"length": 0, "userPortraitList": []}
+
+    def handle_get_user_friend_season_ranking_api_request(self, data: Dict) -> Dict:
+        friend_season_ranking = self.data.item.get_friend_season_ranking(data["userId"])
+        if friend_season_ranking is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userFriendSeasonRankingList": [],
+            }
+
+        friend_season_ranking_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(friend_season_ranking)):
+            tmp = friend_season_ranking[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            tmp["recordDate"] = datetime.strftime(
+                tmp["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
+            )
+            friend_season_ranking_list.append(tmp)
+
+            if len(friend_season_ranking_list) >= max_ct:
+                break
+
+        if len(friend_season_ranking) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userFriendSeasonRankingList": friend_season_ranking_list,
+        }
+
+    def handle_get_user_map_api_request(self, data: Dict) -> Dict:
+        maps = self.data.item.get_maps(data["userId"])
+        if maps is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userMapList": [],
+            }
+
+        map_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(maps)):
+            tmp = maps[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            map_list.append(tmp)
+
+            if len(map_list) >= max_ct:
+                break
+
+        if len(maps) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userMapList": map_list,
+        }
+
+    def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
+        login_bonuses = self.data.item.get_login_bonuses(data["userId"])
+        if login_bonuses is None:
+            return {
+                "userId": data["userId"],
+                "nextIndex": 0,
+                "userLoginBonusList": [],
+            }
+
+        login_bonus_list = []
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(login_bonuses)):
+            tmp = login_bonuses[x]._asdict()
+            tmp.pop("user")
+            tmp.pop("id")
+            login_bonus_list.append(tmp)
+
+            if len(login_bonus_list) >= max_ct:
+                break
+
+        if len(login_bonuses) >= next_idx + max_ct:
+            next_idx += max_ct
+        else:
+            next_idx = 0
+
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_idx,
+            "userLoginBonusList": login_bonus_list,
+        }
+
+    def handle_get_user_region_api_request(self, data: Dict) -> Dict:
+        return {"userId": data["userId"], "length": 0, "userRegionList": []}
+
+    def handle_get_user_music_api_request(self, data: Dict) -> Dict:
+        user_id = data.get("userId", 0)        
+        next_index = data.get("nextIndex", 0)
+        max_ct = data.get("maxCount", 50)
+        upper_lim = next_index + max_ct
+        music_detail_list = []
+
+        if user_id <= 0:
+            self.logger.warning("handle_get_user_music_api_request: Could not find userid in data, or userId is 0")
+            return {}
+        
+        songs = self.data.score.get_best_scores(user_id)
+        if songs is None:
+            self.logger.debug("handle_get_user_music_api_request: get_best_scores returned None!")
+            return {
+            "userId": data["userId"],
+            "nextIndex": 0,
+            "userMusicList": [],
+        }
+
+        num_user_songs = len(songs)
+
+        for x in range(next_index, upper_lim):
+            if num_user_songs <= x:
+                break
+
+            tmp = songs[x]._asdict()
+            tmp.pop("id")
+            tmp.pop("user")
+            music_detail_list.append(tmp)
+
+        next_index = 0 if len(music_detail_list) < max_ct or num_user_songs == upper_lim else upper_lim
+        self.logger.info(f"Send songs {next_index}-{upper_lim} ({len(music_detail_list)}) out of {num_user_songs} for user {user_id} (next idx {next_index})")
+        return {
+            "userId": data["userId"],
+            "nextIndex": next_index,
+            "userMusicList": [{"userMusicDetailList": music_detail_list}],
+        }
+
+    def handle_user_login_api_request(self, data: Dict) -> Dict:
+        ret = super().handle_user_login_api_request(data)
+        if ret is None or not ret:
+            return ret
+        ret['loginId'] = ret.get('loginCount', 0)
+        return ret

titles/mai2/dxplus.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dx import Mai2DX
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2Plus(Mai2Base):
+class Mai2DXPlus(Mai2DX):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_PLUS

titles/mai2/festival.py

@@ -14,7 +14,7 @@ class Mai2Festival(Mai2UniversePlus):
     def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
         user_data = super().handle_cm_get_user_preview_api_request(data)
 
-        # hardcode lastDataVersion for CardMaker 1.36
+        # hardcode lastDataVersion for CardMaker 1.35
         user_data["lastDataVersion"] = "1.30.00"
         return user_data
 

titles/mai2/finale.py

@@ -0,0 +1,23 @@
+from typing import Any, List, Dict
+from datetime import datetime, timedelta
+import pytz
+import json
+
+from core.config import CoreConfig
+from titles.mai2.base import Mai2Base
+from titles.mai2.config import Mai2Config
+from titles.mai2.const import Mai2Constants
+
+
+class Mai2Finale(Mai2Base):
+    def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
+        super().__init__(cfg, game_cfg)
+        self.version = Mai2Constants.VER_MAIMAI_FINALE
+        self.can_deliver = True
+        self.can_usbdl = True
+        
+        if self.core_config.server.is_develop and self.core_config.title.port > 0:
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEY/197/"
+        
+        else:
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEY/197/"

titles/mai2/index.py

@@ -1,4 +1,5 @@
 from twisted.web.http import Request
+from twisted.web.server import NOT_DONE_YET
 import json
 import inflection
 import yaml
@@ -6,7 +7,7 @@ import string
 import logging, coloredlogs
 import zlib
 from logging.handlers import TimedRotatingFileHandler
-from os import path
+from os import path, mkdir
 from typing import Tuple
 
 from core.config import CoreConfig
@@ -14,7 +15,9 @@ from core.utils import Utils
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 from titles.mai2.base import Mai2Base
-from titles.mai2.plus import Mai2Plus
+from titles.mai2.finale import Mai2Finale
+from titles.mai2.dx import Mai2DX
+from titles.mai2.dxplus import Mai2DXPlus
 from titles.mai2.splash import Mai2Splash
 from titles.mai2.splashplus import Mai2SplashPlus
 from titles.mai2.universe import Mai2Universe
@@ -33,7 +36,20 @@ class Mai2Servlet:
 
         self.versions = [
             Mai2Base,
-            Mai2Plus,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+            Mai2Finale,            
+            Mai2DX,
+            Mai2DXPlus,
             Mai2Splash,
             Mai2SplashPlus,
             Mai2Universe,
@@ -42,27 +58,29 @@ class Mai2Servlet:
         ]
 
         self.logger = logging.getLogger("mai2")
-        log_fmt_str = "[%(asctime)s] Mai2 | %(levelname)s | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
-        fileHandler = TimedRotatingFileHandler(
-            "{0}/{1}.log".format(self.core_cfg.server.log_dir, "mai2"),
-            encoding="utf8",
-            when="d",
-            backupCount=10,
-        )
+        if not hasattr(self.logger, "initted"):
+            log_fmt_str = "[%(asctime)s] Mai2 | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.core_cfg.server.log_dir, "mai2"),
+                encoding="utf8",
+                when="d",
+                backupCount=10,
+            )
 
-        fileHandler.setFormatter(log_fmt)
+            fileHandler.setFormatter(log_fmt)
 
-        consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
 
-        self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
 
-        self.logger.setLevel(self.game_cfg.server.loglevel)
-        coloredlogs.install(
-            level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
-        )
+            self.logger.setLevel(self.game_cfg.server.loglevel)
+            coloredlogs.install(
+                level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initted = True
 
     @classmethod
     def get_allnet_info(
@@ -82,18 +100,35 @@ class Mai2Servlet:
             return (
                 True,
                 f"http://{core_cfg.title.hostname}:{core_cfg.title.port}/{game_code}/$v/",
-                f"{core_cfg.title.hostname}:{core_cfg.title.port}/",
+                f"{core_cfg.title.hostname}",
             )
 
         return (
             True,
             f"http://{core_cfg.title.hostname}/{game_code}/$v/",
-            f"{core_cfg.title.hostname}/",
+            f"{core_cfg.title.hostname}",
         )
 
+    def setup(self):
+        if self.game_cfg.uploads.photos and self.game_cfg.uploads.photos_dir and not path.exists(self.game_cfg.uploads.photos_dir):
+            try:
+                mkdir(self.game_cfg.uploads.photos_dir)
+            except Exception:
+                self.logger.error(f"Failed to make photo upload directory at {self.game_cfg.uploads.photos_dir}")
+
+        if self.game_cfg.uploads.movies and self.game_cfg.uploads.movies_dir and not path.exists(self.game_cfg.uploads.movies_dir):
+            try:
+                mkdir(self.game_cfg.uploads.movies_dir)
+            except Exception:
+                self.logger.error(f"Failed to make movie upload directory at {self.game_cfg.uploads.movies_dir}")
+
     def render_POST(self, request: Request, version: int, url_path: str) -> bytes:
         if url_path.lower() == "ping":
             return zlib.compress(b'{"returnCode": "1"}')
+        
+        elif url_path.startswith("api/movie/"):
+            self.logger.info(f"Movie data: {url_path} - {request.content.getvalue()}")
+            return b""
 
         req_raw = request.content.getvalue()
         url = request.uri.decode()
@@ -102,26 +137,58 @@ class Mai2Servlet:
         endpoint = url_split[len(url_split) - 1]
         client_ip = Utils.get_ip_addr(request)
 
-        if version < 105:  # 1.0
-            internal_ver = Mai2Constants.VER_MAIMAI_DX
-        elif version >= 105 and version < 110:  # Plus
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_PLUS
-        elif version >= 110 and version < 115:  # Splash
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH
-        elif version >= 115 and version < 120:  # Splash Plus
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS
-        elif version >= 120 and version < 125:  # Universe
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE
-        elif version >= 125 and version < 130:  # Universe Plus
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS
-        elif version >= 130:  # Festival
-            internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL
+        if request.uri.startswith(b"/SDEZ"):
+            if version < 105:  # 1.0
+                internal_ver = Mai2Constants.VER_MAIMAI_DX
+            elif version >= 105 and version < 110:  # Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_PLUS
+            elif version >= 110 and version < 115:  # Splash
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH
+            elif version >= 115 and version < 120:  # Splash Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS
+            elif version >= 120 and version < 125:  # Universe
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE
+            elif version >= 125 and version < 130:  # Universe Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_UNIVERSE_PLUS
+            elif version >= 130:  # Festival
+                internal_ver = Mai2Constants.VER_MAIMAI_DX_FESTIVAL
 
-        if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
-            # If we get a 32 character long hex string, it's a hash and we're
-            # doing encrypted. The likelyhood of false positives is low but
-            # technically not 0
-            self.logger.error("Encryption not supported at this time")
+        else:
+            if version < 110:  # 1.0
+                internal_ver = Mai2Constants.VER_MAIMAI
+            elif version >= 110 and version < 120:  # Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_PLUS
+            elif version >= 120 and version < 130:  # Green
+                internal_ver = Mai2Constants.VER_MAIMAI_GREEN
+            elif version >= 130 and version < 140:  # Green Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_GREEN_PLUS
+            elif version >= 140 and version < 150:  # Orange
+                internal_ver = Mai2Constants.VER_MAIMAI_ORANGE
+            elif version >= 150 and version < 160:  # Orange Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_ORANGE_PLUS
+            elif version >= 160 and version < 170:  # Pink
+                internal_ver = Mai2Constants.VER_MAIMAI_PINK
+            elif version >= 170 and version < 180:  # Pink Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_PINK_PLUS
+            elif version >= 180 and version < 185:  # Murasaki
+                internal_ver = Mai2Constants.VER_MAIMAI_MURASAKI
+            elif version >= 185 and version < 190:  # Murasaki Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_MURASAKI_PLUS
+            elif version >= 190 and version < 195:  # Milk
+                internal_ver = Mai2Constants.VER_MAIMAI_MILK
+            elif version >= 195 and version < 197:  # Milk Plus
+                internal_ver = Mai2Constants.VER_MAIMAI_MILK_PLUS
+            elif version >= 197:  # Finale
+                internal_ver = Mai2Constants.VER_MAIMAI_FINALE
+        
+        if request.getHeader('Mai-Encoding') is not None or request.getHeader('X-Mai-Encoding') is not None:
+            # The has is some flavor of MD5 of the endpoint with a constant bolted onto the end of it.
+            # See cake.dll's Obfuscator function for details. Hopefully most DLL edits will remove
+            # these two(?) headers to not cause issues, but given the general quality of SEGA data...
+            enc_ver = request.getHeader('Mai-Encoding')
+            if enc_ver is None:
+                enc_ver = request.getHeader('X-Mai-Encoding')
+            self.logger.debug(f"Encryption v{enc_ver} - User-Agent: {request.getHeader('User-Agent')}")
 
         try:
             unzip = zlib.decompress(req_raw)
@@ -159,3 +226,61 @@ class Mai2Servlet:
         self.logger.debug(f"Response {resp}")
 
         return zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))
+
+    def render_GET(self, request: Request, version: int, url_path: str) -> bytes:
+        self.logger.debug(f"v{version} GET {url_path}")
+        url_split = url_path.split("/")
+
+        if (url_split[0] == "api" and url_split[1] == "movie") or url_split[0] == "movie":
+            if url_split[2] == "moviestart":
+                return json.dumps({"moviestart":{"status":"OK"}}).encode()
+            
+            else:
+                request.setResponseCode(404)
+                return b""
+
+        if url_split[0] == "old":
+            if url_split[1] == "ping":
+                self.logger.info(f"v{version} old server ping")
+                return zlib.compress(b"ok")
+
+            elif url_split[1].startswith("userdata"):
+                self.logger.info(f"v{version} old server userdata inquire")
+                return zlib.compress(b"{}")
+
+            elif url_split[1].startswith("friend"):
+                self.logger.info(f"v{version} old server friend inquire")
+                return zlib.compress(b"{}")
+            
+            else:
+                request.setResponseCode(404)
+                return b""
+        
+        elif url_split[0] == "usbdl":
+            if url_split[1] == "CONNECTIONTEST":
+                self.logger.info(f"v{version} usbdl server test")
+                return b""
+            
+            elif self.game_cfg.deliver.udbdl_enable and path.exists(f"{self.game_cfg.deliver.content_folder}/usb/{url_split[-1]}"):
+                with open(f"{self.game_cfg.deliver.content_folder}/usb/{url_split[-1]}", 'rb') as f:
+                    return f.read()
+                
+            else:
+                request.setResponseCode(404)
+                return b""
+
+        elif url_split[0] == "deliver":
+            file = url_split[len(url_split) - 1]
+            self.logger.info(f"v{version} {file} deliver inquire")
+            self.logger.debug(f"{self.game_cfg.deliver.content_folder}/net_deliver/{file}")
+            
+            if self.game_cfg.deliver.enable and path.exists(f"{self.game_cfg.deliver.content_folder}/net_deliver/{file}"):
+                with open(f"{self.game_cfg.deliver.content_folder}/net_deliver/{file}", 'rb') as f:
+                    return f.read()
+            
+            else:
+                request.setResponseCode(404)
+                return b""
+
+        else:
+            return zlib.compress(b"{}")

titles/mai2/read.py

@@ -4,6 +4,9 @@ import os
 import re
 import xml.etree.ElementTree as ET
 from typing import Any, Dict, List, Optional
+from Crypto.Cipher import AES
+import zlib
+import codecs
 
 from core.config import CoreConfig
 from core.data import Data
@@ -34,18 +37,147 @@ class Mai2Reader(BaseReader):
 
     def read(self) -> None:
         data_dirs = []
-        if self.bin_dir is not None:
-            data_dirs += self.get_data_directories(self.bin_dir)
+        if self.version >= Mai2Constants.VER_MAIMAI_DX:
+            if self.bin_dir is not None:
+                data_dirs += self.get_data_directories(self.bin_dir)
 
-        if self.opt_dir is not None:
-            data_dirs += self.get_data_directories(self.opt_dir)
+            if self.opt_dir is not None:
+                data_dirs += self.get_data_directories(self.opt_dir)
 
-        for dir in data_dirs:
-            self.logger.info(f"Read from {dir}")
-            self.get_events(f"{dir}/event")
-            self.disable_events(f"{dir}/information", f"{dir}/scoreRanking")
-            self.read_music(f"{dir}/music")
-            self.read_tickets(f"{dir}/ticket")
+            for dir in data_dirs:
+                self.logger.info(f"Read from {dir}")
+                self.get_events(f"{dir}/event")
+                self.disable_events(f"{dir}/information", f"{dir}/scoreRanking")
+                self.read_music(f"{dir}/music")
+                self.read_tickets(f"{dir}/ticket")
+        
+        else:
+            if not os.path.exists(f"{self.bin_dir}/tables"):
+                self.logger.error(f"tables directory not found in {self.bin_dir}")
+                return
+            
+            if self.version >= Mai2Constants.VER_MAIMAI_MILK:
+                if self.extra is None:
+                    self.logger.error("Milk - Finale requre an AES key via a hex string send as the --extra flag")
+                    return
+                
+                key = bytes.fromhex(self.extra)
+            
+            else:
+                key = None
+            
+            evt_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmEvent.bin", key)
+            txt_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmtextout_jp.bin", key)
+            score_table = self.load_table_raw(f"{self.bin_dir}/tables", "mmScore.bin", key)
+            
+            self.read_old_events(evt_table)
+            self.read_old_music(score_table, txt_table)
+            
+            if self.opt_dir is not None:
+                evt_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmEvent.bin", key)
+                txt_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmtextout_jp.bin", key)
+                score_table = self.load_table_raw(f"{self.opt_dir}/tables", "mmScore.bin", key)
+
+                self.read_old_events(evt_table)
+                self.read_old_music(score_table, txt_table)
+
+            return
+    
+    def load_table_raw(self, dir: str, file: str, key: Optional[bytes]) -> Optional[List[Dict[str, str]]]:
+        if not os.path.exists(f"{dir}/{file}"):
+            self.logger.warning(f"file {file} does not exist in directory {dir}, skipping")
+            return
+        
+        self.logger.info(f"Load table {file} from {dir}")
+        if key is not None:
+            cipher = AES.new(key, AES.MODE_CBC)
+            with open(f"{dir}/{file}", "rb") as f:
+                f_encrypted = f.read()
+                f_data = cipher.decrypt(f_encrypted)[0x10:]
+        
+        else:
+            with open(f"{dir}/{file}", "rb") as f:
+                f_data = f.read()[0x10:]
+        
+        if f_data is None or not f_data:
+            self.logger.warning(f"file {dir} could not be read, skipping")
+            return
+        
+        f_data_deflate = zlib.decompress(f_data, wbits = zlib.MAX_WBITS | 16)[0x12:] # lop off the junk at the beginning
+        f_decoded = codecs.utf_16_le_decode(f_data_deflate)[0]
+        f_split = f_decoded.splitlines()
+
+        has_struct_def = "struct " in f_decoded
+        is_struct = False
+        struct_def = []
+        tbl_content = []
+
+        if has_struct_def:
+            for x in f_split:
+                if x.startswith("struct "):
+                    is_struct = True
+                    struct_name = x[7:-1]
+                    continue
+                
+                if x.startswith("};"):
+                    is_struct = False
+                    break
+
+                if is_struct:
+                    try:
+                        struct_def.append(x[x.rindex("  ") + 2: -1])
+                    except ValueError:
+                        self.logger.warning(f"rindex failed on line {x}")
+            
+            if is_struct:
+                self.logger.warning("Struct not formatted properly")
+            
+            if not struct_def:
+                self.logger.warning("Struct def not found")
+                    
+        name = file[:file.index(".")]
+        if "_" in name:
+            name = name[:file.index("_")]
+        
+        for x in f_split:
+            if not x.startswith(name.upper()):
+                continue
+            
+            line_match = re.match(r"(\w+)\((.*?)\)([ ]+\/{3}<[ ]+(.*))?", x)
+            if line_match is None:
+                continue
+
+            if not line_match.group(1) == name.upper():
+                self.logger.warning(f"Strange regex match for line {x} -> {line_match}")
+                continue
+            
+            vals = line_match.group(2)
+            comment = line_match.group(4)
+            line_dict = {}
+            
+            vals_split = vals.split(",")
+            for y in range(len(vals_split)):
+                stripped = vals_split[y].strip().lstrip("L\"").lstrip("\"").rstrip("\"")
+                if not stripped or stripped is None:
+                    continue
+
+                if has_struct_def and len(struct_def) > y:
+                    line_dict[struct_def[y]] = stripped
+                
+                else:
+                    line_dict[f'item_{y}'] = stripped
+            
+            if comment:
+                line_dict['comment'] = comment
+            
+            tbl_content.append(line_dict)
+
+        if tbl_content:
+            return tbl_content
+        
+        else:
+            self.logger.warning("Failed load table content, skipping")
+            return
 
     def get_events(self, base_dir: str) -> None:
         self.logger.info(f"Reading events from {base_dir}...")
@@ -188,3 +320,24 @@ class Mai2Reader(BaseReader):
                             self.version, id, ticket_type, price, name
                         )
                         self.logger.info(f"Added ticket {id}...")
+
+    def read_old_events(self, events: Optional[List[Dict[str, str]]]) -> None:
+        if events is None:
+            return
+        
+        for event in events:
+            evt_id = int(event.get('イベントID', '0'))
+            evt_expire_time = float(event.get('オフ時強制時期', '0.0'))
+            is_exp = bool(int(event.get('海外許可', '0')))
+            is_aou = bool(int(event.get('AOU許可', '0')))
+            name = event.get('comment', f'evt_{evt_id}')
+
+            self.data.static.put_game_event(self.version, 0, evt_id, name)
+            
+            if not (is_exp or is_aou):
+                self.data.static.toggle_game_event(self.version, evt_id, False)
+    
+    def read_old_music(self, scores: Optional[List[Dict[str, str]]], text: Optional[List[Dict[str, str]]]) -> None:
+        if scores is None or text is None:
+            return
+        # TODO

titles/mai2/schema/item.py

@@ -18,10 +18,11 @@ character = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("characterId", Integer, nullable=False),
-    Column("level", Integer, nullable=False, server_default="1"),
-    Column("awakening", Integer, nullable=False, server_default="0"),
-    Column("useCount", Integer, nullable=False, server_default="0"),
+    Column("characterId", Integer),
+    Column("level", Integer),
+    Column("awakening", Integer),
+    Column("useCount", Integer),
+    Column("point", Integer),
     UniqueConstraint("user", "characterId", name="mai2_item_character_uk"),
     mysql_charset="utf8mb4",
 )
@@ -35,12 +36,12 @@ card = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("cardId", Integer, nullable=False),
-    Column("cardTypeId", Integer, nullable=False),
-    Column("charaId", Integer, nullable=False),
-    Column("mapId", Integer, nullable=False),
-    Column("startDate", TIMESTAMP, server_default="2018-01-01 00:00:00.0"),
-    Column("endDate", TIMESTAMP, server_default="2038-01-01 00:00:00.0"),
+    Column("cardId", Integer),
+    Column("cardTypeId", Integer),
+    Column("charaId", Integer),
+    Column("mapId", Integer),
+    Column("startDate", TIMESTAMP, server_default=func.now()),
+    Column("endDate", TIMESTAMP),
     UniqueConstraint("user", "cardId", "cardTypeId", name="mai2_item_card_uk"),
     mysql_charset="utf8mb4",
 )
@@ -54,10 +55,10 @@ item = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("itemId", Integer, nullable=False),
-    Column("itemKind", Integer, nullable=False),
-    Column("stock", Integer, nullable=False, server_default="1"),
-    Column("isValid", Boolean, nullable=False, server_default="1"),
+    Column("itemId", Integer),
+    Column("itemKind", Integer),
+    Column("stock", Integer),
+    Column("isValid", Boolean),
     UniqueConstraint("user", "itemId", "itemKind", name="mai2_item_item_uk"),
     mysql_charset="utf8mb4",
 )
@@ -71,11 +72,11 @@ map = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("mapId", Integer, nullable=False),
-    Column("distance", Integer, nullable=False),
-    Column("isLock", Boolean, nullable=False, server_default="0"),
-    Column("isClear", Boolean, nullable=False, server_default="0"),
-    Column("isComplete", Boolean, nullable=False, server_default="0"),
+    Column("mapId", Integer),
+    Column("distance", Integer),
+    Column("isLock", Boolean),
+    Column("isClear", Boolean),
+    Column("isComplete", Boolean),
     UniqueConstraint("user", "mapId", name="mai2_item_map_uk"),
     mysql_charset="utf8mb4",
 )
@@ -89,10 +90,10 @@ login_bonus = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("bonusId", Integer, nullable=False),
-    Column("point", Integer, nullable=False),
-    Column("isCurrent", Boolean, nullable=False, server_default="0"),
-    Column("isComplete", Boolean, nullable=False, server_default="0"),
+    Column("bonusId", Integer),
+    Column("point", Integer),
+    Column("isCurrent", Boolean),
+    Column("isComplete", Boolean),
     UniqueConstraint("user", "bonusId", name="mai2_item_login_bonus_uk"),
     mysql_charset="utf8mb4",
 )
@@ -106,12 +107,12 @@ friend_season_ranking = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("seasonId", Integer, nullable=False),
-    Column("point", Integer, nullable=False),
-    Column("rank", Integer, nullable=False),
-    Column("rewardGet", Boolean, nullable=False),
-    Column("userName", String(8), nullable=False),
-    Column("recordDate", TIMESTAMP, nullable=False),
+    Column("seasonId", Integer),
+    Column("point", Integer),
+    Column("rank", Integer),
+    Column("rewardGet", Boolean),
+    Column("userName", String(8)),
+    Column("recordDate", TIMESTAMP),
     UniqueConstraint(
         "user", "seasonId", "userName", name="mai2_item_friend_season_ranking_uk"
     ),
@@ -127,7 +128,7 @@ favorite = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("itemKind", Integer, nullable=False),
+    Column("itemKind", Integer),
     Column("itemIdList", JSON),
     UniqueConstraint("user", "itemKind", name="mai2_item_favorite_uk"),
     mysql_charset="utf8mb4",
@@ -142,10 +143,10 @@ charge = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("chargeId", Integer, nullable=False),
-    Column("stock", Integer, nullable=False),
-    Column("purchaseDate", String(255), nullable=False),
-    Column("validDate", String(255), nullable=False),
+    Column("chargeId", Integer),
+    Column("stock", Integer),
+    Column("purchaseDate", String(255)),
+    Column("validDate", String(255)),
     UniqueConstraint("user", "chargeId", name="mai2_item_charge_uk"),
     mysql_charset="utf8mb4",
 )
@@ -161,11 +162,11 @@ print_detail = Table(
     ),
     Column("orderId", Integer),
     Column("printNumber", Integer),
-    Column("printDate", TIMESTAMP, nullable=False, server_default=func.now()),
-    Column("serialId", String(20), nullable=False),
-    Column("placeId", Integer, nullable=False),
-    Column("clientId", String(11), nullable=False),
-    Column("printerSerialId", String(20), nullable=False),
+    Column("printDate", TIMESTAMP, server_default=func.now()),
+    Column("serialId", String(20)),
+    Column("placeId", Integer),
+    Column("clientId", String(11)),
+    Column("printerSerialId", String(20)),
     Column("cardRomVersion", Integer),
     Column("isHolograph", Boolean, server_default="1"),
     Column("printOption1", Boolean, server_default="0"),
@@ -203,7 +204,7 @@ class Mai2ItemData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_item: failed to insert item! user_id: {user_id}, item_kind: {item_kind}, item_id: {item_id}"
             )
             return None
@@ -260,7 +261,7 @@ class Mai2ItemData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_login_bonus: failed to insert item! user_id: {user_id}, bonus_id: {bonus_id}, point: {point}"
             )
             return None
@@ -311,7 +312,7 @@ class Mai2ItemData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_map: failed to insert item! user_id: {user_id}, map_id: {map_id}, distance: {distance}"
             )
             return None
@@ -332,6 +333,19 @@ class Mai2ItemData(BaseData):
         if result is None:
             return None
         return result.fetchone()
+    
+    def put_character_(self, user_id: int, char_data: Dict) -> Optional[int]:
+        char_data["user"] = user_id
+        sql = insert(character).values(**char_data)
+
+        conflict = sql.on_duplicate_key_update(**char_data)
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warning(
+                f"put_character_: failed to insert item! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
 
     def put_character(
         self,
@@ -357,7 +371,7 @@ class Mai2ItemData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_character: failed to insert item! user_id: {user_id}, character_id: {character_id}, level: {level}"
             )
             return None
@@ -400,7 +414,7 @@ class Mai2ItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_friend_season_ranking: failed to insert",
                 f"friend_season_ranking! aime_id: {aime_id}",
             )
@@ -418,7 +432,7 @@ class Mai2ItemData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_favorite: failed to insert item! user_id: {user_id}, kind: {kind}"
             )
             return None
@@ -444,6 +458,8 @@ class Mai2ItemData(BaseData):
         card_kind: int,
         chara_id: int,
         map_id: int,
+        start_date: datetime,
+        end_date: datetime,
     ) -> Optional[Row]:
         sql = insert(card).values(
             user=user_id,
@@ -451,13 +467,17 @@ class Mai2ItemData(BaseData):
             cardTypeId=card_kind,
             charaId=chara_id,
             mapId=map_id,
+            startDate=start_date,
+            endDate=end_date,
         )
 
-        conflict = sql.on_duplicate_key_update(charaId=chara_id, mapId=map_id)
+        conflict = sql.on_duplicate_key_update(
+            charaId=chara_id, mapId=map_id, startDate=start_date, endDate=end_date
+        )
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_card: failed to insert card! user_id: {user_id}, kind: {card_kind}"
             )
             return None
@@ -496,7 +516,7 @@ class Mai2ItemData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_card: failed to insert charge! user_id: {user_id}, chargeId: {charge_id}"
             )
             return None
@@ -521,7 +541,7 @@ class Mai2ItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_user_print_detail: Failed to insert! aime_id: {aime_id}"
             )
             return None

titles/mai2/schema/profile.py

@@ -99,6 +99,68 @@ detail = Table(
     mysql_charset="utf8mb4",
 )
 
+detail_old = Table(
+   "maimai_profile_detail",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("lastDataVersion", Integer),
+    Column("userName", String(8)),
+    Column("point", Integer),
+    Column("totalPoint", Integer),
+    Column("iconId", Integer),
+    Column("nameplateId", Integer),
+    Column("frameId", Integer),
+    Column("trophyId", Integer),
+    Column("playCount", Integer),
+    Column("playVsCount", Integer),
+    Column("playSyncCount", Integer),
+    Column("winCount", Integer),
+    Column("helpCount", Integer),
+    Column("comboCount", Integer),
+    Column("feverCount", Integer),
+    Column("totalHiScore", Integer),
+    Column("totalEasyHighScore", Integer),
+    Column("totalBasicHighScore", Integer),
+    Column("totalAdvancedHighScore", Integer),
+    Column("totalExpertHighScore", Integer),
+    Column("totalMasterHighScore", Integer),
+    Column("totalReMasterHighScore", Integer),
+    Column("totalHighSync", Integer),
+    Column("totalEasySync", Integer),
+    Column("totalBasicSync", Integer),
+    Column("totalAdvancedSync", Integer),
+    Column("totalExpertSync", Integer),
+    Column("totalMasterSync", Integer),
+    Column("totalReMasterSync", Integer),
+    Column("playerRating", Integer),
+    Column("highestRating", Integer),
+    Column("rankAuthTailId", Integer),
+    Column("eventWatchedDate", String(255)),
+    Column("webLimitDate", String(255)),
+    Column("challengeTrackPhase", Integer),
+    Column("firstPlayBits", Integer),
+    Column("lastPlayDate", String(255)),
+    Column("lastPlaceId", Integer),
+    Column("lastPlaceName", String(255)),
+    Column("lastRegionId", Integer),
+    Column("lastRegionName", String(255)),
+    Column("lastClientId", String(255)),
+    Column("lastCountryCode", String(255)),
+    Column("eventPoint", Integer),
+    Column("totalLv", Integer),
+    Column("lastLoginBonusDay", Integer),
+    Column("lastSurvivalBonusDay", Integer),
+    Column("loginBonusLv", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_detail_uk"),
+    mysql_charset="utf8mb4",
+)
+
 ghost = Table(
     "mai2_profile_ghost",
     metadata,
@@ -223,6 +285,99 @@ option = Table(
     mysql_charset="utf8mb4",
 )
 
+option_old = Table(
+    "maimai_profile_option",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("soudEffect", Integer),
+    Column("mirrorMode", Integer),
+    Column("guideSpeed", Integer),
+    Column("bgInfo", Integer),
+    Column("brightness", Integer),
+    Column("isStarRot", Integer),
+    Column("breakSe", Integer),
+    Column("slideSe", Integer),
+    Column("hardJudge", Integer),
+    Column("isTagJump", Integer),
+    Column("breakSeVol", Integer),
+    Column("slideSeVol", Integer),
+    Column("isUpperDisp", Integer),
+    Column("trackSkip", Integer),
+    Column("optionMode", Integer),
+    Column("simpleOptionParam", Integer),
+    Column("adjustTiming", Integer),
+    Column("dispTiming", Integer),
+    Column("timingPos", Integer),
+    Column("ansVol", Integer),
+    Column("noteVol", Integer),
+    Column("dmgVol", Integer),
+    Column("appealFlame", Integer),
+    Column("isFeverDisp", Integer),
+    Column("dispJudge", Integer),
+    Column("judgePos", Integer),
+    Column("ratingGuard", Integer),
+    Column("selectChara", Integer),
+    Column("sortType", Integer),
+    Column("filterGenre", Integer),
+    Column("filterLevel", Integer),
+    Column("filterRank", Integer),
+    Column("filterVersion", Integer),
+    Column("filterRec", Integer),
+    Column("filterFullCombo", Integer),
+    Column("filterAllPerfect", Integer),
+    Column("filterDifficulty", Integer),
+    Column("filterFullSync", Integer),
+    Column("filterReMaster", Integer),
+    Column("filterMaxFever", Integer),
+    Column("finalSelectId", Integer),
+    Column("finalSelectCategory", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_option_uk"),
+    mysql_charset="utf8mb4",
+)
+
+web_opt = Table(
+    "maimai_profile_web_option",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("isNetMember", Boolean),
+    Column("dispRate", Integer),
+    Column("dispJudgeStyle", Integer),
+    Column("dispRank", Integer),
+    Column("dispHomeRanker", Integer),
+    Column("dispTotalLv", Integer),
+    UniqueConstraint("user", "version", name="maimai_profile_web_option_uk"),
+    mysql_charset="utf8mb4",
+)
+
+grade_status = Table(
+    "maimai_profile_grade_status",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("gradeVersion", Integer),
+    Column("gradeLevel", Integer),
+    Column("gradeSubLevel", Integer),
+    Column("gradeMaxId", Integer),
+    UniqueConstraint("user", "gradeVersion", name="maimai_profile_grade_status_uk"),
+    mysql_charset="utf8mb4",
+)
+
 rating = Table(
     "mai2_profile_rating",
     metadata,
@@ -268,42 +423,91 @@ activity = Table(
         ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
         nullable=False,
     ),
-    Column("kind", Integer, nullable=False),
-    Column("activityId", Integer, nullable=False),
-    Column("param1", Integer, nullable=False),
-    Column("param2", Integer, nullable=False),
-    Column("param3", Integer, nullable=False),
-    Column("param4", Integer, nullable=False),
-    Column("sortNumber", Integer, nullable=False),
+    Column("kind", Integer),
+    Column("activityId", Integer),
+    Column("param1", Integer),
+    Column("param2", Integer),
+    Column("param3", Integer),
+    Column("param4", Integer),
+    Column("sortNumber", Integer),
     UniqueConstraint("user", "kind", "activityId", name="mai2_profile_activity_uk"),
     mysql_charset="utf8mb4",
 )
 
+boss = Table(
+    "maimai_profile_boss",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("pandoraFlagList0", Integer),
+    Column("pandoraFlagList1", Integer),
+    Column("pandoraFlagList2", Integer),
+    Column("pandoraFlagList3", Integer),
+    Column("pandoraFlagList4", Integer),
+    Column("pandoraFlagList5", Integer),
+    Column("pandoraFlagList6", Integer),
+    Column("emblemFlagList", Integer),
+    UniqueConstraint("user", name="mai2_profile_boss_uk"),
+    mysql_charset="utf8mb4",
+)
+
+recent_rating = Table(
+    "maimai_profile_recent_rating",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("userRecentRatingList", JSON),
+    UniqueConstraint("user", name="mai2_profile_recent_rating_uk"),
+    mysql_charset="utf8mb4",
+)
+
+consec_logins = Table(
+        "mai2_profile_consec_logins",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column("version", Integer, nullable=False),
+    Column("logins", Integer),
+    UniqueConstraint("user", "version", name="mai2_profile_consec_logins_uk"),
+    mysql_charset="utf8mb4",
+)
 
 class Mai2ProfileData(BaseData):
     def put_profile_detail(
-        self, user_id: int, version: int, detail_data: Dict
+        self, user_id: int, version: int, detail_data: Dict, is_dx: bool = True
     ) -> Optional[Row]:
         detail_data["user"] = user_id
         detail_data["version"] = version
-        sql = insert(detail).values(**detail_data)
+
+        if is_dx:
+            sql = insert(detail).values(**detail_data)
+        else:
+            sql = insert(detail_old).values(**detail_data)
 
         conflict = sql.on_duplicate_key_update(**detail_data)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
-                f"put_profile: Failed to create profile! user_id {user_id}"
+            self.logger.warning(
+                f"put_profile: Failed to create profile! user_id {user_id} is_dx {is_dx}"
             )
             return None
         return result.lastrowid
 
-    def get_profile_detail(self, user_id: int, version: int) -> Optional[Row]:
-        sql = (
-            select(detail)
-            .where(and_(detail.c.user == user_id, detail.c.version <= version))
-            .order_by(detail.c.version.desc())
-        )
+    def get_profile_detail(self, user_id: int, version: int, is_dx: bool = True) -> Optional[Row]:
+        if is_dx:
+            sql = (
+                select(detail)
+                .where(and_(detail.c.user == user_id, detail.c.version <= version))
+                .order_by(detail.c.version.desc())
+            )
+        
+        else:
+            sql = (
+                select(detail_old)
+                .where(and_(detail_old.c.user == user_id, detail_old.c.version <= version))
+                .order_by(detail_old.c.version.desc())
+            )
 
         result = self.execute(sql)
         if result is None:
@@ -321,7 +525,7 @@ class Mai2ProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"put_profile_ghost: failed to update! {user_id}")
+            self.logger.warning(f"put_profile_ghost: failed to update! {user_id}")
             return None
         return result.lastrowid
 
@@ -348,7 +552,7 @@ class Mai2ProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"put_profile_extend: failed to update! {user_id}")
+            self.logger.warning(f"put_profile_extend: failed to update! {user_id}")
             return None
         return result.lastrowid
 
@@ -365,26 +569,36 @@ class Mai2ProfileData(BaseData):
         return result.fetchone()
 
     def put_profile_option(
-        self, user_id: int, version: int, option_data: Dict
+        self, user_id: int, version: int, option_data: Dict, is_dx: bool = True
     ) -> Optional[int]:
         option_data["user"] = user_id
         option_data["version"] = version
 
-        sql = insert(option).values(**option_data)
+        if is_dx:
+            sql = insert(option).values(**option_data)
+        else:
+            sql = insert(option_old).values(**option_data)
         conflict = sql.on_duplicate_key_update(**option_data)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"put_profile_option: failed to update! {user_id}")
+            self.logger.warning(f"put_profile_option: failed to update! {user_id} is_dx {is_dx}")
             return None
         return result.lastrowid
 
-    def get_profile_option(self, user_id: int, version: int) -> Optional[Row]:
-        sql = (
-            select(option)
-            .where(and_(option.c.user == user_id, option.c.version <= version))
-            .order_by(option.c.version.desc())
-        )
+    def get_profile_option(self, user_id: int, version: int, is_dx: bool = True) -> Optional[Row]:
+        if is_dx:
+            sql = (
+                select(option)
+                .where(and_(option.c.user == user_id, option.c.version <= version))
+                .order_by(option.c.version.desc())
+            )
+        else:
+            sql = (
+                select(option_old)
+                .where(and_(option_old.c.user == user_id, option_old.c.version <= version))
+                .order_by(option_old.c.version.desc())
+            )
 
         result = self.execute(sql)
         if result is None:
@@ -402,7 +616,7 @@ class Mai2ProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"put_profile_rating: failed to update! {user_id}")
+            self.logger.warning(f"put_profile_rating: failed to update! {user_id}")
             return None
         return result.lastrowid
 
@@ -429,7 +643,7 @@ class Mai2ProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"put_region: failed to update! {user_id}")
+            self.logger.warning(f"put_region: failed to update! {user_id}")
             return None
         return result.lastrowid
 
@@ -454,7 +668,7 @@ class Mai2ProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_activity: failed to update! user_id: {user_id}"
             )
             return None
@@ -474,3 +688,130 @@ class Mai2ProfileData(BaseData):
         if result is None:
             return None
         return result.fetchall()
+
+    def put_web_option(self, user_id: int, version: int, web_opts: Dict) -> Optional[int]:
+        web_opts["user"] = user_id
+        web_opts["version"] = version
+        sql = insert(web_opt).values(**web_opts)
+
+        conflict = sql.on_duplicate_key_update(**web_opts)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warning(
+                f"put_web_option: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_web_option(self, user_id: int, version: int) -> Optional[Row]:
+        sql = web_opt.select(and_(web_opt.c.user == user_id, web_opt.c.version == version))
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_grade_status(self, user_id: int, grade_stat: Dict) -> Optional[int]:
+        grade_stat["user"] = user_id
+        sql = insert(grade_status).values(**grade_stat)
+
+        conflict = sql.on_duplicate_key_update(**grade_stat)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warning(
+                f"put_grade_status: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_grade_status(self, user_id: int) -> Optional[Row]:
+        sql = grade_status.select(grade_status.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_boss_list(self, user_id: int, boss_stat: Dict) -> Optional[int]:
+        boss_stat["user"] = user_id
+        sql = insert(boss).values(**boss_stat)
+
+        conflict = sql.on_duplicate_key_update(**boss_stat)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warning(
+                f"put_boss_list: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_boss_list(self, user_id: int) -> Optional[Row]:
+        sql = boss.select(boss.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_recent_rating(self, user_id: int, rr: Dict) -> Optional[int]:
+        sql = insert(recent_rating).values(user=user_id, userRecentRatingList=rr)
+
+        conflict = sql.on_duplicate_key_update({'userRecentRatingList': rr})
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warning(
+                f"put_recent_rating: failed to update! user_id: {user_id}"
+            )
+            return None
+        return result.lastrowid
+    
+    def get_recent_rating(self, user_id: int) -> Optional[Row]:
+        sql = recent_rating.select(recent_rating.c.user == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def add_consec_login(self, user_id: int, version: int) -> None:
+        sql = insert(consec_logins).values(
+            user=user_id,
+            version=version,
+            logins=1
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            logins=consec_logins.c.logins + 1
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(f"Failed to update consecutive login count for user {user_id} version {version}")
+
+    def get_consec_login(self, user_id: int, version: int) -> Optional[Row]:
+        sql = select(consec_logins).where(and_(
+            consec_logins.c.user==user_id,
+            consec_logins.c.version==version,
+        ))
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def reset_consec_login(self, user_id: int, version: int) -> Optional[Row]:
+        sql = consec_logins.update(and_(
+            consec_logins.c.user==user_id,
+            consec_logins.c.version==version,
+        )).values(
+            logins=1
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()

titles/mai2/schema/score.py

@@ -7,6 +7,7 @@ from sqlalchemy.engine import Row
 from sqlalchemy.dialects.mysql import insert
 
 from core.data.schema import BaseData, metadata
+from core.data import cached
 
 best_score = Table(
     "mai2_score_best",
@@ -174,29 +175,137 @@ course = Table(
     mysql_charset="utf8mb4",
 )
 
+playlog_old = Table(
+    "maimai_playlog",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer),
+    # Pop access code
+    Column("orderId", Integer),
+    Column("sortNumber", Integer),
+    Column("placeId", Integer),
+    Column("placeName", String(255)),
+    Column("country", String(255)),
+    Column("regionId", Integer),
+    Column("playDate", String(255)),
+    Column("userPlayDate", String(255)),
+    Column("musicId", Integer),
+    Column("level", Integer),
+    Column("gameMode", Integer),
+    Column("rivalNum", Integer),
+    Column("track", Integer),
+    Column("eventId", Integer),
+    Column("isFreeToPlay", Boolean),
+    Column("playerRating", Integer),
+    Column("playedUserId1", Integer),
+    Column("playedUserId2", Integer),
+    Column("playedUserId3", Integer),
+    Column("playedUserName1", String(255)),
+    Column("playedUserName2", String(255)),
+    Column("playedUserName3", String(255)),
+    Column("playedMusicLevel1", Integer),
+    Column("playedMusicLevel2", Integer),
+    Column("playedMusicLevel3", Integer),
+    Column("achievement", Integer),
+    Column("score", Integer),
+    Column("tapScore", Integer),
+    Column("holdScore", Integer),
+    Column("slideScore", Integer),
+    Column("breakScore", Integer),
+    Column("syncRate", Integer),
+    Column("vsWin", Integer),
+    Column("isAllPerfect", Boolean),
+    Column("fullCombo", Integer),
+    Column("maxFever", Integer),
+    Column("maxCombo", Integer),
+    Column("tapPerfect", Integer),
+    Column("tapGreat", Integer),
+    Column("tapGood", Integer),
+    Column("tapBad", Integer),
+    Column("holdPerfect", Integer),
+    Column("holdGreat", Integer),
+    Column("holdGood", Integer),
+    Column("holdBad", Integer),
+    Column("slidePerfect", Integer),
+    Column("slideGreat", Integer),
+    Column("slideGood", Integer),
+    Column("slideBad", Integer),
+    Column("breakPerfect", Integer),
+    Column("breakGreat", Integer),
+    Column("breakGood", Integer),
+    Column("breakBad", Integer),
+    Column("judgeStyle", Integer),
+    Column("isTrackSkip", Boolean),
+    Column("isHighScore", Boolean),
+    Column("isChallengeTrack", Boolean),
+    Column("challengeLife", Integer),
+    Column("challengeRemain", Integer),
+    Column("isAllPerfectPlus", Integer),
+    mysql_charset="utf8mb4",
+)
+
+best_score_old = Table(
+    "maimai_score_best",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("musicId", Integer),
+    Column("level", Integer),
+    Column("playCount", Integer),
+    Column("achievement", Integer),
+    Column("scoreMax", Integer),
+    Column("syncRateMax", Integer),
+    Column("isAllPerfect", Boolean),
+    Column("isAllPerfectPlus", Integer),
+    Column("fullCombo", Integer),
+    Column("maxFever", Integer),
+    UniqueConstraint("user", "musicId", "level", name="maimai_score_best_uk"),
+    mysql_charset="utf8mb4",
+)
 
 class Mai2ScoreData(BaseData):
-    def put_best_score(self, user_id: int, score_data: Dict) -> Optional[int]:
+    def put_best_score(self, user_id: int, score_data: Dict, is_dx: bool = True) -> Optional[int]:
         score_data["user"] = user_id
-        sql = insert(best_score).values(**score_data)
 
+        if is_dx:
+            sql = insert(best_score).values(**score_data)
+        else:
+            sql = insert(best_score_old).values(**score_data)
         conflict = sql.on_duplicate_key_update(**score_data)
 
         result = self.execute(conflict)
         if result is None:
             self.logger.error(
-                f"put_best_score:  Failed to insert best score! user_id {user_id}"
+                f"put_best_score:  Failed to insert best score! user_id {user_id} is_dx {is_dx}"
             )
             return None
         return result.lastrowid
 
-    def get_best_scores(self, user_id: int, song_id: int = None) -> Optional[List[Row]]:
-        sql = best_score.select(
-            and_(
-                best_score.c.user == user_id,
-                (best_score.c.song_id == song_id) if song_id is not None else True,
+    @cached(2)
+    def get_best_scores(self, user_id: int, song_id: int = None, is_dx: bool = True) -> Optional[List[Row]]:
+        if is_dx:
+            sql = best_score.select(
+                and_(
+                    best_score.c.user == user_id,
+                    (best_score.c.song_id == song_id) if song_id is not None else True,
+                )
+            )
+        else:
+            sql = best_score_old.select(
+                and_(
+                    best_score_old.c.user == user_id,
+                    (best_score_old.c.song_id == song_id) if song_id is not None else True,
+                )
             )
-        )
 
         result = self.execute(sql)
         if result is None:
@@ -219,15 +328,19 @@ class Mai2ScoreData(BaseData):
             return None
         return result.fetchone()
 
-    def put_playlog(self, user_id: int, playlog_data: Dict) -> Optional[int]:
+    def put_playlog(self, user_id: int, playlog_data: Dict, is_dx: bool = True) -> Optional[int]:
         playlog_data["user"] = user_id
-        sql = insert(playlog).values(**playlog_data)
+
+        if is_dx:
+            sql = insert(playlog).values(**playlog_data)
+        else:
+            sql = insert(playlog_old).values(**playlog_data)
 
         conflict = sql.on_duplicate_key_update(**playlog_data)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.error(f"put_playlog:  Failed to insert! user_id {user_id}")
+            self.logger.error(f"put_playlog:  Failed to insert! user_id {user_id} is_dx {is_dx}")
             return None
         return result.lastrowid
 
@@ -249,4 +362,4 @@ class Mai2ScoreData(BaseData):
         result = self.execute(sql)
         if result is None:
             return None
-        return result.fetchone()
+        return result.fetchall()

titles/mai2/schema/static.py

@@ -161,7 +161,7 @@ class Mai2StaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert song {song_id} chart {chart_id}")
+            self.logger.warning(f"Failed to insert song {song_id} chart {chart_id}")
             return None
         return result.lastrowid
 
@@ -187,7 +187,7 @@ class Mai2StaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert charge {ticket_id} type {ticket_type}")
+            self.logger.warning(f"Failed to insert charge {ticket_id} type {ticket_type}")
             return None
         return result.lastrowid
 
@@ -237,7 +237,7 @@ class Mai2StaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert card {card_id}")
+            self.logger.warning(f"Failed to insert card {card_id}")
             return None
         return result.lastrowid
 

titles/mai2/splash.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.dxplus import Mai2DXPlus
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2Splash(Mai2Base):
+class Mai2Splash(Mai2DXPlus):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_SPLASH

titles/mai2/splashplus.py

@@ -4,12 +4,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.splash import Mai2Splash
 from titles.mai2.config import Mai2Config
 from titles.mai2.const import Mai2Constants
 
 
-class Mai2SplashPlus(Mai2Base):
+class Mai2SplashPlus(Mai2Splash):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_SPLASH_PLUS

titles/mai2/universe.py

@@ -5,12 +5,12 @@ import pytz
 import json
 
 from core.config import CoreConfig
-from titles.mai2.base import Mai2Base
+from titles.mai2.splashplus import Mai2SplashPlus
 from titles.mai2.const import Mai2Constants
 from titles.mai2.config import Mai2Config
 
 
-class Mai2Universe(Mai2Base):
+class Mai2Universe(Mai2SplashPlus):
     def __init__(self, cfg: CoreConfig, game_cfg: Mai2Config) -> None:
         super().__init__(cfg, game_cfg)
         self.version = Mai2Constants.VER_MAIMAI_DX_UNIVERSE
@@ -70,13 +70,13 @@ class Mai2Universe(Mai2Base):
             tmp.pop("cardName")
             tmp.pop("enabled")
 
-            tmp["startDate"] = datetime.strftime(tmp["startDate"], "%Y-%m-%d %H:%M:%S")
-            tmp["endDate"] = datetime.strftime(tmp["endDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["startDate"] = datetime.strftime(tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT)
+            tmp["endDate"] = datetime.strftime(tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT)
             tmp["noticeStartDate"] = datetime.strftime(
-                tmp["noticeStartDate"], "%Y-%m-%d %H:%M:%S"
+                tmp["noticeStartDate"], Mai2Constants.DATE_TIME_FORMAT
             )
             tmp["noticeEndDate"] = datetime.strftime(
-                tmp["noticeEndDate"], "%Y-%m-%d %H:%M:%S"
+                tmp["noticeEndDate"], Mai2Constants.DATE_TIME_FORMAT
             )
 
             selling_card_list.append(tmp)
@@ -104,8 +104,12 @@ class Mai2Universe(Mai2Base):
             tmp.pop("id")
             tmp.pop("user")
 
-            tmp["startDate"] = datetime.strftime(tmp["startDate"], "%Y-%m-%d %H:%M:%S")
-            tmp["endDate"] = datetime.strftime(tmp["endDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["startDate"] = datetime.strftime(
+                tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["endDate"] = datetime.strftime(
+                tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
             card_list.append(tmp)
 
         return {
@@ -154,6 +158,10 @@ class Mai2Universe(Mai2Base):
         # set a random card serial number
         serial_id = "".join([str(randint(0, 9)) for _ in range(20)])
 
+        # calculate start and end date of the card
+        start_date = datetime.utcnow()
+        end_date = datetime.utcnow() + timedelta(days=15)
+
         user_card = upsert["userCard"]
         self.data.item.put_card(
             user_id,
@@ -161,8 +169,26 @@ class Mai2Universe(Mai2Base):
             user_card["cardTypeId"],
             user_card["charaId"],
             user_card["mapId"],
+            # add the correct start date and also the end date in 15 days
+            start_date,
+            end_date,
         )
 
+        # get the profile extend to save the new bought card
+        extend = self.data.profile.get_profile_extend(user_id, self.version)
+        if extend:
+            extend = extend._asdict()
+            # parse the selectedCardList
+            # 6 = Freedom Pass, 4 = Gold Pass (cardTypeId)
+            selected_cards: List = extend["selectedCardList"]
+
+            # if no pass is already added, add the corresponding pass
+            if not user_card["cardTypeId"] in selected_cards:
+                selected_cards.insert(0, user_card["cardTypeId"])
+
+            extend["selectedCardList"] = selected_cards
+            self.data.profile.put_profile_extend(user_id, self.version, extend)
+
         # properly format userPrintDetail for the database
         upsert.pop("userCard")
         upsert.pop("serialId")
@@ -174,8 +200,8 @@ class Mai2Universe(Mai2Base):
             "returnCode": 1,
             "orderId": 0,
             "serialId": serial_id,
-            "startDate": "2018-01-01 00:00:00",
-            "endDate": "2038-01-01 00:00:00",
+            "startDate": datetime.strftime(start_date, Mai2Constants.DATE_TIME_FORMAT),
+            "endDate": datetime.strftime(end_date, Mai2Constants.DATE_TIME_FORMAT),
         }
 
     def handle_cm_upsert_user_printlog_api_request(self, data: Dict) -> Dict:

titles/ongeki/__init__.py

@@ -2,9 +2,11 @@ from titles.ongeki.index import OngekiServlet
 from titles.ongeki.const import OngekiConstants
 from titles.ongeki.database import OngekiData
 from titles.ongeki.read import OngekiReader
+from titles.ongeki.frontend import OngekiFrontend
 
 index = OngekiServlet
 database = OngekiData
 reader = OngekiReader
+frontend = OngekiFrontend
 game_codes = [OngekiConstants.GAME_CODE]
-current_schema_version = 4
+current_schema_version = 5

titles/ongeki/base.py

@@ -142,7 +142,7 @@ class OngekiBase:
 
     def handle_get_game_point_api_request(self, data: Dict) -> Dict:
         """
-        Sets the GP ammount for A and B sets for 1 - 3 crdits
+        Sets the GP amount for A and B sets for 1 - 3 credits
         """
         return {
             "length": 6,
@@ -155,13 +155,13 @@ class OngekiBase:
                 },
                 {
                     "type": 1,
-                    "cost": 200,
+                    "cost": 230,
                     "startDate": "2000-01-01 05:00:00.0",
                     "endDate": "2099-01-01 05:00:00.0",
                 },
                 {
                     "type": 2,
-                    "cost": 300,
+                    "cost": 370,
                     "startDate": "2000-01-01 05:00:00.0",
                     "endDate": "2099-01-01 05:00:00.0",
                 },
@@ -256,7 +256,11 @@ class OngekiBase:
                 {
                     "type": event["type"],
                     "id": event["eventId"],
-                    "startDate": "2017-12-05 07:00:00.0",
+                    # actually use the startDate from the import so it
+                    # properly shows all the events when new ones are imported
+                    "startDate": datetime.strftime(
+                        event["startDate"], "%Y-%m-%d %H:%M:%S.0"
+                    ),
                     "endDate": "2099-12-31 00:00:00.0",
                 }
             )
@@ -268,7 +272,7 @@ class OngekiBase:
         }
 
     def handle_get_game_id_list_api_request(self, data: Dict) -> Dict:
-        game_idlist: list[str, Any] = []  # 1 to 230 & 8000 to 8050
+        game_idlist: List[str, Any] = []  # 1 to 230 & 8000 to 8050
 
         if data["type"] == 1:
             for i in range(1, 231):
@@ -443,7 +447,7 @@ class OngekiBase:
                 "userItemList": [],
             }
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(data["nextIndex"] % 10000000000, len(p)):
             if len(items) > data["maxCount"]:
                 break
@@ -560,7 +564,11 @@ class OngekiBase:
     def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
         recent_rating = self.data.profile.get_profile_recent_rating(data["userId"])
         if recent_rating is None:
-            return {}
+            return {
+                "userId": data["userId"],
+                "length": 0,
+                "userRecentRatingList": [],
+            }
 
         userRecentRatingList = recent_rating["recentRating"]
 
@@ -970,35 +978,38 @@ class OngekiBase:
         """
         Added in Bright
         """
-        rival_list = self.data.profile.get_rivals(data["userId"])
-        if rival_list is None or len(rival_list) < 1:
+
+        rival_list = []
+        user_rivals = self.data.profile.get_rivals(data["userId"])
+        for rival in user_rivals:
+            tmp = {}
+            tmp["rivalUserId"] = rival[0]
+            rival_list.append(tmp)
+
+        if user_rivals is None or len(rival_list) < 1:
             return {
                 "userId": data["userId"],
                 "length": 0,
                 "userRivalList": [],
             }
-
         return {
             "userId": data["userId"],
             "length": len(rival_list),
-            "userRivalList": rival_list._asdict(),
+            "userRivalList": rival_list,
         }
 
-    def handle_get_user_rival_data_api_reqiest(self, data: Dict) -> Dict:
+    def handle_get_user_rival_data_api_request(self, data: Dict) -> Dict:
         """
         Added in Bright
         """
         rivals = []
-
         for rival in data["userRivalList"]:
             name = self.data.profile.get_profile_name(
                 rival["rivalUserId"], self.version
             )
             if name is None:
                 continue
-
-            rivals.append({"rivalUserId": rival["rival"], "rivalUserName": name})
-
+            rivals.append({"rivalUserId": rival["rivalUserId"], "rivalUserName": name})
         return {
             "userId": data["userId"],
             "length": len(rivals),
@@ -1019,7 +1030,6 @@ class OngekiBase:
         for song in music["userMusicList"]:
             song["userRivalMusicDetailList"] = song["userMusicDetailList"]
             song.pop("userMusicDetailList")
-
         return {
             "userId": data["userId"],
             "rivalUserId": rival_id,

titles/ongeki/bright.py

@@ -43,15 +43,15 @@ class OngekiBright(OngekiBase):
         user_data.pop("user")
         user_data.pop("version")
 
-        # TODO: replace datetime objects with strings
-
         # add access code that we don't store
         user_data["accessCode"] = cards[0]["access_code"]
 
-        # hardcode Card Maker version for now
-        # Card Maker 1.34.00 = 1.30.01
-        # Card Maker 1.36.00 = 1.35.04
-        user_data["compatibleCmVersion"] = "1.30.01"
+        # add the compatible card maker version from config
+        card_maker_ver = self.game_cfg.version.version(self.version)
+        if card_maker_ver and card_maker_ver.get("card_maker"):
+            # Card Maker 1.30 = 1.30.01+
+            # Card Maker 1.35 = 1.35.03+
+            user_data["compatibleCmVersion"] = card_maker_ver.get("card_maker")
 
         return {"userId": data["userId"], "userData": user_data}
 
@@ -333,6 +333,8 @@ class OngekiBright(OngekiBase):
         select_point = data["selectPoint"]
 
         total_gacha_count, ceiling_gacha_count = 0, 0
+        # 0 = can still use Gacha Select, 1 = already used Gacha Select
+        use_select_point = 0
         daily_gacha_cnt, five_gacha_cnt, eleven_gacha_cnt = 0, 0, 0
         daily_gacha_date = datetime.strptime("2000-01-01", "%Y-%m-%d")
 
@@ -344,6 +346,9 @@ class OngekiBright(OngekiBase):
             daily_gacha_cnt = user_gacha["dailyGachaCnt"]
             five_gacha_cnt = user_gacha["fiveGachaCnt"]
             eleven_gacha_cnt = user_gacha["elevenGachaCnt"]
+            # if the Gacha Select has been used, make sure to keep it
+            if user_gacha["useSelectPoint"] == 1:
+                use_select_point = 1
             # parse just the year, month and date
             daily_gacha_date = user_gacha["dailyGachaDate"]
 
@@ -359,7 +364,7 @@ class OngekiBright(OngekiBase):
             totalGachaCnt=total_gacha_count + gacha_count,
             ceilingGachaCnt=ceiling_gacha_count + gacha_count,
             selectPoint=select_point,
-            useSelectPoint=0,
+            useSelectPoint=use_select_point,
             dailyGachaCnt=daily_gacha_cnt + gacha_count,
             fiveGachaCnt=five_gacha_cnt + 1 if gacha_count == 5 else five_gacha_cnt,
             elevenGachaCnt=eleven_gacha_cnt + 1

titles/ongeki/brightmemory.py

@@ -136,14 +136,3 @@ class OngekiBrightMemory(OngekiBright):
 
     def handle_get_game_music_release_state_api_request(self, data: Dict) -> Dict:
         return {"techScore": 0, "cardNum": 0}
-
-    def handle_cm_get_user_data_api_request(self, data: Dict) -> Dict:
-        # check for a bright memory profile
-        user_data = super().handle_cm_get_user_data_api_request(data)
-
-        # hardcode Card Maker version for now
-        # Card Maker 1.34 = 1.30.01
-        # Card Maker 1.35 = 1.35.03
-        user_data["userData"]["compatibleCmVersion"] = "1.35.03"
-
-        return user_data

titles/ongeki/config.py

@@ -1,3 +1,4 @@
+from ast import Dict
 from typing import List
 
 from core.config import CoreConfig
@@ -33,7 +34,44 @@ class OngekiGachaConfig:
         )
 
 
+class OngekiCardMakerVersionConfig:
+    def __init__(self, parent_config: "OngekiConfig") -> None:
+        self.__config = parent_config
+
+    def version(self, version: int) -> Dict:
+        """
+        in the form of:
+        <ongeki version>: {"card_maker": <compatible card maker version>}
+        6: {"card_maker": 1.30.01}
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "ongeki", "version", default={}
+        ).get(version)
+
+class OngekiCryptoConfig:
+    def __init__(self, parent_config: "OngekiConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def keys(self) -> Dict:
+        """
+        in the form of:
+        internal_version: [key, iv]
+        all values are hex strings
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "ongeki", "crypto", "keys", default={}
+        )
+
+    @property
+    def encrypted_only(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "ongeki", "crypto", "encrypted_only", default=False
+        )
+
 class OngekiConfig(dict):
     def __init__(self) -> None:
         self.server = OngekiServerConfig(self)
         self.gachas = OngekiGachaConfig(self)
+        self.version = OngekiCardMakerVersionConfig(self)
+        self.crypto = OngekiCryptoConfig(self)

titles/ongeki/const.py

@@ -66,13 +66,13 @@ class OngekiConstants:
 
     VERSION_NAMES = (
         "ONGEKI",
-        "ONGEKI+",
-        "ONGEKI Summer",
-        "ONGEKI Summer+",
-        "ONGEKI Red",
-        "ONGEKI Red+",
-        "ONGEKI Bright",
-        "ONGEKI Bright Memory",
+        "ONGEKI +",
+        "ONGEKI SUMMER",
+        "ONGEKI SUMMER +",
+        "ONGEKI R.E.D.",
+        "ONGEKI R.E.D. +",
+        "ONGEKI bright",
+        "ONGEKI bright MEMORY",
     )
 
     @classmethod

titles/ongeki/frontend.py

@@ -0,0 +1,87 @@
+import yaml
+import jinja2
+from twisted.web.http import Request
+from os import path
+from twisted.web.util import redirectTo
+from twisted.web.server import Session
+
+from core.frontend import FE_Base, IUserSession
+from core.config import CoreConfig
+
+from titles.ongeki.config import OngekiConfig
+from titles.ongeki.const import OngekiConstants
+from titles.ongeki.database import OngekiData
+from titles.ongeki.base import OngekiBase
+
+
+class OngekiFrontend(FE_Base):
+    def __init__(
+        self, cfg: CoreConfig, environment: jinja2.Environment, cfg_dir: str
+    ) -> None:
+        super().__init__(cfg, environment)
+        self.data = OngekiData(cfg)
+        self.game_cfg = OngekiConfig()
+        if path.exists(f"{cfg_dir}/{OngekiConstants.CONFIG_NAME}"):
+            self.game_cfg.update(
+                yaml.safe_load(open(f"{cfg_dir}/{OngekiConstants.CONFIG_NAME}"))
+            )
+        self.nav_name = "O.N.G.E.K.I."
+        self.version_list = OngekiConstants.VERSION_NAMES
+
+    def render_GET(self, request: Request) -> bytes:
+        template = self.environment.get_template(
+            "titles/ongeki/frontend/ongeki_index.jinja"
+        )
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        self.version = usr_sesh.ongeki_version
+        if getattr(usr_sesh, "userId", 0) != 0:
+            profile_data =self.data.profile.get_profile_data(usr_sesh.userId, self.version)
+            rival_list = self.data.profile.get_rivals(usr_sesh.userId)
+            rival_data = {
+                "userRivalList": rival_list,
+                "userId": usr_sesh.userId
+            }
+            rival_info = OngekiBase.handle_get_user_rival_data_api_request(self, rival_data)
+
+            return template.render(
+                data=self.data.profile,
+                title=f"{self.core_config.server.name} | {self.nav_name}",
+                game_list=self.environment.globals["game_list"],
+                gachas=self.game_cfg.gachas.enabled_gachas,
+                profile_data=profile_data,
+                rival_info=rival_info["userRivalDataList"],
+                version_list=self.version_list,
+                version=self.version,
+                sesh=vars(usr_sesh)
+            ).encode("utf-16")
+        else:
+            return redirectTo(b"/gate/", request)
+    
+    def render_POST(self, request: Request):
+        uri = request.uri.decode()
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if hasattr(usr_sesh, "userId"):
+            if uri == "/game/ongeki/rival.add":
+                rival_id = request.args[b"rivalUserId"][0].decode()
+                self.data.profile.put_rival(usr_sesh.userId, rival_id)
+                # self.logger.info(f"{usr_sesh.userId} added a rival")
+                return redirectTo(b"/game/ongeki/", request)
+            
+            elif uri == "/game/ongeki/rival.delete":
+                rival_id = request.args[b"rivalUserId"][0].decode()
+                self.data.profile.delete_rival(usr_sesh.userId, rival_id)
+                # self.logger.info(f"{response}")
+                return redirectTo(b"/game/ongeki/", request)
+            
+            elif uri == "/game/ongeki/version.change":
+                ongeki_version=request.args[b"version"][0].decode()
+                if(ongeki_version.isdigit()):
+                    usr_sesh.ongeki_version=int(ongeki_version)
+                return redirectTo(b"/game/ongeki/", request)
+            
+            else:
+                return b"Something went wrong"
+        else:
+            return b"User is not logged in"

titles/ongeki/frontend/js/ongeki_scripts.js

@@ -0,0 +1,24 @@
+function deleteRival(rivalUserId){
+
+    $(document).ready(function () {
+        $.post("/game/ongeki/rival.delete",
+        {
+            rivalUserId
+        },
+        function(data,status){
+            window.location.replace("/game/ongeki/")
+        })
+    });
+}
+function changeVersion(sel){
+
+    $(document).ready(function () {
+        $.post("/game/ongeki/version.change",
+        {
+            version: sel.value
+        },
+        function(data,status){
+            window.location.replace("/game/ongeki/")
+        })
+    });
+}

titles/ongeki/frontend/ongeki_index.jinja

@@ -0,0 +1,83 @@
+{% extends "core/frontend/index.jinja" %}
+{% block content %}
+
+{% if sesh is defined and sesh["userId"] > 0 %}
+<br>
+<br>
+<br>
+
+<div class="container">
+    <div class="row">
+        <h2> Profile </h2>
+        <h3>Version: 
+            <select name="version" id="version" onChange="changeVersion(this)">
+            {% for ver in version_list %}
+                <option value={{loop.index0}} {{ "selected" if loop.index0==version else "" }} >{{ver}}</option>
+            {% endfor %}
+            </select>
+        </h3>
+    <hr>
+    </div>
+    <div class="row">
+     <div class="col">
+        <h2> Name: {{ profile_data.userName if profile_data.userName is defined else "Profile not found" }}</h2>
+     </div>
+        <div class="col">
+            <h4> ID: {{ profile_data.user if profile_data.user is defined else 'Profile not found' }}</h4>
+        </div>
+    </div>
+    <hr>
+    <div class="row">
+        <h2> Rivals <button class="btn btn-success" data-bs-toggle="modal" data-bs-target="#rival_add">Add</button></h2>
+    </div>
+    <div class="row">
+    <table class="table table-dark table-hover">
+        <thead>
+        <tr>
+            <th scope="col">ID</th>
+            <th scope="col">Name</th>
+            <th scope="col">Delete</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for rival in rival_info%}
+            <tr id="{{rival.rivalUserId}}">
+                <td>{{rival.rivalUserId}}</td>
+                <td>{{rival.rivalUserName}}</td>
+                <td><button class="btn-danger btn btn-sm" onclick="deleteRival({{rival.rivalUserId}})">Delete</button></td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+    </div>
+    <div class="modal fade" id="rival_add" tabindex="-1" aria-labelledby="card_add_label"  data-bs-theme="dark" aria-hidden="true">
+        <form id="rival" action="/game/ongeki/rival.add" method="post">
+            <div class="modal-dialog">
+                <div class="modal-content">
+                    <div class="modal-header">
+                        <h5 class="modal-title">Modal title</h5>
+                        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+                    </div>
+                    <div class="modal-body">
+                        Note:<br>
+                        Please use the ID show next to your name in the profile page.
+                        <br>
+                        <label for="rivalUserId">ID:&nbsp;</label><input form="rival" id="rivalUserId" name="rivalUserId" maxlength="5" type="number" required> 
+                    </div>
+                    <div class="modal-footer">
+                        <input type=submit class="btn btn-primary" type="button" form="rival" value="Add">
+                        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+                    </div>
+                </div>
+            </div>
+        </form>
+    </div>
+</div>
+
+<script>
+{% include 'titles/ongeki/frontend/js/ongeki_scripts.js' %}
+</script>
+{% else %}
+<h2>Not Currently Logged In</h2>
+{% endif %}
+{% endblock content %}

titles/ongeki/index.py

@@ -7,10 +7,15 @@ import logging
 import coloredlogs
 import zlib
 from logging.handlers import TimedRotatingFileHandler
+from Crypto.Cipher import AES
+from Crypto.Util.Padding import pad
+from Crypto.Protocol.KDF import PBKDF2
+from Crypto.Hash import SHA1
 from os import path
 from typing import Tuple
 
 from core.config import CoreConfig
+from core.utils import Utils
 from titles.ongeki.config import OngekiConfig
 from titles.ongeki.const import OngekiConstants
 from titles.ongeki.base import OngekiBase
@@ -27,6 +32,7 @@ class OngekiServlet:
     def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
         self.core_cfg = core_cfg
         self.game_cfg = OngekiConfig()
+        self.hash_table: Dict[Dict[str, str]] = {}
         if path.exists(f"{cfg_dir}/{OngekiConstants.CONFIG_NAME}"):
             self.game_cfg.update(
                 yaml.safe_load(open(f"{cfg_dir}/{OngekiConstants.CONFIG_NAME}"))
@@ -44,27 +50,60 @@ class OngekiServlet:
         ]
 
         self.logger = logging.getLogger("ongeki")
-        log_fmt_str = "[%(asctime)s] Ongeki | %(levelname)s | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
-        fileHandler = TimedRotatingFileHandler(
-            "{0}/{1}.log".format(self.core_cfg.server.log_dir, "ongeki"),
-            encoding="utf8",
-            when="d",
-            backupCount=10,
-        )
 
-        fileHandler.setFormatter(log_fmt)
+        if not hasattr(self.logger, "inited"):
+            log_fmt_str = "[%(asctime)s] Ongeki | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.core_cfg.server.log_dir, "ongeki"),
+                encoding="utf8",
+                when="d",
+                backupCount=10,
+            )
 
-        consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
+            fileHandler.setFormatter(log_fmt)
 
-        self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
 
-        self.logger.setLevel(self.game_cfg.server.loglevel)
-        coloredlogs.install(
-            level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
-        )
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(self.game_cfg.server.loglevel)
+            coloredlogs.install(
+                level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.inited = True
+
+        for version, keys in self.game_cfg.crypto.keys.items():
+            if len(keys) < 3:
+                continue
+
+            self.hash_table[version] = {}
+
+            method_list = [
+                method
+                for method in dir(self.versions[version])
+                if not method.startswith("__")
+            ]
+            for method in method_list:
+                method_fixed = inflection.camelize(method)[6:-7]
+                # number of iterations is 64 on Bright Memory
+                iter_count = 64
+                hash = PBKDF2(
+                    method_fixed,
+                    bytes.fromhex(keys[2]),
+                    128,
+                    count=iter_count,
+                    hmac_hash_module=SHA1,
+                )
+
+                hashed_name = hash.hex()[:32] # truncate unused bytes like the game does
+                self.hash_table[version][hashed_name] = method_fixed
+
+                self.logger.debug(
+                    f"Hashed v{version} method {method_fixed} with {bytes.fromhex(keys[2])} to get {hash.hex()[:32]}"
+                )
 
     @classmethod
     def get_allnet_info(
@@ -99,8 +138,10 @@ class OngekiServlet:
 
         req_raw = request.content.getvalue()
         url_split = url_path.split("/")
+        encrtped = False
         internal_ver = 0
         endpoint = url_split[len(url_split) - 1]
+        client_ip = Utils.get_ip_addr(request)
 
         if version < 105:  # 1.0
             internal_ver = OngekiConstants.VER_ONGEKI
@@ -123,8 +164,45 @@ class OngekiServlet:
             # If we get a 32 character long hex string, it's a hash and we're
             # doing encrypted. The likelyhood of false positives is low but
             # technically not 0
-            self.logger.error("Encryption not supported at this time")
-            return b""
+            if internal_ver not in self.hash_table:
+                self.logger.error(
+                    f"v{version} does not support encryption or no keys entered"
+                )
+                return zlib.compress(b'{"stat": "0"}')
+
+            elif endpoint.lower() not in self.hash_table[internal_ver]:
+                self.logger.error(
+                    f"No hash found for v{version} endpoint {endpoint}"
+                )
+                return zlib.compress(b'{"stat": "0"}')
+
+            endpoint = self.hash_table[internal_ver][endpoint.lower()]
+
+            try:
+                crypt = AES.new(
+                    bytes.fromhex(self.game_cfg.crypto.keys[internal_ver][0]),
+                    AES.MODE_CBC,
+                    bytes.fromhex(self.game_cfg.crypto.keys[internal_ver][1]),
+                )
+
+                req_raw = crypt.decrypt(req_raw)
+
+            except Exception as e:
+                self.logger.error(
+                    f"Failed to decrypt v{version} request to {endpoint} -> {e}"
+                )
+                return zlib.compress(b'{"stat": "0"}')
+
+            encrtped = True
+
+        if (
+            not encrtped
+            and self.game_cfg.crypto.encrypted_only
+        ):
+            self.logger.error(
+                f"Unencrypted v{version} {endpoint} request, but config is set to encrypted only: {req_raw}"
+            )
+            return zlib.compress(b'{"stat": "0"}')
 
         try:
             unzip = zlib.decompress(req_raw)
@@ -137,7 +215,10 @@ class OngekiServlet:
 
         req_data = json.loads(unzip)
 
-        self.logger.info(f"v{version} {endpoint} request - {req_data}")
+        self.logger.info(
+            f"v{version} {endpoint} request from {client_ip}"
+        )
+        self.logger.debug(req_data)
 
         func_to_find = "handle_" + inflection.underscore(endpoint) + "_request"
 
@@ -156,6 +237,19 @@ class OngekiServlet:
         if resp == None:
             resp = {"returnCode": 1}
 
-        self.logger.info(f"Response {resp}")
+        self.logger.debug(f"Response {resp}")
 
-        return zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))
+        zipped = zlib.compress(json.dumps(resp, ensure_ascii=False).encode("utf-8"))
+
+        if not encrtped:
+            return zipped
+
+        padded = pad(zipped, 16)
+
+        crypt = AES.new(
+            bytes.fromhex(self.game_cfg.crypto.keys[internal_ver][0]),
+            AES.MODE_CBC,
+            bytes.fromhex(self.game_cfg.crypto.keys[internal_ver][1]),
+        )
+
+        return crypt.encrypt(padded)

titles/ongeki/schema/item.py

@@ -326,7 +326,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_card: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_card: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -346,7 +346,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_character: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_character: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -366,7 +366,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_deck: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_deck: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -394,7 +394,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_boss: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_boss: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -406,7 +406,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_story: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_story: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -426,7 +426,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_chapter: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_chapter: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -446,7 +446,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_item: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_item: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -479,7 +479,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_music_item: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_music_item: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -499,7 +499,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_login_bonus: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_login_bonus: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -521,7 +521,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_mission_point: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_mission_point: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -541,7 +541,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_event_point: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_event_point: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -561,7 +561,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_scenerio: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_scenerio: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -581,7 +581,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_trade_item: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_trade_item: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -601,7 +601,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_event_music: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_event_music: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -621,7 +621,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_tech_event: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_tech_event: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -651,7 +651,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_memorychapter: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_memorychapter: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -694,7 +694,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
+            self.logger.warning(f"put_user_gacha: Failed to insert! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -709,7 +709,7 @@ class OngekiItemData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_user_print_detail: Failed to insert! aime_id: {aime_id}"
             )
             return None

titles/ongeki/schema/log.py

@@ -63,7 +63,7 @@ class OngekiLogData(BaseData):
 
         result = self.execute(sql)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_gp_log: Failed to insert GP log! aime_id: {aime_id} kind {kind} pattern {pattern} current_gp {current_gp}"
             )
         return result.lastrowid

titles/ongeki/schema/profile.py

@@ -3,7 +3,7 @@ from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, an
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, BigInteger
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.schema import ForeignKey
-from sqlalchemy.sql import func, select
+from sqlalchemy.sql import func, select, delete
 from sqlalchemy.engine import Row
 from sqlalchemy.dialects.mysql import insert
 
@@ -269,7 +269,7 @@ class OngekiProfileData(BaseData):
             return None
 
         return row["userName"]
-
+    
     def get_profile_preview(self, aime_id: int, version: int) -> Optional[Row]:
         sql = (
             select([profile, option])
@@ -316,7 +316,7 @@ class OngekiProfileData(BaseData):
         return result.fetchone()
 
     def get_profile_rating_log(self, aime_id: int) -> Optional[List[Row]]:
-        sql = select(rating_log).where(recent_rating.c.user == aime_id)
+        sql = select(rating_log).where(rating_log.c.user == aime_id)
 
         result = self.execute(sql)
         if result is None:
@@ -364,7 +364,7 @@ class OngekiProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_profile_data: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_profile_data: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -376,7 +376,7 @@ class OngekiProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_options: Failed to update! aime_id: {aime_id}"
             )
             return None
@@ -393,7 +393,7 @@ class OngekiProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_recent_rating: failed to update recent rating! aime_id {aime_id}"
             )
             return None
@@ -415,7 +415,7 @@ class OngekiProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_rating_log: failed to update rating log! aime_id {aime_id} data_version {data_version} highest_rating {highest_rating}"
             )
             return None
@@ -449,7 +449,7 @@ class OngekiProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_activity: failed to put activity! aime_id {aime_id} kind {kind} activity_id {activity_id}"
             )
             return None
@@ -466,7 +466,7 @@ class OngekiProfileData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_profile_region: failed to update! aime_id {aime_id} region {region}"
             )
             return None
@@ -480,7 +480,7 @@ class OngekiProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_best_score: Failed to add score! aime_id: {aime_id}")
+            self.logger.warning(f"put_best_score: Failed to add score! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -492,19 +492,26 @@ class OngekiProfileData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_kop: Failed to add score! aime_id: {aime_id}")
+            self.logger.warning(f"put_kop: Failed to add score! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
     def put_rival(self, aime_id: int, rival_id: int) -> Optional[int]:
         sql = insert(rival).values(user=aime_id, rivalUserId=rival_id)
 
-        conflict = sql.on_duplicate_key_update(rival=rival_id)
+        conflict = sql.on_duplicate_key_update(rivalUserId=rival_id)
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_rival: failed to update! aime_id: {aime_id}, rival_id: {rival_id}"
             )
             return None
         return result.lastrowid
+    def delete_rival(self, aime_id: int, rival_id: int) -> Optional[int]:
+        sql = delete(rival).where(rival.c.user==aime_id, rival.c.rivalUserId==rival_id)
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"delete_rival: failed to delete! aime_id: {aime_id}, rival_id: {rival_id}")
+        else:
+            return result.rowcount

titles/ongeki/schema/score.py

@@ -139,7 +139,7 @@ class OngekiScoreData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_tech_count: Failed to update! aime_id: {aime_id}")
+            self.logger.warning(f"put_tech_count: Failed to update! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -164,7 +164,7 @@ class OngekiScoreData(BaseData):
         result = self.execute(conflict)
 
         if result is None:
-            self.logger.warn(f"put_best_score: Failed to add score! aime_id: {aime_id}")
+            self.logger.warning(f"put_best_score: Failed to add score! aime_id: {aime_id}")
             return None
         return result.lastrowid
 
@@ -175,6 +175,6 @@ class OngekiScoreData(BaseData):
 
         result = self.execute(sql)
         if result is None:
-            self.logger.warn(f"put_playlog: Failed to add playlog! aime_id: {aime_id}")
+            self.logger.warning(f"put_playlog: Failed to add playlog! aime_id: {aime_id}")
             return None
         return result.lastrowid

titles/ongeki/schema/static.py

@@ -16,6 +16,7 @@ events = Table(
     Column("eventId", Integer),
     Column("type", Integer),
     Column("name", String(255)),
+    Column("startDate", TIMESTAMP, server_default=func.now()),
     Column("enabled", Boolean, server_default="1"),
     UniqueConstraint("version", "eventId", "type", name="ongeki_static_events_uk"),
     mysql_charset="utf8mb4",
@@ -104,7 +105,7 @@ class OngekiStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert card! card_id {card_id}")
+            self.logger.warning(f"Failed to insert card! card_id {card_id}")
             return None
         return result.lastrowid
 
@@ -179,7 +180,7 @@ class OngekiStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert gacha! gacha_id {gacha_id}")
+            self.logger.warning(f"Failed to insert gacha! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
@@ -214,7 +215,7 @@ class OngekiStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert gacha card! gacha_id {gacha_id}")
+            self.logger.warning(f"Failed to insert gacha card! gacha_id {gacha_id}")
             return None
         return result.lastrowid
 
@@ -242,7 +243,7 @@ class OngekiStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert event! event_id {event_id}")
+            self.logger.warning(f"Failed to insert event! event_id {event_id}")
             return None
         return result.lastrowid
 
@@ -303,7 +304,7 @@ class OngekiStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"Failed to insert chart! song_id: {song_id}, chart_id: {chart_id}"
             )
             return None

titles/pokken/base.py

@@ -1,6 +1,6 @@
 from datetime import datetime, timedelta
 import json, logging
-from typing import Any, Dict
+from typing import Any, Dict, List
 import random
 
 from core.data import Data
@@ -8,6 +8,7 @@ from core import CoreConfig
 from .config import PokkenConfig
 from .proto import jackal_pb2
 from .database import PokkenData
+from .const import PokkenConstants
 
 
 class PokkenBase:
@@ -44,19 +45,19 @@ class PokkenBase:
         biwa_setting = {
             "MatchingServer": {
                 "host": f"https://{self.game_cfg.server.hostname}",
-                "port": self.game_cfg.server.port,
+                "port": self.game_cfg.ports.game,
                 "url": "/SDAK/100/matching",
             },
             "StunServer": {
-                "addr": self.game_cfg.server.hostname,
-                "port": self.game_cfg.server.port_stun,
+                "addr": self.game_cfg.server.stun_server_host,
+                "port": self.game_cfg.server.stun_server_port,
             },
             "TurnServer": {
-                "addr": self.game_cfg.server.hostname,
-                "port": self.game_cfg.server.port_turn,
+                "addr": self.game_cfg.server.stun_server_host,
+                "port": self.game_cfg.server.stun_server_port,
             },
-            "AdmissionUrl": f"ws://{self.game_cfg.server.hostname}:{self.game_cfg.server.port_admission}",
-            "locationId": 123,
+            "AdmissionUrl": f"ws://{self.game_cfg.server.hostname}:{self.game_cfg.ports.admission}",
+            "locationId": 123, # FIXME: Get arcade's ID from the database
             "logfilename": "JackalMatchingLibrary.log",
             "biwalogfilename": "./biwa.log",
         }
@@ -94,6 +95,7 @@ class PokkenBase:
         res.type = jackal_pb2.MessageType.LOAD_CLIENT_SETTINGS
         settings = jackal_pb2.LoadClientSettingsResponseData()
 
+        # TODO: Make configurable
         settings.money_magnification = 1
         settings.continue_bonus_exp = 100
         settings.continue_fight_money = 100
@@ -155,7 +157,6 @@ class PokkenBase:
         support_set_3
         aid_skill_list
         achievement_flag
-        pokemon_data
         event_achievement_flag
         event_achievement_param
         """
@@ -198,7 +199,7 @@ class PokkenBase:
         load_usr.home_loc_name = profile_dict.get("home_loc_name", "")
         load_usr.pref_code = profile_dict.get("pref_code", 0)
         load_usr.trainer_name = profile_dict.get(
-            "trainer_name", "Newb" + str(random.randint(1111, 999999))
+            "trainer_name", f"Newb{str(user_id).zfill(4)}"
         )
         load_usr.trainer_rank_point = profile_dict.get("trainer_rank_point", 0)
         load_usr.wallet = profile_dict.get("wallet", 0)
@@ -260,6 +261,29 @@ class PokkenBase:
         load_usr.sp_bonus_key_value_2 = profile_dict.get("sp_bonus_key_value_2", 0)
         load_usr.last_play_event_id = profile_dict.get("last_play_event_id", 0)
 
+        if pokemon_data is not None:
+            for pkmn in pokemon_data:
+                pkmn_d = pkmn._asdict()
+                pkm = jackal_pb2.LoadUserResponseData.PokemonData()
+
+                pkm.char_id = pkmn_d.get('char_id', 0)
+                pkm.illustration_book_no = pkmn_d.get('illustration_book_no', 0)
+                pkm.pokemon_exp = pkmn_d.get('pokemon_exp', 0)
+                pkm.battle_num_vs_wan = pkmn_d.get('battle_num_vs_wan', 0)
+                pkm.win_vs_wan = pkmn_d.get('win_vs_wan', 0)
+                pkm.battle_num_vs_lan = pkmn_d.get('battle_num_vs_lan', 0)
+                pkm.win_vs_lan = pkmn_d.get('win_vs_lan', 0)
+                pkm.battle_num_vs_cpu = pkmn_d.get('battle_num_vs_cpu', 0)
+                pkm.win_cpu = pkmn_d.get('win_cpu', 0)
+                pkm.battle_all_num_tutorial = pkmn_d.get('battle_all_num_tutorial', 0)
+                pkm.battle_num_tutorial = pkmn_d.get('battle_num_tutorial', 0)
+                pkm.bp_point_atk = pkmn_d.get('bp_point_atk', 0)
+                pkm.bp_point_res = pkmn_d.get('bp_point_res', 0)
+                pkm.bp_point_def = pkmn_d.get('bp_point_def', 0)
+                pkm.bp_point_sp = pkmn_d.get('bp_point_sp', 0)
+
+                load_usr.pokemon_data.append(pkm)
+
         res.load_user.CopyFrom(load_usr)
         return res.SerializeToString()
 
@@ -274,6 +298,100 @@ class PokkenBase:
         res.result = 1
         res.type = jackal_pb2.MessageType.SAVE_USER
 
+        req = request.save_user
+        user_id = req.banapass_id
+        
+        tut_flgs: List[int] = []
+        ach_flgs: List[int] = []
+        evt_flgs: List[int] = []
+        evt_params: List[int] = []
+
+        get_rank_pts: int = req.get_trainer_rank_point if req.get_trainer_rank_point else 0
+        get_money: int = req.get_money
+        get_score_pts: int = req.get_score_point if req.get_score_point else 0
+        grade_max: int = req.grade_max_num
+        extra_counter: int = req.extra_counter
+        evt_reward_get_flg: int = req.event_reward_get_flag
+        num_continues: int = req.continue_num
+        total_play_days: int = req.total_play_days
+        awake_num: int = req.awake_num  # ?
+        use_support_ct: int = req.use_support_num
+        beat_num: int = req.beat_num # ?
+        evt_state: int = req.event_state
+        aid_skill: int = req.aid_skill
+        last_evt: int = req.last_play_event_id
+
+        battle = req.battle_data
+        mon = req.pokemon_data
+
+        p = self.data.profile.touch_profile(user_id)
+        if p is None or not p:
+            self.data.profile.create_profile(user_id)
+
+        if req.trainer_name_pending is not None and req.trainer_name_pending: # we're saving for the first time
+            self.data.profile.set_profile_name(user_id, req.trainer_name_pending, req.avatar_gender if req.avatar_gender else None)
+
+        for tut_flg in req.tutorial_progress_flag:
+            tut_flgs.append(tut_flg)
+        
+        self.data.profile.update_profile_tutorial_flags(user_id, tut_flgs)
+
+        for ach_flg in req.achievement_flag:
+            ach_flgs.append(ach_flg)
+        
+        self.data.profile.update_profile_tutorial_flags(user_id, ach_flg)
+
+        for evt_flg in req.event_achievement_flag:
+            evt_flgs.append(evt_flg)
+
+        for evt_param in req.event_achievement_param:
+            evt_params.append(evt_param)
+
+        self.data.profile.update_profile_event(user_id, evt_state, evt_flgs, evt_params, req.last_play_event_id)
+        
+        for reward in req.reward_data:
+            self.data.item.add_reward(user_id, reward.get_category_id, reward.get_content_id, reward.get_type_id)
+        
+        self.data.profile.add_profile_points(user_id, get_rank_pts, get_money, get_score_pts, grade_max)
+        
+        self.data.profile.update_support_team(user_id, 1, req.support_set_1[0], req.support_set_1[1])
+        self.data.profile.update_support_team(user_id, 2, req.support_set_2[0], req.support_set_2[1])
+        self.data.profile.update_support_team(user_id, 3, req.support_set_3[0], req.support_set_3[1])
+
+        self.data.profile.put_pokemon(user_id, mon.char_id, mon.illustration_book_no, mon.bp_point_atk, mon.bp_point_res, mon.bp_point_def, mon.bp_point_sp)
+        self.data.profile.add_pokemon_xp(user_id, mon.char_id, mon.get_pokemon_exp)
+        
+        for x in range(len(battle.play_mode)):
+            self.data.profile.put_pokemon_battle_result(
+                user_id, 
+                mon.char_id, 
+                PokkenConstants.BATTLE_TYPE(battle.play_mode[x]), 
+                PokkenConstants.BATTLE_RESULT(battle.result[x])
+            )
+
+        self.data.profile.put_stats(
+            user_id,
+            battle.ex_ko_num,
+            battle.wko_num,
+            battle.timeup_win_num,
+            battle.cool_ko_num,
+            battle.perfect_ko_num,
+            num_continues
+        )
+
+        self.data.profile.put_extra(
+            user_id,
+            extra_counter,
+            evt_reward_get_flg,
+            total_play_days,
+            awake_num,
+            use_support_ct,
+            beat_num,
+            aid_skill,
+            last_evt
+        )
+
+
         return res.SerializeToString()
 
     def handle_save_ingame_log(self, data: jackal_pb2.Request) -> bytes:
@@ -302,16 +420,35 @@ class PokkenBase:
         self, data: Dict = {}, client_ip: str = "127.0.0.1"
     ) -> Dict:
         """
-        "sessionId":"12345678",
+                "sessionId":"12345678",
+                "A":{
+                    "pcb_id": data["data"]["must"]["pcb_id"],
+                    "gip": client_ip
+                },
+        """
+        return {
+            "data": {
+                "sessionId":"12345678",
                 "A":{
                     "pcb_id": data["data"]["must"]["pcb_id"],
                     "gip": client_ip
                 },
                 "list":[]
-        """
-        return {}
+            }
+        }
 
     def handle_matching_stop_matching(
         self, data: Dict = {}, client_ip: str = "127.0.0.1"
     ) -> Dict:
         return {}
+
+    def handle_admission_noop(self, data: Dict, req_ip: str = "127.0.0.1") -> Dict:
+        return {}
+    
+    def handle_admission_joinsession(self, data: Dict, req_ip: str = "127.0.0.1") -> Dict:
+        self.logger.info(f"Admission: JoinSession from {req_ip}")
+        return {
+            'data': {
+                "id": 12345678
+            }
+        }

titles/pokken/config.py

@@ -25,30 +25,6 @@ class PokkenServerConfig:
             )
         )
 
-    @property
-    def port(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "pokken", "server", "port", default=9000
-        )
-
-    @property
-    def port_stun(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "pokken", "server", "port_stun", default=9001
-        )
-
-    @property
-    def port_turn(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "pokken", "server", "port_turn", default=9002
-        )
-
-    @property
-    def port_admission(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "pokken", "server", "port_admission", default=9003
-        )
-
     @property
     def auto_register(self) -> bool:
         """
@@ -59,7 +35,51 @@ class PokkenServerConfig:
             self.__config, "pokken", "server", "auto_register", default=True
         )
 
+    @property
+    def enable_matching(self) -> bool:
+        """
+        If global matching should happen
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "server", "enable_matching", default=False
+        )
+    
+    @property
+    def stun_server_host(self) -> str:
+        """
+        Hostname of the EXTERNAL stun server the game should connect to. This is not handled by artemis.
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "server", "stun_server_host", default="stunserver.stunprotocol.org"
+        )
+
+    @property
+    def stun_server_port(self) -> int:
+        """
+        Port of the EXTERNAL stun server the game should connect to. This is not handled by artemis.
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "server", "stun_server_port", default=3478
+        )
+
+class PokkenPortsConfig:
+    def __init__(self, parent_config: "PokkenConfig"):
+        self.__config = parent_config
+    
+    @property
+    def game(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "ports", "game", default=9000
+        )
+
+    @property
+    def admission(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "pokken", "ports", "admission", default=9001
+        )
+    
 
 class PokkenConfig(dict):
     def __init__(self) -> None:
         self.server = PokkenServerConfig(self)
+        self.ports = PokkenPortsConfig(self)

titles/pokken/const.py

@@ -11,14 +11,15 @@ class PokkenConstants:
     VERSION_NAMES = "Pokken Tournament"
 
     class BATTLE_TYPE(Enum):
-        BATTLE_TYPE_TUTORIAL = 1
-        BATTLE_TYPE_AI = 2
-        BATTLE_TYPE_LAN = 3
-        BATTLE_TYPE_WAN = 4
+        TUTORIAL = 1
+        AI = 2
+        LAN = 3
+        WAN = 4
+        TUTORIAL_3 = 7
 
     class BATTLE_RESULT(Enum):
-        BATTLE_RESULT_WIN = 1
-        BATTLE_RESULT_LOSS = 2
+        WIN = 1
+        LOSS = 2
 
     @classmethod
     def game_ver_to_string(cls, ver: int):

titles/pokken/frontend.py

@@ -2,8 +2,9 @@ import yaml
 import jinja2
 from twisted.web.http import Request
 from os import path
+from twisted.web.server import Session
 
-from core.frontend import FE_Base
+from core.frontend import FE_Base, IUserSession
 from core.config import CoreConfig
 from .database import PokkenData
 from .config import PokkenConfig
@@ -27,7 +28,12 @@ class PokkenFrontend(FE_Base):
         template = self.environment.get_template(
             "titles/pokken/frontend/pokken_index.jinja"
         )
+
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+
         return template.render(
             title=f"{self.core_config.server.name} | {self.nav_name}",
             game_list=self.environment.globals["game_list"],
+            sesh=vars(usr_sesh)
         ).encode("utf-16")

titles/pokken/index.py

@@ -1,6 +1,7 @@
 from typing import Tuple
 from twisted.web.http import Request
 from twisted.web import resource
+from twisted.internet import reactor
 import json, ast
 from datetime import datetime
 import yaml
@@ -11,10 +12,11 @@ from os import path
 from google.protobuf.message import DecodeError
 
 from core import CoreConfig, Utils
-from titles.pokken.config import PokkenConfig
-from titles.pokken.base import PokkenBase
-from titles.pokken.const import PokkenConstants
-from titles.pokken.proto import jackal_pb2
+from .config import PokkenConfig
+from .base import PokkenBase
+from .const import PokkenConstants
+from .proto import jackal_pb2
+from .services import PokkenAdmissionFactory
 
 
 class PokkenServlet(resource.Resource):
@@ -69,7 +71,7 @@ class PokkenServlet(resource.Resource):
 
         return (
             True,
-            f"https://{game_cfg.server.hostname}:{game_cfg.server.port}/{game_code}/$v/",
+            f"https://{game_cfg.server.hostname}:{game_cfg.ports.game}/{game_code}/$v/",
             f"{game_cfg.server.hostname}/SDAK/$v/",
         )
 
@@ -90,8 +92,10 @@ class PokkenServlet(resource.Resource):
         return (True, "PKF1")
 
     def setup(self) -> None:
-        # TODO: Setup stun, turn (UDP) and admission (WSS) servers
-        pass
+        if self.game_cfg.server.enable_matching:
+            reactor.listenTCP(
+                self.game_cfg.ports.admission, PokkenAdmissionFactory(self.core_cfg, self.game_cfg)
+            )
 
     def render_POST(
         self, request: Request, version: int = 0, endpoints: str = ""
@@ -108,7 +112,7 @@ class PokkenServlet(resource.Resource):
         try:
             pokken_request.ParseFromString(content)
         except DecodeError as e:
-            self.logger.warn(f"{e} {content}")
+            self.logger.warning(f"{e} {content}")
             return b""
 
         endpoint = jackal_pb2.MessageType.DESCRIPTOR.values_by_number[
@@ -119,7 +123,7 @@ class PokkenServlet(resource.Resource):
 
         handler = getattr(self.base, f"handle_{endpoint}", None)
         if handler is None:
-            self.logger.warn(f"No handler found for message type {endpoint}")
+            self.logger.warning(f"No handler found for message type {endpoint}")
             return self.base.handle_noop(pokken_request)
 
         self.logger.info(f"{endpoint} request from {Utils.get_ip_addr(request)}")
@@ -128,6 +132,9 @@ class PokkenServlet(resource.Resource):
         return ret
 
     def handle_matching(self, request: Request) -> bytes:
+        if not self.game_cfg.server.enable_matching:
+            return b""
+        
         content = request.content.getvalue()
         client_ip = Utils.get_ip_addr(request)
 
@@ -150,7 +157,7 @@ class PokkenServlet(resource.Resource):
             None,
         )
         if handler is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"No handler found for message type {json_content['call']}"
             )
             return json.dumps(self.base.handle_matching_noop()).encode()

titles/pokken/schema/item.py

@@ -31,4 +31,20 @@ class PokkenItemData(BaseData):
     Items obtained as rewards
     """
 
-    pass
+    def add_reward(self, user_id: int, category: int, content: int, item_type: int) -> Optional[int]:
+        sql = insert(item).values(
+            user=user_id,
+            category=category,
+            content=content,
+            type=item_type,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            content=content,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warning(f"Failed to insert reward for user {user_id}: {category}-{content}-{item_type}")
+            return None
+        return result.lastrowid

titles/pokken/schema/profile.py

@@ -1,8 +1,9 @@
-from typing import Optional, Dict, List
+from typing import Optional, Dict, List, Union
 from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_, case
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select, update, delete
+from sqlalchemy.sql.functions import coalesce
 from sqlalchemy.engine import Row
 from sqlalchemy.dialects.mysql import insert
 
@@ -125,7 +126,7 @@ pokemon_data = Table(
     Column("win_vs_lan", Integer),
     Column("battle_num_vs_cpu", Integer),  # 2
     Column("win_cpu", Integer),
-    Column("battle_all_num_tutorial", Integer),
+    Column("battle_all_num_tutorial", Integer), # ???
     Column("battle_num_tutorial", Integer),  # 1?
     Column("bp_point_atk", Integer),
     Column("bp_point_res", Integer),
@@ -137,6 +138,14 @@ pokemon_data = Table(
 
 
 class PokkenProfileData(BaseData):
+    def touch_profile(self, user_id: int) -> Optional[int]:
+        sql = select([profile.c.id]).where(profile.c.user == user_id)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()['id']
+
     def create_profile(self, user_id: int) -> Optional[int]:
         sql = insert(profile).values(user=user_id)
         conflict = sql.on_duplicate_key_update(user=user_id)
@@ -147,11 +156,10 @@ class PokkenProfileData(BaseData):
             return None
         return result.lastrowid
 
-    def set_profile_name(self, user_id: int, new_name: str) -> None:
-        sql = (
-            update(profile)
-            .where(profile.c.user == user_id)
-            .values(trainer_name=new_name)
+    def set_profile_name(self, user_id: int, new_name: str, gender: Union[int, None] = None) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            trainer_name=new_name,
+            avatar_gender=gender if gender is not None else profile.c.avatar_gender
         )
         result = self.execute(sql)
         if result is None:
@@ -159,13 +167,75 @@ class PokkenProfileData(BaseData):
                 f"Failed to update pokken profile name for user {user_id}!"
             )
 
-    def update_profile_tutorial_flags(self, user_id: int, tutorial_flags: Dict) -> None:
-        pass
+    def put_extra(
+        self, 
+        user_id: int, 
+        extra_counter: int,
+        evt_reward_get_flg: int,
+        total_play_days: int,
+        awake_num: int,
+        use_support_ct: int,
+        beat_num: int,
+        aid_skill: int,
+        last_evt: int
+    ) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            extra_counter=extra_counter,
+            event_reward_get_flag=evt_reward_get_flg,
+            total_play_days=total_play_days,
+            awake_num=awake_num,
+            use_support_num=use_support_ct,
+            beat_num=beat_num,
+            aid_skill=aid_skill,
+            last_play_event_id=last_evt
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"Failed to put extra data for user {user_id}")
+
+    def update_profile_tutorial_flags(self, user_id: int, tutorial_flags: List) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            tutorial_progress_flag=tutorial_flags,
+        )
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"Failed to update pokken profile tutorial flags for user {user_id}!"
+            )
+
+    def update_profile_achievement_flags(self, user_id: int, achievement_flags: List) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            achievement_flag=achievement_flags,
+        )
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"Failed to update pokken profile achievement flags for user {user_id}!"
+            )
+
+    def update_profile_event(self, user_id: int, event_state: List, event_flags: List[int], event_param: List[int], last_evt: int = None) -> None:
+        sql = update(profile).where(profile.c.user == user_id).values(
+            event_state=event_state,
+            event_achievement_flag=event_flags,
+            event_achievement_param=event_param,
+            last_play_event_id=last_evt if last_evt is not None else profile.c.last_play_event_id,
+        )
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"Failed to update pokken profile event state for user {user_id}!"
+            )
 
     def add_profile_points(
-        self, user_id: int, rank_pts: int, money: int, score_pts: int
+        self, user_id: int, rank_pts: int, money: int, score_pts: int, grade_max: int
     ) -> None:
-        pass
+        sql = update(profile).where(profile.c.user == user_id).values(
+            trainer_rank_point = profile.c.trainer_rank_point + rank_pts,
+            fight_money = profile.c.fight_money + money,
+            score_point = profile.c.score_point + score_pts,
+            grade_max_num = grade_max
+        )
 
     def get_profile(self, user_id: int) -> Optional[Row]:
         sql = profile.select(profile.c.user == user_id)
@@ -174,32 +244,101 @@ class PokkenProfileData(BaseData):
             return None
         return result.fetchone()
 
-    def put_pokemon_data(
+    def put_pokemon(
         self,
         user_id: int,
         pokemon_id: int,
         illust_no: int,
-        get_exp: int,
         atk: int,
         res: int,
         defe: int,
-        sp: int,
+        sp: int
     ) -> Optional[int]:
-        pass
+        sql = insert(pokemon_data).values(
+            user=user_id,
+            char_id=pokemon_id,
+            illustration_book_no=illust_no,
+            pokemon_exp=0,
+            battle_num_vs_wan=0,
+            win_vs_wan=0,
+            battle_num_vs_lan=0,
+            win_vs_lan=0,
+            battle_num_vs_cpu=0,
+            win_cpu=0,
+            battle_all_num_tutorial=0,
+            battle_num_tutorial=0,
+            bp_point_atk=1+atk,
+            bp_point_res=1+res,
+            bp_point_def=1+defe,
+            bp_point_sp=1+sp,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            illustration_book_no=illust_no,
+            bp_point_atk=pokemon_data.c.bp_point_atk + atk,
+            bp_point_res=pokemon_data.c.bp_point_res + res,
+            bp_point_def=pokemon_data.c.bp_point_def + defe,
+            bp_point_sp=pokemon_data.c.bp_point_sp + sp,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.warning(f"Failed to insert pokemon ID {pokemon_id} for user {user_id}")
+            return None
+        return result.lastrowid
+
+    def add_pokemon_xp(
+        self,
+        user_id: int,
+        pokemon_id: int,
+        xp: int
+    ) -> None:
+        sql = update(pokemon_data).where(and_(pokemon_data.c.user==user_id, pokemon_data.c.char_id==pokemon_id)).values(
+            pokemon_exp=coalesce(pokemon_data.c.pokemon_exp, 0) + xp
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warning(f"Failed to add {xp} XP to pokemon ID {pokemon_id} for user {user_id}")
 
     def get_pokemon_data(self, user_id: int, pokemon_id: int) -> Optional[Row]:
-        pass
+        sql = pokemon_data.select(and_(pokemon_data.c.user == user_id, pokemon_data.c.char_id == pokemon_id))
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
 
     def get_all_pokemon_data(self, user_id: int) -> Optional[List[Row]]:
-        pass
+        sql = pokemon_data.select(pokemon_data.c.user == user_id)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
 
-    def put_results(
-        self, user_id: int, pokemon_id: int, match_type: int, match_result: int
+    def put_pokemon_battle_result(
+        self, user_id: int, pokemon_id: int, match_type: PokkenConstants.BATTLE_TYPE, match_result: PokkenConstants.BATTLE_RESULT
     ) -> None:
         """
         Records the match stats (type and win/loss) for the pokemon and profile
+        coalesce(pokemon_data.c.win_vs_wan, 0)
         """
-        pass
+        sql = update(pokemon_data).where(and_(pokemon_data.c.user==user_id, pokemon_data.c.char_id==pokemon_id)).values(
+            battle_num_tutorial=coalesce(pokemon_data.c.battle_num_tutorial, 0) + 1 if match_type==PokkenConstants.BATTLE_TYPE.TUTORIAL else coalesce(pokemon_data.c.battle_num_tutorial, 0),
+            battle_all_num_tutorial=coalesce(pokemon_data.c.battle_all_num_tutorial, 0) + 1 if match_type==PokkenConstants.BATTLE_TYPE.TUTORIAL else coalesce(pokemon_data.c.battle_all_num_tutorial, 0),
+
+            battle_num_vs_cpu=coalesce(pokemon_data.c.battle_num_vs_cpu, 0) + 1 if match_type==PokkenConstants.BATTLE_TYPE.AI else coalesce(pokemon_data.c.battle_num_vs_cpu, 0),
+            win_cpu=coalesce(pokemon_data.c.win_cpu, 0) + 1 if match_type==PokkenConstants.BATTLE_TYPE.AI and match_result==PokkenConstants.BATTLE_RESULT.WIN else coalesce(pokemon_data.c.win_cpu, 0),
+
+            battle_num_vs_lan=coalesce(pokemon_data.c.battle_num_vs_lan, 0) + 1 if match_type==PokkenConstants.BATTLE_TYPE.LAN else coalesce(pokemon_data.c.battle_num_vs_lan, 0),
+            win_vs_lan=coalesce(pokemon_data.c.win_vs_lan, 0) + 1 if match_type==PokkenConstants.BATTLE_TYPE.LAN and match_result==PokkenConstants.BATTLE_RESULT.WIN else coalesce(pokemon_data.c.win_vs_lan, 0),
+
+            battle_num_vs_wan=coalesce(pokemon_data.c.battle_num_vs_wan, 0) + 1 if match_type==PokkenConstants.BATTLE_TYPE.WAN else coalesce(pokemon_data.c.battle_num_vs_wan, 0),
+            win_vs_wan=coalesce(pokemon_data.c.win_vs_wan, 0) + 1 if match_type==PokkenConstants.BATTLE_TYPE.WAN and match_result==PokkenConstants.BATTLE_RESULT.WIN else coalesce(pokemon_data.c.win_vs_wan, 0),
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warning(f"Failed to record match stats for user {user_id}'s pokemon {pokemon_id} (type {match_type.name} | result {match_result.name})")
 
     def put_stats(
         self,
@@ -214,4 +353,29 @@ class PokkenProfileData(BaseData):
         """
         Records profile stats
         """
-        pass
+        sql = update(profile).where(profile.c.user==user_id).values(
+            ex_ko_num=coalesce(profile.c.ex_ko_num, 0) + exkos,
+            wko_num=coalesce(profile.c.wko_num, 0) + wkos,
+            timeup_win_num=coalesce(profile.c.timeup_win_num, 0) + timeout_wins,
+            cool_ko_num=coalesce(profile.c.cool_ko_num, 0) + cool_kos,
+            perfect_ko_num=coalesce(profile.c.perfect_ko_num, 0) + perfects,
+            continue_num=continues,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warning(f"Failed to update stats for user {user_id}")
+
+    def update_support_team(self, user_id: int, support_id: int, support1: int = None, support2: int = None) -> None:
+        sql = update(profile).where(profile.c.user==user_id).values(
+            support_set_1_1=support1 if support_id == 1 else profile.c.support_set_1_1,
+            support_set_1_2=support2 if support_id == 1 else profile.c.support_set_1_2,
+            support_set_2_1=support1 if support_id == 2 else profile.c.support_set_2_1,
+            support_set_2_2=support2 if support_id == 2 else profile.c.support_set_2_2,
+            support_set_3_1=support1 if support_id == 3 else profile.c.support_set_3_1,
+            support_set_3_2=support2 if support_id == 3 else profile.c.support_set_3_2,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.warning(f"Failed to update support team {support_id} for user {user_id}")

titles/pokken/services.py

@@ -0,0 +1,66 @@
+from twisted.internet.interfaces import IAddress
+from twisted.internet.protocol import Protocol
+from autobahn.twisted.websocket import WebSocketServerProtocol, WebSocketServerFactory
+from autobahn.websocket.types import ConnectionRequest
+from typing import Dict
+import logging
+import json
+
+from core.config import CoreConfig
+from .config import PokkenConfig
+from .base import PokkenBase
+
+class PokkenAdmissionProtocol(WebSocketServerProtocol):
+    def __init__(self, cfg: CoreConfig, game_cfg: PokkenConfig):
+        super().__init__()
+        self.core_config = cfg
+        self.game_config = game_cfg
+        self.logger = logging.getLogger("pokken")
+
+        self.base = PokkenBase(cfg, game_cfg)
+    
+    def onConnect(self, request: ConnectionRequest) -> None:
+        self.logger.debug(f"Admission: Connection from {request.peer}")
+    
+    def onClose(self, wasClean: bool, code: int, reason: str) -> None:
+        self.logger.debug(f"Admission: Connection with {self.transport.getPeer().host} closed {'cleanly ' if wasClean else ''}with code {code} - {reason}")
+
+    def onMessage(self, payload, isBinary: bool) -> None:
+        msg: Dict = json.loads(payload)
+        self.logger.debug(f"Admission: Message from {self.transport.getPeer().host}:{self.transport.getPeer().port} - {msg}")
+        
+        api = msg.get("api", "noop")
+        handler = getattr(self.base, f"handle_admission_{api.lower()}")
+        resp = handler(msg, self.transport.getPeer().host)
+        
+        if resp is None:
+            resp = {}
+
+        if "type" not in resp:
+            resp['type'] = "res"
+        if "data" not in resp:
+            resp['data'] = {}
+        if "api" not in resp:
+            resp['api'] = api
+        if "result" not in resp:
+            resp['result'] = 'true'
+        
+        self.logger.debug(f"Websocket response: {resp}")
+        self.sendMessage(json.dumps(resp).encode(), isBinary)
+
+class PokkenAdmissionFactory(WebSocketServerFactory):
+    protocol = PokkenAdmissionProtocol
+
+    def __init__(
+        self,
+        cfg: CoreConfig,
+        game_cfg: PokkenConfig
+    ) -> None:
+        self.core_config = cfg
+        self.game_config = game_cfg
+        super().__init__(f"ws://{self.game_config.server.hostname}:{self.game_config.ports.admission}")
+    
+    def buildProtocol(self, addr: IAddress) -> Protocol:
+        p = self.protocol(self.core_config, self.game_config)
+        p.factory = self
+        return p

titles/sao/__init__.py

@@ -0,0 +1,10 @@
+from .index import SaoServlet
+from .const import SaoConstants
+from .database import SaoData
+from .read import SaoReader
+
+index = SaoServlet
+database = SaoData
+reader = SaoReader
+game_codes = [SaoConstants.GAME_CODE]
+current_schema_version = 1

titles/sao/config.py

@@ -0,0 +1,47 @@
+from core.config import CoreConfig
+
+
+class SaoServerConfig:
+    def __init__(self, parent_config: "SaoConfig"):
+        self.__config = parent_config
+
+    @property
+    def hostname(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "sao", "server", "hostname", default="localhost"
+        )
+
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "sao", "server", "enable", default=True
+        )
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "sao", "server", "loglevel", default="info"
+            )
+        )
+
+    @property
+    def port(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "sao", "server", "port", default=9000
+        )
+
+    @property
+    def auto_register(self) -> bool:
+        """
+        Automatically register users in `aime_user` on first carding in with sao
+        if they don't exist already. Set to false to display an error instead.
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "sao", "server", "auto_register", default=True
+        )
+
+
+class SaoConfig(dict):
+    def __init__(self) -> None:
+        self.server = SaoServerConfig(self)

titles/sao/const.py

@@ -0,0 +1,15 @@
+from enum import Enum
+
+
+class SaoConstants:
+    GAME_CODE = "SDEW"
+
+    CONFIG_NAME = "sao.yaml"
+
+    VER_SAO = 0
+
+    VERSION_NAMES = ("Sword Art Online Arcade")
+
+    @classmethod
+    def game_ver_to_string(cls, ver: int):
+        return cls.VERSION_NAMES[ver]

titles/sao/data/EquipmentLevel.csv

@@ -0,0 +1,121 @@
+EquipmentLevelId,RequireExp
+1,200,
+2,400,
+3,600,
+4,800,
+5,1000,
+6,1200,
+7,1400,
+8,1600,
+9,1800,
+10,2000,
+11,2200,
+12,2400,
+13,2600,
+14,2800,
+15,3000,
+16,3200,
+17,3400,
+18,3600,
+19,3800,
+20,4000,
+21,4200,
+22,4400,
+23,4600,
+24,4800,
+25,5000,
+26,5200,
+27,5400,
+28,5600,
+29,5800,
+30,6000,
+31,6200,
+32,6400,
+33,6600,
+34,6800,
+35,7000,
+36,7200,
+37,7400,
+38,7600,
+39,7800,
+40,8000,
+41,8200,
+42,8400,
+43,8600,
+44,8800,
+45,9000,
+46,9200,
+47,9400,
+48,9600,
+49,9800,
+50,10000,
+51,10200,
+52,10400,
+53,10600,
+54,10800,
+55,11000,
+56,11200,
+57,11400,
+58,11600,
+59,11800,
+60,12000,
+61,12200,
+62,12400,
+63,12600,
+64,12800,
+65,13000,
+66,13200,
+67,13400,
+68,13600,
+69,13800,
+70,14000,
+71,14200,
+72,14400,
+73,14600,
+74,14800,
+75,15000,
+76,15200,
+77,15400,
+78,15600,
+79,15800,
+80,16000,
+81,16200,
+82,16400,
+83,16600,
+84,16800,
+85,17000,
+86,17200,
+87,17400,
+88,17600,
+89,17800,
+90,18000,
+91,18200,
+92,18400,
+93,18600,
+94,18800,
+95,19000,
+96,19200,
+97,19400,
+98,19600,
+99,19800,
+100,100000,
+101,150000,
+102,200000,
+103,250000,
+104,300000,
+105,350000,
+106,400000,
+107,450000,
+108,500000,
+109,550000,
+110,600000,
+111,650000,
+112,700000,
+113,750000,
+114,800000,
+115,850000,
+116,900000,
+117,950000,
+118,1000000,
+119,1000000,
+120,1000000,

titles/sao/data/HeroLogLevel.csv

@@ -0,0 +1,121 @@
+HeroLogLevelId,RequireExp
+1,0,
+2,1000,
+3,1200,
+4,1400,
+5,1600,
+6,1900,
+7,2200,
+8,2500,
+9,2800,
+10,3100,
+11,3500,
+12,3900,
+13,4300,
+14,4700,
+15,5100,
+16,5550,
+17,6000,
+18,6450,
+19,6900,
+20,7350,
+21,7850,
+22,8350,
+23,8850,
+24,9350,
+25,9850,
+26,10450,
+27,11050,
+28,11650,
+29,12250,
+30,12850,
+31,13550,
+32,14250,
+33,14950,
+34,15650,
+35,16350,
+36,17150,
+37,17950,
+38,18750,
+39,19550,
+40,20350,
+41,21250,
+42,22150,
+43,23050,
+44,23950,
+45,24850,
+46,25850,
+47,26850,
+48,27850,
+49,28850,
+50,29850,
+51,30950,
+52,32050,
+53,33150,
+54,34250,
+55,35350,
+56,36550,
+57,37750,
+58,38950,
+59,40150,
+60,41350,
+61,42650,
+62,43950,
+63,45250,
+64,46550,
+65,47850,
+66,49250,
+67,50650,
+68,52050,
+69,53450,
+70,54850,
+71,56350,
+72,57850,
+73,59350,
+74,60850,
+75,62350,
+76,63950,
+77,65550,
+78,67150,
+79,68750,
+80,70350,
+81,72050,
+82,73750,
+83,75450,
+84,77150,
+85,78850,
+86,80650,
+87,82450,
+88,84250,
+89,86050,
+90,87850,
+91,89750,
+92,91650,
+93,93550,
+94,95450,
+95,97350,
+96,99350,
+97,101350,
+98,103350,
+99,105350,
+100,107350,
+101,200000,
+102,300000,
+103,450000,
+104,600000,
+105,800000,
+106,1000000,
+107,1250000,
+108,1500000,
+109,1800000,
+110,2100000,
+111,2100000,
+112,2100000,
+113,2100000,
+114,2100000,
+115,2100000,
+116,2100000,
+117,2100000,
+118,2100000,
+119,2100000,
+120,2100000,

titles/sao/data/PlayerRank.csv

@@ -0,0 +1,301 @@
+PlayerRankId,TotalExp,
+1,0,
+2,100,
+3,300,
+4,500,
+5,800,
+6,1100,
+7,1400,
+8,1800,
+9,2200,
+10,2600,,
+11,3000,,
+12,3500,,
+13,4000,,
+14,4500,,
+15,5000,,
+16,5500,,
+17,6100,,
+18,6700,,
+19,7300,,
+20,7900,,
+21,8500,,
+22,9100,,
+23,9800,,
+24,10500,
+25,11200,
+26,11900,
+27,12600,
+28,13300,
+29,14000,
+30,14800,
+31,15600,
+32,16400,
+33,17200,
+34,18000,
+35,18800,
+36,19600,
+37,20400,
+38,21300,
+39,22200,
+40,23100,
+41,24000,
+42,24900,
+43,25800,
+44,26700,
+45,27600,
+46,28500,
+47,29500,
+48,30600,
+49,31800,
+50,33100,
+51,34500,
+52,36000,
+53,37600,
+54,39300,
+55,41100,
+56,43000,
+57,45000,
+58,47100,
+59,49300,
+60,51600,
+61,54000,
+62,56500,
+63,59100,
+64,61800,
+65,64600,
+66,67500,
+67,70500,
+68,73600,
+69,76800,
+70,80100,
+71,83500,
+72,87000,
+73,90600,
+74,94300,
+75,98100,
+76,102000,
+77,106000,
+78,110100,
+79,114300,
+80,118600,
+81,123000,
+82,127500,
+83,132100,
+84,136800,
+85,141600,
+86,146500,
+87,151500,
+88,156600,
+89,161800,
+90,167100,
+91,172500,
+92,178000,
+93,183600,
+94,189300,
+95,195100,
+96,201000,
+97,207000,
+98,213100,
+99,219300,
+100,225600,
+101,232000,
+102,238500,
+103,245100,
+104,251800,
+105,258600,
+106,265500,
+107,272500,
+108,279600,
+109,286800,
+110,294100,
+111,301500,
+112,309000,
+113,316600,
+114,324300,
+115,332100,
+116,340000,
+117,348000,
+118,356100,
+119,364300,
+120,372600,
+121,381000,
+122,389500,
+123,398100,
+124,406800,
+125,415600,
+126,424500,
+127,433500,
+128,442600,
+129,451800,
+130,461100,
+131,470500,
+132,480000,
+133,489600,
+134,499300,
+135,509100,
+136,519000,
+137,529000,
+138,539100,
+139,549300,
+140,559600,
+141,570000,
+142,580500,
+143,591100,
+144,601800,
+145,612600,
+146,623500,
+147,634500,
+148,645600,
+149,656800,
+150,668100,
+151,679500,
+152,691000,
+153,702600,
+154,714300,
+155,726100,
+156,738000,
+157,750000,
+158,762100,
+159,774300,
+160,786600,
+161,799000,
+162,811500,
+163,824100,
+164,836800,
+165,849600,
+166,862500,
+167,875500,
+168,888600,
+169,901800,
+170,915100,
+171,928500,
+172,942000,
+173,955600,
+174,969300,
+175,983100,
+176,997000,
+177,1011000,
+178,1025100,
+179,1039300,
+180,1053600,
+181,1068000,
+182,1082500,
+183,1097100,
+184,1111800,
+185,1126600,
+186,1141500,
+187,1156500,
+188,1171600,
+189,1186800,
+190,1202100,
+191,1217500,
+192,1233000,
+193,1248600,
+194,1264300,
+195,1280100,
+196,1296000,
+197,1312000,
+198,1328100,
+199,1344300,
+200,1360600,
+201,1377000,
+202,1393500,
+203,1410100,
+204,1426800,
+205,1443600,
+206,1460500,
+207,1477500,
+208,1494600,
+209,1511800,
+210,1529100,
+211,1546500,
+212,1564000,
+213,1581600,
+214,1599300,
+215,1617100,
+216,1635000,
+217,1653000,
+218,1671100,
+219,1689300,
+220,1707600,
+221,1726000,
+222,1744500,
+223,1763100,
+224,1781800,
+225,1800600,
+226,1819500,
+227,1838500,
+228,1857600,
+229,1876800,
+230,1896100,
+231,1915500,
+232,1935000,
+233,1954600,
+234,1974300,
+235,1994100,
+236,2014000,
+237,2034000,
+238,2054100,
+239,2074300,
+240,2094600,
+241,2115000,
+242,2135500,
+243,2156100,
+244,2176800,
+245,2197600,
+246,2218500,
+247,2239500,
+248,2260600,
+249,2281800,
+250,2303100,
+251,2324500,
+252,2346000,
+253,2367600,
+254,2389300,
+255,2411100,
+256,2433000,
+257,2455000,
+258,2477100,
+259,2499300,
+260,2521600,
+261,2544000,
+262,2566500,
+263,2589100,
+264,2611800,
+265,2634600,
+266,2657500,
+267,2680500,
+268,2703600,
+269,2726800,
+270,2750100,
+271,2773500,
+272,2797000,
+273,2820600,
+274,2844300,
+275,2868100,
+276,2892000,
+277,2916000,
+278,2940100,
+279,2964300,
+280,2988600,
+281,3013000,
+282,3037500,
+283,3062100,
+284,3086800,
+285,3111600,
+286,3136500,
+287,3161500,
+288,3186600,
+289,3211800,
+290,3237100,
+291,3262500,
+292,3288000,
+293,3313600,
+294,3339300,
+295,3365100,
+296,3391000,
+297,3417000,
+298,3443100,
+299,3469300,
+300,3495600,

titles/sao/database.py

@@ -0,0 +1,13 @@
+from core.data import Data
+from core.config import CoreConfig
+
+from .schema import *
+
+
+class SaoData(Data):
+    def __init__(self, cfg: CoreConfig) -> None:
+        super().__init__(cfg)
+
+        self.item = SaoItemData(cfg, self.session)
+        self.profile = SaoProfileData(cfg, self.session)
+        self.static = SaoStaticData(cfg, self.session)

titles/sao/handlers/__init__.py

@@ -0,0 +1 @@
+from titles.sao.handlers.base import *

titles/sao/index.py

@@ -0,0 +1,117 @@
+from typing import Tuple
+from twisted.web.http import Request
+from twisted.web import resource
+import json, ast
+from datetime import datetime
+import yaml
+import logging, coloredlogs
+from logging.handlers import TimedRotatingFileHandler
+import inflection
+from os import path
+
+from core import CoreConfig, Utils
+from titles.sao.config import SaoConfig
+from titles.sao.const import SaoConstants
+from titles.sao.base import SaoBase
+from titles.sao.handlers.base import *
+
+
+class SaoServlet(resource.Resource):
+    def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
+        self.isLeaf = True
+        self.core_cfg = core_cfg
+        self.config_dir = cfg_dir
+        self.game_cfg = SaoConfig()
+        if path.exists(f"{cfg_dir}/sao.yaml"):
+            self.game_cfg.update(yaml.safe_load(open(f"{cfg_dir}/sao.yaml")))
+
+        self.logger = logging.getLogger("sao")
+        if not hasattr(self.logger, "inited"):
+            log_fmt_str = "[%(asctime)s] SAO | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.core_cfg.server.log_dir, "sao"),
+                encoding="utf8",
+                when="d",
+                backupCount=10,
+            )
+
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(self.game_cfg.server.loglevel)
+            coloredlogs.install(
+                level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.inited = True
+
+        self.base = SaoBase(core_cfg, self.game_cfg)
+
+    @classmethod
+    def get_allnet_info(
+        cls, game_code: str, core_cfg: CoreConfig, cfg_dir: str
+    ) -> Tuple[bool, str, str]:
+        game_cfg = SaoConfig()
+
+        if path.exists(f"{cfg_dir}/{SaoConstants.CONFIG_NAME}"):
+            game_cfg.update(
+                yaml.safe_load(open(f"{cfg_dir}/{SaoConstants.CONFIG_NAME}"))
+            )
+
+        if not game_cfg.server.enable:
+            return (False, "", "")
+
+        return (
+            True,
+            f"http://{game_cfg.server.hostname}:{game_cfg.server.port}/{game_code}/$v/",
+            f"{game_cfg.server.hostname}/SDEW/$v/",
+        )
+
+    @classmethod
+    def get_mucha_info(
+        cls, core_cfg: CoreConfig, cfg_dir: str
+    ) -> Tuple[bool, str, str]:
+        game_cfg = SaoConfig()
+
+        if path.exists(f"{cfg_dir}/{SaoConstants.CONFIG_NAME}"):
+            game_cfg.update(
+                yaml.safe_load(open(f"{cfg_dir}/{SaoConstants.CONFIG_NAME}"))
+            )
+
+        if not game_cfg.server.enable:
+            return (False, "")
+
+        return (True, "SAO1")
+
+    def setup(self) -> None:
+        pass
+
+    def render_POST(
+        self, request: Request, version: int = 0, endpoints: str = ""
+    ) -> bytes:
+        req_url = request.uri.decode()
+        if req_url == "/matching":
+            self.logger.info("Matching request")
+        
+        request.responseHeaders.addRawHeader(b"content-type", b"text/html; charset=utf-8")
+
+        sao_request = request.content.getvalue().hex()
+
+        handler = getattr(self.base, f"handle_{sao_request[:4]}", None)
+        if handler is None:
+            self.logger.info(f"Generic Handler for {req_url} - {sao_request[:4]}")
+            self.logger.debug(f"Request: {request.content.getvalue().hex()}")
+            resp = SaoNoopResponse(int.from_bytes(bytes.fromhex(sao_request[:4]), "big")+1)
+            self.logger.debug(f"Response: {resp.make().hex()}")
+            return resp.make()
+
+        self.logger.info(f"Handler {req_url} - {sao_request[:4]} request")
+        self.logger.debug(f"Request: {request.content.getvalue().hex()}")
+        resp = handler(sao_request)
+        self.logger.debug(f"Response: {resp.hex()}")
+        return resp

titles/sao/read.py

@@ -0,0 +1,254 @@
+from typing import Optional, Dict, List
+from os import walk, path
+import urllib
+import csv
+
+from read import BaseReader
+from core.config import CoreConfig
+from titles.sao.database import SaoData
+from titles.sao.const import SaoConstants
+
+
+class SaoReader(BaseReader):
+    def __init__(
+        self,
+        config: CoreConfig,
+        version: int,
+        bin_arg: Optional[str],
+        opt_arg: Optional[str],
+        extra: Optional[str],
+    ) -> None:
+        super().__init__(config, version, bin_arg, opt_arg, extra)
+        self.data = SaoData(config)
+
+        try:
+            self.logger.info(
+                f"Start importer for {SaoConstants.game_ver_to_string(version)}"
+            )
+        except IndexError:
+            self.logger.error(f"Invalid project SAO version {version}")
+            exit(1)
+
+    def read(self) -> None:
+        pull_bin_ram = True
+
+        if not path.exists(f"{self.bin_dir}"):
+            self.logger.warning(f"Couldn't find csv file in {self.bin_dir}, skipping")
+            pull_bin_ram = False
+
+        if pull_bin_ram:
+            self.read_csv(f"{self.bin_dir}")
+
+    def read_csv(self, bin_dir: str) -> None:
+        self.logger.info(f"Read csv from {bin_dir}")
+
+        self.logger.info("Now reading QuestScene.csv")
+        try:
+            fullPath = bin_dir + "/QuestScene.csv"
+            with open(fullPath, encoding="UTF-8") as fp:
+                reader = csv.DictReader(fp)
+                for row in reader:
+                    questSceneId = row["QuestSceneId"]
+                    sortNo = row["SortNo"]
+                    name = row["Name"]
+                    enabled = True
+
+                    self.logger.info(f"Added quest {questSceneId} | Name: {name}")
+                    
+                    try:
+                        self.data.static.put_quest(
+                            questSceneId,
+                            0,
+                            sortNo,
+                            name,
+                            enabled
+                        )
+                    except Exception as err:
+                        print(err)
+        except Exception:
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")
+        
+        self.logger.info("Now reading HeroLog.csv")
+        try:
+            fullPath = bin_dir + "/HeroLog.csv"
+            with open(fullPath, encoding="UTF-8") as fp:
+                reader = csv.DictReader(fp)
+                for row in reader:
+                    heroLogId = row["HeroLogId"]
+                    name = row["Name"]
+                    nickname = row["Nickname"]
+                    rarity = row["Rarity"]
+                    skillTableSubId = row["SkillTableSubId"]
+                    awakeningExp = row["AwakeningExp"]
+                    flavorText = row["FlavorText"]
+                    enabled = True
+
+                    self.logger.info(f"Added hero {heroLogId} | Name: {name}")
+                    
+                    try:
+                        self.data.static.put_hero(
+                            0,
+                            heroLogId,
+                            name,
+                            nickname,
+                            rarity,
+                            skillTableSubId,
+                            awakeningExp,
+                            flavorText,
+                            enabled
+                        )
+                    except Exception as err:
+                        print(err)
+        except Exception:
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")
+        
+        self.logger.info("Now reading Equipment.csv")
+        try:
+            fullPath = bin_dir + "/Equipment.csv"
+            with open(fullPath, encoding="UTF-8") as fp:
+                reader = csv.DictReader(fp)
+                for row in reader:
+                    equipmentId = row["EquipmentId"]
+                    equipmentType = row["EquipmentType"]
+                    weaponTypeId = row["WeaponTypeId"]
+                    name = row["Name"]
+                    rarity = row["Rarity"]
+                    flavorText = row["FlavorText"]
+                    enabled = True
+
+                    self.logger.info(f"Added equipment {equipmentId} | Name: {name}")
+                    
+                    try:
+                        self.data.static.put_equipment(
+                            0,
+                            equipmentId,
+                            name,
+                            equipmentType,
+                            weaponTypeId,
+                            rarity,
+                            flavorText,
+                            enabled
+                        )
+                    except Exception as err:
+                        print(err)
+        except Exception:
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")
+
+        self.logger.info("Now reading Item.csv")
+        try:
+            fullPath = bin_dir + "/Item.csv"
+            with open(fullPath, encoding="UTF-8") as fp:
+                reader = csv.DictReader(fp)
+                for row in reader:
+                    itemId = row["ItemId"]
+                    itemTypeId = row["ItemTypeId"]
+                    name = row["Name"]
+                    rarity = row["Rarity"]
+                    flavorText = row["FlavorText"]
+                    enabled = True
+
+                    self.logger.info(f"Added item {itemId} | Name: {name}")
+                    
+                    try:
+                        self.data.static.put_item(
+                            0,
+                            itemId,
+                            name,
+                            itemTypeId,
+                            rarity,
+                            flavorText,
+                            enabled
+                        )
+                    except Exception as err:
+                        print(err)
+        except Exception:
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")
+        
+        self.logger.info("Now reading SupportLog.csv")
+        try:
+            fullPath = bin_dir + "/SupportLog.csv"
+            with open(fullPath, encoding="UTF-8") as fp:
+                reader = csv.DictReader(fp)
+                for row in reader:
+                    supportLogId = row["SupportLogId"]
+                    charaId = row["CharaId"]
+                    name = row["Name"]
+                    rarity = row["Rarity"]
+                    salePrice = row["SalePrice"]
+                    skillName = row["SkillName"]
+                    enabled = True
+
+                    self.logger.info(f"Added support log {supportLogId} | Name: {name}")
+                    
+                    try:
+                        self.data.static.put_support_log(
+                            0,
+                            supportLogId,
+                            charaId,
+                            name,
+                            rarity,
+                            salePrice,
+                            skillName,
+                            enabled
+                        )
+                    except Exception as err:
+                        print(err)
+        except Exception:
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")
+            
+        self.logger.info("Now reading Title.csv")
+        try:
+            fullPath = bin_dir + "/Title.csv"
+            with open(fullPath, encoding="UTF-8") as fp:
+                reader = csv.DictReader(fp)
+                for row in reader:
+                    titleId = row["TitleId"]
+                    displayName = row["DisplayName"]
+                    requirement = row["Requirement"]
+                    rank = row["Rank"]
+                    imageFilePath = row["ImageFilePath"]
+                    enabled = True
+
+                    self.logger.info(f"Added title {titleId} | Name: {displayName}")
+                    
+                    if len(titleId) > 5:
+                        try:
+                            self.data.static.put_title(
+                                0,
+                                titleId,
+                                displayName,
+                                requirement,
+                                rank,
+                                imageFilePath,
+                                enabled
+                            )
+                        except Exception as err:
+                            print(err)
+                    elif len(titleId) < 6: # current server code cannot have multiple lengths for the id
+                        continue
+        except Exception:
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")
+
+        self.logger.info("Now reading RareDropTable.csv")
+        try:
+            fullPath = bin_dir + "/RareDropTable.csv"
+            with open(fullPath, encoding="UTF-8") as fp:
+                reader = csv.DictReader(fp)
+                for row in reader:
+                    questRareDropId = row["QuestRareDropId"]
+                    commonRewardId = row["CommonRewardId"]
+                    enabled = True
+
+                    self.logger.info(f"Added rare drop {questRareDropId} | Reward: {commonRewardId}")
+                    
+                    try:
+                        self.data.static.put_rare_drop(
+                            0,
+                            questRareDropId,
+                            commonRewardId,
+                            enabled
+                        )
+                    except Exception as err:
+                        print(err)
+        except Exception:
+            self.logger.warning(f"Couldn't read csv file in {self.bin_dir}, skipping")

titles/sao/schema/__init__.py

@@ -0,0 +1,3 @@
+from .profile import SaoProfileData
+from .static import SaoStaticData
+from .item import SaoItemData

titles/sao/schema/item.py

@@ -0,0 +1,506 @@
+from typing import Optional, Dict, List
+from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_, case
+from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
+from sqlalchemy.schema import ForeignKey
+from sqlalchemy.sql import func, select, update, delete
+from sqlalchemy.engine import Row
+from sqlalchemy.dialects.mysql import insert
+
+from core.data.schema import BaseData, metadata
+
+equipment_data = Table(
+    "sao_equipment_data",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("equipment_id", Integer, nullable=False),
+    Column("enhancement_value", Integer, nullable=False),
+    Column("enhancement_exp", Integer, nullable=False),
+    Column("awakening_exp", Integer, nullable=False),
+    Column("awakening_stage", Integer, nullable=False),
+    Column("possible_awakening_flag", Integer, nullable=False),
+    Column("get_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    UniqueConstraint("user", "equipment_id", name="sao_equipment_data_uk"),
+    mysql_charset="utf8mb4",
+)
+
+item_data = Table(
+    "sao_item_data",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("item_id", Integer, nullable=False),
+    Column("get_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    UniqueConstraint("user", "item_id", name="sao_item_data_uk"),
+    mysql_charset="utf8mb4",
+)
+
+hero_log_data = Table(
+    "sao_hero_log_data",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("user_hero_log_id", Integer, nullable=False),
+    Column("log_level", Integer, nullable=False),
+    Column("log_exp", Integer, nullable=False),
+    Column("main_weapon", Integer, nullable=False),
+    Column("sub_equipment", Integer, nullable=False),
+    Column("skill_slot1_skill_id", Integer, nullable=False),
+    Column("skill_slot2_skill_id", Integer, nullable=False),
+    Column("skill_slot3_skill_id", Integer, nullable=False),
+    Column("skill_slot4_skill_id", Integer, nullable=False),
+    Column("skill_slot5_skill_id", Integer, nullable=False),
+    Column("get_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    UniqueConstraint("user", "user_hero_log_id", name="sao_hero_log_data_uk"),
+    mysql_charset="utf8mb4",
+)
+
+hero_party = Table(
+    "sao_hero_party",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("user_party_team_id", Integer, nullable=False),
+    Column("user_hero_log_id_1", Integer, nullable=False),
+    Column("user_hero_log_id_2", Integer, nullable=False),
+    Column("user_hero_log_id_3", Integer, nullable=False),
+    UniqueConstraint("user", "user_party_team_id", name="sao_hero_party_uk"),
+    mysql_charset="utf8mb4",
+)
+
+quest = Table(
+    "sao_player_quest",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("episode_id", Integer, nullable=False),
+    Column("quest_clear_flag", Boolean, nullable=False),
+    Column("clear_time", Integer, nullable=False),
+    Column("combo_num", Integer, nullable=False),
+    Column("total_damage", Integer, nullable=False),
+    Column("concurrent_destroying_num", Integer, nullable=False),
+    Column("play_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    UniqueConstraint("user", "episode_id", name="sao_player_quest_uk"),
+    mysql_charset="utf8mb4",
+)
+
+sessions = Table(
+    "sao_play_sessions",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("user_party_team_id", Integer, nullable=False),
+    Column("episode_id", Integer, nullable=False),
+    Column("play_mode", Integer, nullable=False),
+    Column("quest_drop_boost_apply_flag", Integer, nullable=False),
+    Column("play_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    UniqueConstraint("user", "user_party_team_id", "play_date", name="sao_play_sessions_uk"),
+    mysql_charset="utf8mb4",
+)
+
+end_sessions = Table(
+    "sao_end_sessions",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("quest_id", Integer, nullable=False),
+    Column("play_result_flag", Boolean, nullable=False),
+    Column("reward_data", JSON, nullable=True),
+    Column("play_date", TIMESTAMP, nullable=False, server_default=func.now()),
+    mysql_charset="utf8mb4",
+)
+
+class SaoItemData(BaseData):
+    def create_session(self, user_id: int, user_party_team_id: int, episode_id: int, play_mode: int, quest_drop_boost_apply_flag: int) -> Optional[int]:
+        sql = insert(sessions).values(
+            user=user_id,
+            user_party_team_id=user_party_team_id,
+            episode_id=episode_id,
+            play_mode=play_mode,
+            quest_drop_boost_apply_flag=quest_drop_boost_apply_flag
+            )
+
+        conflict = sql.on_duplicate_key_update(user=user_id)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(f"Failed to create SAO session for user {user_id}!")
+            return None
+        return result.lastrowid
+
+    def create_end_session(self, user_id: int, quest_id: int, play_result_flag: bool, reward_data: JSON) -> Optional[int]:
+        sql = insert(end_sessions).values(
+            user=user_id,
+            quest_id=quest_id,
+            play_result_flag=play_result_flag,
+            reward_data=reward_data,
+            )
+
+        conflict = sql.on_duplicate_key_update(user=user_id)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(f"Failed to create SAO end session for user {user_id}!")
+            return None
+        return result.lastrowid
+
+    def put_item(self, user_id: int, item_id: int) -> Optional[int]:
+        sql = insert(item_data).values(
+            user=user_id,
+            item_id=item_id,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            item_id=item_id,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert item! user: {user_id}, item_id: {item_id}"
+            )
+            return None
+
+        return result.lastrowid
+    
+    def put_equipment_data(self, user_id: int, equipment_id: int, enhancement_value: int, enhancement_exp: int, awakening_exp: int, awakening_stage: int, possible_awakening_flag: int) -> Optional[int]:
+        sql = insert(equipment_data).values(
+            user=user_id,
+            equipment_id=equipment_id,
+            enhancement_value=enhancement_value,
+            enhancement_exp=enhancement_exp,
+            awakening_exp=awakening_exp,
+            awakening_stage=awakening_stage,
+            possible_awakening_flag=possible_awakening_flag,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            enhancement_value=enhancement_value,
+            enhancement_exp=enhancement_exp,
+            awakening_exp=awakening_exp,
+            awakening_stage=awakening_stage,
+            possible_awakening_flag=possible_awakening_flag,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert equipment! user: {user_id}, equipment_id: {equipment_id}"
+            )
+            return None
+
+        return result.lastrowid
+
+    def put_hero_log(self, user_id: int, user_hero_log_id: int, log_level: int, log_exp: int, main_weapon: int, sub_equipment: int, skill_slot1_skill_id: int, skill_slot2_skill_id: int, skill_slot3_skill_id: int, skill_slot4_skill_id: int, skill_slot5_skill_id: int) -> Optional[int]:
+        sql = insert(hero_log_data).values(
+            user=user_id,
+            user_hero_log_id=user_hero_log_id,
+            log_level=log_level,
+            log_exp=log_exp,
+            main_weapon=main_weapon,
+            sub_equipment=sub_equipment,
+            skill_slot1_skill_id=skill_slot1_skill_id,
+            skill_slot2_skill_id=skill_slot2_skill_id,
+            skill_slot3_skill_id=skill_slot3_skill_id,
+            skill_slot4_skill_id=skill_slot4_skill_id,
+            skill_slot5_skill_id=skill_slot5_skill_id,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            log_level=log_level,
+            log_exp=log_exp,
+            main_weapon=main_weapon,
+            sub_equipment=sub_equipment,
+            skill_slot1_skill_id=skill_slot1_skill_id,
+            skill_slot2_skill_id=skill_slot2_skill_id,
+            skill_slot3_skill_id=skill_slot3_skill_id,
+            skill_slot4_skill_id=skill_slot4_skill_id,
+            skill_slot5_skill_id=skill_slot5_skill_id,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert hero! user: {user_id}, user_hero_log_id: {user_hero_log_id}"
+            )
+            return None
+
+        return result.lastrowid
+
+    def put_hero_party(self, user_id: int, user_party_team_id: int, user_hero_log_id_1: int, user_hero_log_id_2: int, user_hero_log_id_3: int) -> Optional[int]:
+        sql = insert(hero_party).values(
+            user=user_id,
+            user_party_team_id=user_party_team_id,
+            user_hero_log_id_1=user_hero_log_id_1,
+            user_hero_log_id_2=user_hero_log_id_2,
+            user_hero_log_id_3=user_hero_log_id_3,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            user_hero_log_id_1=user_hero_log_id_1,
+            user_hero_log_id_2=user_hero_log_id_2,
+            user_hero_log_id_3=user_hero_log_id_3,
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert hero party! user: {user_id}, user_party_team_id: {user_party_team_id}"
+            )
+            return None
+
+        return result.lastrowid
+
+    def put_player_quest(self, user_id: int, episode_id: int, quest_clear_flag: bool, clear_time: int, combo_num: int, total_damage: int, concurrent_destroying_num: int) -> Optional[int]:
+        sql = insert(quest).values(
+            user=user_id,
+            episode_id=episode_id,
+            quest_clear_flag=quest_clear_flag,
+            clear_time=clear_time,
+            combo_num=combo_num,
+            total_damage=total_damage,
+            concurrent_destroying_num=concurrent_destroying_num
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            quest_clear_flag=quest_clear_flag,
+            clear_time=clear_time,
+            combo_num=combo_num,
+            total_damage=total_damage,
+            concurrent_destroying_num=concurrent_destroying_num
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert quest! user: {user_id}, episode_id: {episode_id}"
+            )
+            return None
+
+        return result.lastrowid
+
+    def get_user_equipment(self, user_id: int, equipment_id: int) -> Optional[Dict]:
+        sql = equipment_data.select(equipment_data.c.user == user_id and equipment_data.c.equipment_id == equipment_id)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def get_user_equipments(
+        self, user_id: int
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all equipments lookup given a profile
+        """
+        sql = equipment_data.select(
+            and_(
+                equipment_data.c.user == user_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_user_items(
+        self, user_id: int
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all items lookup given a profile
+        """
+        sql = item_data.select(
+            and_(
+                item_data.c.user == user_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_hero_log(
+        self, user_id: int, user_hero_log_id: int = None
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all hero lookup given a profile and user_party_team_id and ID specifiers
+        """
+        sql = hero_log_data.select(
+            and_(
+                hero_log_data.c.user == user_id,
+                hero_log_data.c.user_hero_log_id == user_hero_log_id if user_hero_log_id is not None else True,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_hero_logs(
+        self, user_id: int
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all hero lookup given a profile
+        """
+        sql = hero_log_data.select(
+            and_(
+                hero_log_data.c.user == user_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_hero_party(
+        self, user_id: int, user_party_team_id: int = None
+    ) -> Optional[List[Row]]:
+        sql = hero_party.select(
+            and_(
+                hero_party.c.user == user_id,
+                hero_party.c.user_party_team_id == user_party_team_id if user_party_team_id is not None else True,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_quest_log(
+        self, user_id: int, episode_id: int = None
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all quest lookup given a profile and episode_id
+        """
+        sql = quest.select(
+            and_(
+                quest.c.user == user_id,
+                quest.c.episode_id == episode_id if episode_id is not None else True,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_quest_logs(
+        self, user_id: int
+    ) -> Optional[List[Row]]:
+        """
+        A catch-all quest lookup given a profile
+        """
+        sql = quest.select(
+            and_(
+                quest.c.user == user_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_session(
+        self, user_id: int = None
+    ) -> Optional[List[Row]]:
+        sql = sessions.select(
+            and_(
+                sessions.c.user == user_id,
+            )
+        ).order_by(
+            sessions.c.play_date.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_end_session(
+        self, user_id: int = None
+    ) -> Optional[List[Row]]:
+        sql = end_sessions.select(
+            and_(
+                end_sessions.c.user == user_id,
+            )
+        ).order_by(
+            end_sessions.c.play_date.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def remove_hero_log(self, user_id: int, user_hero_log_id: int) -> None:
+        sql = hero_log_data.delete(
+           and_(
+                hero_log_data.c.user == user_id,
+                hero_log_data.c.user_hero_log_id == user_hero_log_id,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to remove hero log! profile: {user_id}, user_hero_log_id: {user_hero_log_id}"
+            )
+        return None
+
+    def remove_equipment(self, user_id: int, equipment_id: int) -> None:
+        sql = equipment_data.delete(
+            and_(equipment_data.c.user == user_id, equipment_data.c.equipment_id == equipment_id)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to remove equipment! profile: {user_id}, equipment_id: {equipment_id}"
+            )
+        return None
+
+    def remove_item(self, user_id: int, item_id: int) -> None:
+        sql = item_data.delete(
+            and_(item_data.c.user == user_id, item_data.c.item_id == item_id)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to remove item! profile: {user_id}, item_id: {item_id}"
+            )
+        return None

titles/sao/schema/profile.py

@@ -0,0 +1,80 @@
+from typing import Optional, Dict, List
+from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_, case
+from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
+from sqlalchemy.schema import ForeignKey
+from sqlalchemy.sql import func, select, update, delete
+from sqlalchemy.engine import Row
+from sqlalchemy.dialects.mysql import insert
+
+from core.data.schema import BaseData, metadata
+from ..const import SaoConstants
+
+profile = Table(
+    "sao_profile",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+        unique=True,
+    ),
+    Column("user_type", Integer, server_default="1"),
+    Column("nick_name", String(16), server_default="PLAYER"),
+    Column("rank_num", Integer, server_default="1"),
+    Column("rank_exp", Integer, server_default="0"),
+    Column("own_col", Integer, server_default="0"),
+    Column("own_vp", Integer, server_default="0"),
+    Column("own_yui_medal", Integer, server_default="0"),
+    Column("setting_title_id", Integer, server_default="20005"),
+)
+
+class SaoProfileData(BaseData):
+    def create_profile(self, user_id: int) -> Optional[int]:
+        sql = insert(profile).values(user=user_id)
+        conflict = sql.on_duplicate_key_update(user=user_id)
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(f"Failed to create SAO profile for user {user_id}!")
+            return None
+        return result.lastrowid
+
+    def put_profile(self, user_id: int, user_type: int, nick_name: str, rank_num: int, rank_exp: int, own_col: int, own_vp: int, own_yui_medal: int, setting_title_id: int) -> Optional[int]:
+        sql = insert(profile).values(
+            user=user_id,
+            user_type=user_type,
+            nick_name=nick_name,
+            rank_num=rank_num,
+            rank_exp=rank_exp,
+            own_col=own_col,
+            own_vp=own_vp,
+            own_yui_medal=own_yui_medal,
+            setting_title_id=setting_title_id
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            rank_num=rank_num,
+            rank_exp=rank_exp,
+            own_col=own_col,
+            own_vp=own_vp,
+            own_yui_medal=own_yui_medal,
+            setting_title_id=setting_title_id
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            self.logger.error(
+                f"{__name__} failed to insert profile! user: {user_id}"
+            )
+            return None
+
+        print(result.lastrowid)
+        return result.lastrowid
+
+    def get_profile(self, user_id: int) -> Optional[Row]:
+        sql = profile.select(profile.c.user == user_id)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()

titles/sao/schema/static.py

@@ -0,0 +1,368 @@
+from typing import Dict, List, Optional
+from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float
+from sqlalchemy.engine.base import Connection
+from sqlalchemy.engine import Row
+from sqlalchemy.schema import ForeignKey
+from sqlalchemy.sql import func, select
+from sqlalchemy.dialects.mysql import insert
+
+from core.data.schema import BaseData, metadata
+
+quest = Table(
+    "sao_static_quest",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("version", Integer),
+    Column("questSceneId", Integer),
+    Column("sortNo", Integer),
+    Column("name", String(255)),
+    Column("enabled", Boolean),
+    UniqueConstraint(
+        "version", "questSceneId", name="sao_static_quest_uk"
+    ),
+    mysql_charset="utf8mb4",
+)
+
+hero = Table(
+    "sao_static_hero_list",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("version", Integer),
+    Column("heroLogId", Integer),
+    Column("name", String(255)),
+    Column("nickname", String(255)),
+    Column("rarity", Integer),
+    Column("skillTableSubId", Integer),
+    Column("awakeningExp", Integer),
+    Column("flavorText", String(255)),
+    Column("enabled", Boolean),
+    UniqueConstraint(
+        "version", "heroLogId", name="sao_static_hero_list_uk"
+    ),
+    mysql_charset="utf8mb4",
+)
+
+equipment = Table(
+    "sao_static_equipment_list",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("version", Integer),
+    Column("equipmentId", Integer),
+    Column("equipmentType", Integer),
+    Column("weaponTypeId", Integer),
+    Column("name", String(255)),
+    Column("rarity", Integer),
+    Column("flavorText", String(255)),
+    Column("enabled", Boolean),
+    UniqueConstraint(
+        "version", "equipmentId", name="sao_static_equipment_list_uk"
+    ),
+    mysql_charset="utf8mb4",
+)
+
+item = Table(
+    "sao_static_item_list",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("version", Integer),
+    Column("itemId", Integer),
+    Column("itemTypeId", Integer),
+    Column("name", String(255)),
+    Column("rarity", Integer),
+    Column("flavorText", String(255)),
+    Column("enabled", Boolean),
+    UniqueConstraint(
+        "version", "itemId", name="sao_static_item_list_uk"
+    ),
+    mysql_charset="utf8mb4",
+)
+
+support = Table(
+    "sao_static_support_log_list",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("version", Integer),
+    Column("supportLogId", Integer),
+    Column("charaId", Integer),
+    Column("name", String(255)),
+    Column("rarity", Integer),
+    Column("salePrice", Integer),
+    Column("skillName", String(255)),
+    Column("enabled", Boolean),
+    UniqueConstraint(
+        "version", "supportLogId", name="sao_static_support_log_list_uk"
+    ),
+    mysql_charset="utf8mb4",
+)
+
+rare_drop = Table(
+    "sao_static_rare_drop_list",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("version", Integer),
+    Column("questRareDropId", Integer),
+    Column("commonRewardId", Integer),
+    Column("enabled", Boolean),
+    UniqueConstraint(
+        "version", "questRareDropId", "commonRewardId", name="sao_static_rare_drop_list_uk"
+    ),
+    mysql_charset="utf8mb4",
+)
+
+title = Table(
+    "sao_static_title_list",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column("version", Integer),
+    Column("titleId", Integer),
+    Column("displayName", String(255)),
+    Column("requirement", Integer),
+    Column("rank", Integer),
+    Column("imageFilePath", String(255)),
+    Column("enabled", Boolean),
+    UniqueConstraint(
+        "version", "titleId", name="sao_static_title_list_uk"
+    ),
+    mysql_charset="utf8mb4",
+)
+
+class SaoStaticData(BaseData):
+    def put_quest( self, questSceneId: int, version: int, sortNo: int, name: str, enabled: bool ) -> Optional[int]:
+        sql = insert(quest).values(
+            questSceneId=questSceneId,
+            version=version,
+            sortNo=sortNo,
+            name=name,
+            tutorial=tutorial,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            name=name, questSceneId=questSceneId, version=version
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+    
+    def put_hero( self, version: int, heroLogId: int, name: str, nickname: str, rarity: int, skillTableSubId: int, awakeningExp: int, flavorText: str, enabled: bool ) -> Optional[int]:
+        sql = insert(hero).values(
+            version=version,
+            heroLogId=heroLogId,
+            name=name,
+            nickname=nickname,
+            rarity=rarity,
+            skillTableSubId=skillTableSubId,
+            awakeningExp=awakeningExp,
+            flavorText=flavorText,
+            enabled=enabled
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            name=name, heroLogId=heroLogId
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+    
+    def put_equipment( self, version: int, equipmentId: int, name: str, equipmentType: int, weaponTypeId:int, rarity: int, flavorText: str, enabled: bool ) -> Optional[int]:
+        sql = insert(equipment).values(
+            version=version,
+            equipmentId=equipmentId,
+            name=name,
+            equipmentType=equipmentType,
+            weaponTypeId=weaponTypeId,
+            rarity=rarity,
+            flavorText=flavorText,
+            enabled=enabled
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            name=name, equipmentId=equipmentId
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+    
+    def put_item( self, version: int, itemId: int, name: str, itemTypeId: int, rarity: int, flavorText: str, enabled: bool ) -> Optional[int]:
+        sql = insert(item).values(
+            version=version,
+            itemId=itemId,
+            name=name,
+            itemTypeId=itemTypeId,
+            rarity=rarity,
+            flavorText=flavorText,
+            enabled=enabled
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            name=name, itemId=itemId
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+    
+    def put_support_log( self, version: int, supportLogId: int, charaId: int, name: str, rarity: int, salePrice: int, skillName: str, enabled: bool ) -> Optional[int]:
+        sql = insert(support).values(
+            version=version,
+            supportLogId=supportLogId,
+            charaId=charaId,
+            name=name,
+            rarity=rarity,
+            salePrice=salePrice,
+            skillName=skillName,
+            enabled=enabled
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            name=name, supportLogId=supportLogId
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def put_rare_drop( self, version: int, questRareDropId: int, commonRewardId: int, enabled: bool ) -> Optional[int]:
+        sql = insert(rare_drop).values(
+            version=version,
+            questRareDropId=questRareDropId,
+            commonRewardId=commonRewardId,
+            enabled=enabled,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            questRareDropId=questRareDropId, commonRewardId=commonRewardId, version=version
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+    
+    def put_title( self, version: int, titleId: int, displayName: str, requirement: int, rank: int, imageFilePath: str, enabled: bool ) -> Optional[int]:
+        sql = insert(title).values(
+            version=version,
+            titleId=titleId,
+            displayName=displayName,
+            requirement=requirement,
+            rank=rank,
+            imageFilePath=imageFilePath,
+            enabled=enabled
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            displayName=displayName, titleId=titleId
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def get_quests_id(self, sortNo: int) -> Optional[Dict]:
+        sql = quest.select(quest.c.sortNo == sortNo)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_quests_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
+        sql = quest.select(quest.c.version == version and quest.c.enabled == enabled).order_by(
+            quest.c.questSceneId.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return [list[2] for list in result.fetchall()]
+    
+    def get_hero_id(self, heroLogId: int) -> Optional[Dict]:
+        sql = hero.select(hero.c.heroLogId == heroLogId)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def get_hero_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
+        sql = hero.select(hero.c.version == version and hero.c.enabled == enabled).order_by(
+            hero.c.heroLogId.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return [list[2] for list in result.fetchall()]
+    
+    def get_equipment_id(self, equipmentId: int) -> Optional[Dict]:
+        sql = equipment.select(equipment.c.equipmentId == equipmentId)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def get_equipment_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
+        sql = equipment.select(equipment.c.version == version and equipment.c.enabled == enabled).order_by(
+            equipment.c.equipmentId.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return [list[2] for list in result.fetchall()]
+
+    def get_item_id(self, itemId: int) -> Optional[Dict]:
+        sql = item.select(item.c.itemId == itemId)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_rare_drop_id(self, questRareDropId: int) -> Optional[Dict]:
+        sql = rare_drop.select(rare_drop.c.questRareDropId == questRareDropId)
+        
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+    
+    def get_item_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
+        sql = item.select(item.c.version == version and item.c.enabled == enabled).order_by(
+            item.c.itemId.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return [list[2] for list in result.fetchall()]
+    
+    def get_support_log_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
+        sql = support.select(support.c.version == version and support.c.enabled == enabled).order_by(
+            support.c.supportLogId.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return [list[2] for list in result.fetchall()]
+    
+    def get_title_ids(self, version: int, enabled: bool) -> Optional[List[Dict]]:
+        sql = title.select(title.c.version == version and title.c.enabled == enabled).order_by(
+            title.c.titleId.asc()
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return [list[2] for list in result.fetchall()]

titles/wacca/base.py

@@ -192,7 +192,7 @@ class WaccaBase:
         else:
             profile = self.data.profile.get_profile(req.userId)
             if profile is None:
-                self.logger.warn(
+                self.logger.warning(
                     f"Unknown user id {req.userId} attempted login from {req.chipId}"
                 )
                 return resp.make()
@@ -282,7 +282,7 @@ class WaccaBase:
 
         profile = self.data.profile.get_profile(req.userId)
         if profile is None:
-            self.logger.warn(f"Unknown profile {req.userId}")
+            self.logger.warning(f"Unknown profile {req.userId}")
             return resp.make()
 
         self.logger.info(f"Get detail for profile {req.userId}")
@@ -426,7 +426,7 @@ class WaccaBase:
                         elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]:
                             resp.userItems.noteSounds.append(itm_send)
 
-                except:
+                except Exception:
                     self.logger.error(
                         f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}"
                     )
@@ -624,10 +624,10 @@ class WaccaBase:
         current_wp = profile["wp"]
 
         tickets = self.data.item.get_tickets(user_id)
-        new_tickets = []
+        new_tickets: List[TicketItem] = []
 
         for ticket in tickets:
-            new_tickets.append([ticket["id"], ticket["ticket_id"], 9999999999])
+            new_tickets.append(TicketItem(ticket["id"], ticket["ticket_id"], 9999999999))
 
         for item in req.itemsUsed:
             if (
@@ -645,11 +645,11 @@ class WaccaBase:
                 and not self.game_config.mods.infinite_tickets
             ):
                 for x in range(len(new_tickets)):
-                    if new_tickets[x][1] == item.itemId:
+                    if new_tickets[x].ticketId == item.itemId:
                         self.logger.debug(
-                            f"Remove ticket ID {new_tickets[x][0]} type {new_tickets[x][1]} from {user_id}"
+                            f"Remove ticket ID {new_tickets[x].userTicketId} type {new_tickets[x].ticketId} from {user_id}"
                         )
-                        self.data.item.spend_ticket(new_tickets[x][0])
+                        self.data.item.spend_ticket(new_tickets[x].userTicketId)
                         new_tickets.pop(x)
                         break
 
@@ -709,7 +709,7 @@ class WaccaBase:
         profile = self.data.profile.get_profile(req.profileId)
 
         if profile is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"handle_user_music_update_request: No profile for game_id {req.profileId}"
             )
             return resp.make()
@@ -1003,7 +1003,7 @@ class WaccaBase:
 
         profile = self.data.profile.get_profile(req.profileId)
         if profile is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"handle_user_vip_get_request no profile with ID {req.profileId}"
             )
             return BaseResponse().make()
@@ -1074,17 +1074,17 @@ class WaccaBase:
                         old_score = self.data.score.get_best_score(
                             user_id, item.itemId, item.quantity
                         )
-                    if not old_score:
-                        self.data.score.put_best_score(
-                            user_id,
-                            item.itemId,
-                            item.quantity,
-                            0,
-                            [0] * 5,
-                            [0] * 13,
-                            0,
-                            0,
-                        )
+                        if not old_score:
+                            self.data.score.put_best_score(
+                                user_id,
+                                item.itemId,
+                                item.quantity,
+                                0,
+                                [0] * 5,
+                                [0] * 13,
+                                0,
+                                0,
+                            )
 
                     if item.quantity == 0:
                         item.quantity = WaccaConstants.Difficulty.HARD.value

titles/wacca/const.py

@@ -221,5 +221,5 @@ class WaccaConstants:
                 cls.Region.YAMANASHI,
                 cls.Region.WAKAYAMA,
             ][region]
-        except:
+        except Exception:
             return None

titles/wacca/frontend.py

@@ -2,8 +2,9 @@ import yaml
 import jinja2
 from twisted.web.http import Request
 from os import path
+from twisted.web.server import Session
 
-from core.frontend import FE_Base
+from core.frontend import FE_Base, IUserSession
 from core.config import CoreConfig
 from titles.wacca.database import WaccaData
 from titles.wacca.config import WaccaConfig
@@ -27,7 +28,11 @@ class WaccaFrontend(FE_Base):
         template = self.environment.get_template(
             "titles/wacca/frontend/wacca_index.jinja"
         )
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        
         return template.render(
             title=f"{self.core_config.server.name} | {self.nav_name}",
             game_list=self.environment.globals["game_list"],
+            sesh=vars(usr_sesh)
         ).encode("utf-16")

titles/wacca/handlers/advertise.py

@@ -8,12 +8,12 @@ from titles.wacca.handlers.helpers import Notice
 class GetNewsResponseV1(BaseResponse):
     def __init__(self) -> None:
         super().__init__()
-        self.notices: list[Notice] = []
-        self.copywrightListings: list[str] = []
-        self.stoppedSongs: list[int] = []
-        self.stoppedJackets: list[int] = []
-        self.stoppedMovies: list[int] = []
-        self.stoppedIcons: list[int] = []
+        self.notices: List[Notice] = []
+        self.copywrightListings: List[str] = []
+        self.stoppedSongs: List[int] = []
+        self.stoppedJackets: List[int] = []
+        self.stoppedMovies: List[int] = []
+        self.stoppedIcons: List[int] = []
 
     def make(self) -> Dict:
         note = []
@@ -34,7 +34,7 @@ class GetNewsResponseV1(BaseResponse):
 
 
 class GetNewsResponseV2(GetNewsResponseV1):
-    stoppedProducts: list[int] = []
+    stoppedProducts: List[int] = []
 
     def make(self) -> Dict:
         super().make()
@@ -44,8 +44,8 @@ class GetNewsResponseV2(GetNewsResponseV1):
 
 
 class GetNewsResponseV3(GetNewsResponseV2):
-    stoppedNavs: list[int] = []
-    stoppedNavVoices: list[int] = []
+    stoppedNavs: List[int] = []
+    stoppedNavVoices: List[int] = []
 
     def make(self) -> Dict:
         super().make()

titles/wacca/handlers/user_music.py

@@ -93,13 +93,10 @@ class UserMusicUnlockRequest(BaseRequest):
 
 
 class UserMusicUnlockResponse(BaseResponse):
-    def __init__(self, current_wp: int = 0, tickets_remaining: List = []) -> None:
+    def __init__(self, current_wp: int = 0, tickets_remaining: List[TicketItem] = []) -> None:
         super().__init__()
         self.wp = current_wp
-        self.tickets: List[TicketItem] = []
-
-        for ticket in tickets_remaining:
-            self.tickets.append(TicketItem(ticket[0], ticket[1], ticket[2]))
+        self.tickets = tickets_remaining
 
     def make(self) -> Dict:
         tickets = []

titles/wacca/index.py

@@ -93,7 +93,7 @@ class WaccaServlet:
         try:
             req_json = json.loads(request.content.getvalue())
             version_full = Version(req_json["appVersion"])
-        except:
+        except Exception:
             self.logger.error(
                 f"Failed to parse request to {url_path} -> {request.content.getvalue()}"
             )
@@ -146,7 +146,7 @@ class WaccaServlet:
         self.logger.debug(req_json)
 
         if not hasattr(self.versions[internal_ver], func_to_find):
-            self.logger.warn(
+            self.logger.warning(
                 f"{req_json['appVersion']} has no handler for {func_to_find}"
             )
             resp = BaseResponse().make()

titles/wacca/lily.py

@@ -157,7 +157,7 @@ class WaccaLily(WaccaS):
         else:
             profile = self.data.profile.get_profile(req.userId)
             if profile is None:
-                self.logger.warn(
+                self.logger.warning(
                     f"Unknown user id {req.userId} attempted login from {req.chipId}"
                 )
                 return resp.make()
@@ -198,7 +198,7 @@ class WaccaLily(WaccaS):
 
         profile = self.data.profile.get_profile(req.userId)
         if profile is None:
-            self.logger.warn(f"Unknown profile {req.userId}")
+            self.logger.warning(f"Unknown profile {req.userId}")
             return resp.make()
 
         self.logger.info(f"Get detail for profile {req.userId}")
@@ -424,7 +424,7 @@ class WaccaLily(WaccaS):
                         elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]:
                             resp.userItems.noteSounds.append(itm_send)
 
-                except:
+                except Exception:
                     self.logger.error(
                         f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}"
                     )

titles/wacca/read.py

@@ -41,7 +41,7 @@ class WaccaReader(BaseReader):
 
     def read_music(self, base_dir: str, table: str) -> None:
         if not self.check_valid_pair(base_dir, table):
-            self.logger.warn(
+            self.logger.warning(
                 f"Cannot find {table} uasset/uexp pair at {base_dir}, music will not be read"
             )
             return

titles/wacca/reverse.py

@@ -58,7 +58,7 @@ class WaccaReverse(WaccaLilyR):
 
         profile = self.data.profile.get_profile(req.userId)
         if profile is None:
-            self.logger.warn(f"Unknown profile {req.userId}")
+            self.logger.warning(f"Unknown profile {req.userId}")
             return resp.make()
 
         self.logger.info(f"Get detail for profile {req.userId}")
@@ -289,7 +289,7 @@ class WaccaReverse(WaccaLilyR):
                         elif item["type"] == WaccaConstants.ITEM_TYPES["note_sound"]:
                             resp.userItems.noteSounds.append(itm_send)
 
-                except:
+                except Exception:
                     self.logger.error(
                         f"{__name__} Failed to load item {item['item_id']} for user {profile['user']}"
                     )

titles/wacca/schema/item.py

@@ -169,7 +169,7 @@ class WaccaItemData(BaseData):
 
         result = self.execute(sql)
         if result is None:
-            self.logger.warn(f"Failed to delete ticket id {id}")
+            self.logger.warning(f"Failed to delete ticket id {id}")
             return None
 
     def get_trophies(self, user_id: int, season: int = None) -> Optional[List[Row]]:

titles/wacca/schema/profile.py

@@ -218,7 +218,7 @@ class WaccaProfileData(BaseData):
 
         result = self.execute(sql)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"update_profile_dan: Failed to update! profile {profile_id}"
             )
             return None

titles/wacca/schema/score.py

@@ -294,7 +294,7 @@ class WaccaScoreData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(
+            self.logger.warning(
                 f"put_stageup: failed to update! user_id: {user_id} version: {version} stage_id: {stage_id}"
             )
             return None

titles/wacca/schema/static.py

@@ -63,7 +63,7 @@ class WaccaStaticData(BaseData):
 
         result = self.execute(conflict)
         if result is None:
-            self.logger.warn(f"Failed to insert music {song_id} chart {chart_id}")
+            self.logger.warning(f"Failed to insert music {song_id} chart {chart_id}")
             return None
         return result.lastrowid