[O] Optimize auto-ban

src/main/java/icu/samnyan/aqua/net/CardController.kt

@@ -13,6 +13,7 @@ import icu.samnyan.aqua.sega.general.model.Card
 import icu.samnyan.aqua.sega.general.service.CardService
 import icu.samnyan.aqua.sega.maimai2.model.Mai2UserDataRepo
 import icu.samnyan.aqua.sega.wacca.model.db.WcUserRepo
+import jakarta.persistence.EntityManager
 import org.springframework.scheduling.annotation.Scheduled
 import org.springframework.stereotype.Service
 import org.springframework.web.bind.annotation.RestController
@@ -178,7 +179,8 @@ class CardGameService(
     val ongeki: icu.samnyan.aqua.sega.ongeki.dao.userdata.UserDataRepository,
     val diva: icu.samnyan.aqua.sega.diva.dao.userdata.PlayerProfileRepository,
     val safety: AquaNetSafetyService,
-    val cardRepo: CardRepository
+    val cardRepo: CardRepository,
+    val em: EntityManager
 ) {
     companion object {
         val log = logger()
@@ -220,16 +222,20 @@ class CardGameService(
     @Scheduled(fixedDelay = 3600000)
     suspend fun autoBan() {
         log.info("Running auto-ban")
+        val time = millis()
 
         // Ban any players with unacceptable names
         for (repo in listOf(maimai2, chusan, wacca, ongeki)) {
-            repo.findAll().filter { it.card != null && !it.card!!.rankingBanned }.forEach { data ->
+            val all = async { repo.findAllNonBanned() }
-                if (!safety.isSafe(data.userName)) {
+            val isSafe = safety.isSafeBatch(all.map { it.userName })
-                    log.info("Banning user ${data.userName} ${data.card!!.id}")
+            val toSave = all.filterIndexed { i, _ -> !isSafe[i] }.mapNotNull { it.card }
-                    data.card!!.rankingBanned = true
+            if (toSave.isNotEmpty()) {
-                    async { cardRepo.save(data.card!!) }
+                log.info("Banning users ${toSave.joinToString(", ")}")
-                }
+                toSave.forEach { it.rankingBanned = true }
+                async { cardRepo.saveAll(toSave) }
             }
         }
+
+        log.info("Auto-ban completed in ${millis() - time}ms")
     }
 }

src/main/java/icu/samnyan/aqua/net/Safety.kt

@@ -1,7 +1,6 @@
 package icu.samnyan.aqua.net
 
 import ext.HTTP
-import ext.async
 import ext.toJson
 import icu.samnyan.aqua.net.games.BaseEntity
 import io.ktor.client.call.*
@@ -50,25 +49,31 @@ class AquaNetSafetyService(
     val safety: AquaNetSafetyRepo,
     val openAIConfig: OpenAIConfig
 ) {
-    suspend fun isSafe(rawContent: String): Boolean {
+    /**
-        // NFKC normalize
+     * It is very inefficient to have query inside a loop, so we batch the query.
-        val content = Normalizer.normalize(rawContent, Normalizer.Form.NFKC)
+     */
-        if (content.isBlank()) return true
+    suspend fun isSafeBatch(rawContents: List<String>): List<Boolean> {
+        val contents = rawContents.map { Normalizer.normalize(it, Normalizer.Form.NFKC) }
+        val map = safety.findAll().associateBy { it.content.lowercase().trim() }.toMutableMap()
 
-        async { safety.findByContent(content) }?.let { return it.safe }
+        // Process unseen content with OpenAI
-
+        val news = contents.filter { it.lowercase().trim() !in map }.map { inp ->
-        // Query OpenAI
+            HTTP.post("https://api.openai.com/v1/moderations") {
-        HTTP.post("https://api.openai.com/v1/moderations") {
+                header("Authorization", "Bearer ${openAIConfig.apiKey}")
-            header("Authorization", "Bearer ${openAIConfig.apiKey}")
+                header("Content-Type", "application/json")
-            header("Content-Type", "application/json")
+                setBody(mapOf("input" to inp).toJson())
-            setBody(mapOf("input" to content).toJson())
+            }.let {
-        }.let {
+                if (!it.status.isSuccess()) throw Exception("OpenAI request failed for $inp")
-            if (!it.status.isSuccess()) return true
+                val body = it.body<OpenAIResp<OpenAIMod>>()
-            val body = it.body<OpenAIResp<OpenAIMod>>()
+                AquaNetSafety().apply {
-            return AquaNetSafety().apply {
+                    content = inp
-                this.content = content
+                    safe = !body.results.first().flagged
-                this.safe = !body.results.first().flagged
+                }
-            }.also { safety.save(it) }.safe
+            }
         }
+        if (news.isNotEmpty()) safety.saveAll(news)
+        news.associateByTo(map) { it.content.lowercase().trim() }
+
+        return contents.map { map[it.lowercase().trim()]!!.safe }
     }
 }

src/main/java/icu/samnyan/aqua/net/games/Models.kt

@@ -1,13 +1,13 @@
 package icu.samnyan.aqua.net.games
 
 import com.fasterxml.jackson.annotation.JsonIgnore
-import com.fasterxml.jackson.annotation.JsonProperty
-import com.fasterxml.jackson.databind.annotation.JsonSerialize
 import ext.JACKSON
 import ext.JavaSerializable
 import icu.samnyan.aqua.sega.general.model.Card
-import icu.samnyan.aqua.sega.util.jackson.AccessCodeSerializer
+import jakarta.persistence.GeneratedValue
-import jakarta.persistence.*
+import jakarta.persistence.GenerationType
+import jakarta.persistence.Id
+import jakarta.persistence.MappedSuperclass
 import kotlinx.serialization.Serializable
 import org.springframework.data.domain.Page
 import org.springframework.data.domain.Pageable
@@ -95,6 +95,7 @@ interface IUserData {
 }
 
 interface IGenericGamePlaylog {
+    val user: IUserData
     val musicId: Int
     val level: Int
     val userPlayDate: Any
@@ -116,21 +117,15 @@ open class BaseEntity(
     override fun toString() = JACKSON.writeValueAsString(this)
 }
 
-@MappedSuperclass
-open class UserDataEntity : BaseEntity() {
-    @JsonSerialize(using = AccessCodeSerializer::class)
-    @JsonProperty(value = "accessCode", access = JsonProperty.Access.READ_ONLY)
-    @OneToOne
-    @JoinColumn(name = "aime_card_id", unique = true)
-    var card: Card? = null
-}
-
 @NoRepositoryBean
 interface GenericUserDataRepo<T : IUserData> : JpaRepository<T, Long> {
     fun findByCard(card: Card): T?
     fun findByCard_ExtId(extId: Long): Optional<T>
     @Query("select count(*) from #{#entityName} e where e.playerRating > :rating and e.card.rankingBanned = false")
     fun getRanking(rating: Int): Long
+
+    @Query("select e from #{#entityName} e where e.card.rankingBanned = false")
+    fun findAllNonBanned(): List<T>
 }
 
 @NoRepositoryBean

src/main/java/icu/samnyan/aqua/sega/maimai/model/userdata/UserData.java

@@ -31,7 +31,7 @@ public class UserData implements Serializable {
 
     @JsonSerialize(using = AccessCodeSerializer.class)
     @JsonProperty(value = "accessCode", access = JsonProperty.Access.READ_ONLY)
-    @OneToOne
+    @OneToOne(fetch = FetchType.LAZY)
     @JoinColumn(name = "aime_card_id")
     private Card card;
 

src/main/java/icu/samnyan/aqua/sega/maimai2/model/userdata/Mai2UserDetail.kt

@@ -16,7 +16,7 @@ import jakarta.persistence.*
 class Mai2UserDetail(
     @get:JsonSerialize(using = AccessCodeSerializer::class)
     @get:JsonProperty(value = "accessCode", access = JsonProperty.Access.READ_ONLY)
-    @OneToOne
+    @OneToOne(fetch = FetchType.LAZY)
     @JoinColumn(name = "aime_card_id", unique = true)
     override var card: Card? = null,
 

src/main/java/icu/samnyan/aqua/sega/ongeki/model/userdata/UserData.java

@@ -32,7 +32,7 @@ public class UserData implements Serializable, IUserData {
 
     @JsonSerialize(using = AccessCodeSerializer.class)
     @JsonProperty(value = "accessCode", access = JsonProperty.Access.READ_ONLY)
-    @OneToOne
+    @OneToOne(fetch = FetchType.LAZY)
     @JoinColumn(name = "aime_card_id", unique = true)
     private Card card;
     // Access code in card

src/main/java/icu/samnyan/aqua/sega/wacca/model/db/WaccaUser.kt

@@ -16,7 +16,7 @@ import java.util.*
  */
 @Entity @Table(name = "wacca_user")
 class WaccaUser : BaseEntity(), IUserData {
-    @OneToOne
+    @OneToOne(fetch = FetchType.LAZY)
     @JoinColumn(name = "aime_card_id", unique = true)
     override var card: Card? = Card()