refactor(db)!: new charts view & models refactor

src/arcaea_offline/calculate.py

@@ -2,7 +2,7 @@ from decimal import Decimal
 from math import floor
 from typing import Dict, List
 
-from .models.scores import Calculated
+from .models.scores import ScoreCalculated
 
 
 def calculate_score_range(notes: int, pure: int, far: int):
@@ -30,8 +30,10 @@ def calculate_shiny_pure(notes: int, score: int, pure: int, far: int) -> int:
     return score - floor(actual_score)
 
 
-def get_b30_calculated_list(calculated_list: List[Calculated]) -> List[Calculated]:
+def get_b30_calculated_list(
-    best_scores: Dict[str, Calculated] = {}
+    calculated_list: List[ScoreCalculated],
+) -> List[ScoreCalculated]:
+    best_scores: Dict[str, ScoreCalculated] = {}
     for calculated in calculated_list:
         key = f"{calculated.song_id}_{calculated.rating_class}"
         stored = best_scores.get(key)
@@ -42,7 +44,7 @@ def get_b30_calculated_list(calculated_list: List[Calculated]) -> List[Calculate
     return ret_list
 
 
-def calculate_b30(calculated_list: List[Calculated]) -> Decimal:
+def calculate_b30(calculated_list: List[ScoreCalculated]) -> Decimal:
     ptt_list = [Decimal(c.potential) for c in get_b30_calculated_list(calculated_list)]
     sum_ptt_list = sum(ptt_list)
     return (sum_ptt_list / len(ptt_list)) if sum_ptt_list else Decimal("0.0")

src/arcaea_offline/database.py

@@ -56,9 +56,11 @@ class Database(metaclass=Singleton):
             # > https://github.com/kvesteri/sqlalchemy-utils/issues/396
             # > view.create_view() causes DuplicateTableError on Base.metadata.create_all(checkfirst=True)
             # so if `checkfirst` is True, drop these views before creating
+            SongsViewBase.metadata.drop_all(self.engine)
             ScoresViewBase.metadata.drop_all(self.engine)
 
         SongsBase.metadata.create_all(self.engine, checkfirst=checkfirst)
+        SongsViewBase.metadata.create_all(self.engine)
         ScoresBase.metadata.create_all(self.engine, checkfirst=checkfirst)
         ScoresViewBase.metadata.create_all(self.engine)
         ConfigBase.metadata.create_all(self.engine, checkfirst=checkfirst)
@@ -68,7 +70,7 @@ class Database(metaclass=Singleton):
             stmt = select(Property.value).where(Property.key == "version")
             result = session.execute(stmt).fetchone()
             if not checkfirst or not result:
-                session.add(Property(key="version", value="2"))
+                session.add(Property(key="version", value="3"))
                 session.commit()
 
     def check_init(self) -> bool:
@@ -78,8 +80,8 @@ class Database(metaclass=Singleton):
             + list(ScoresBase.metadata.tables.keys())
             + list(ConfigBase.metadata.tables.keys())
             + [
-                Calculated.__tablename__,
+                ScoreCalculated.__tablename__,
-                Best.__tablename__,
+                ScoreBest.__tablename__,
                 CalculatedPotential.__tablename__,
             ]
         )
@@ -97,6 +99,12 @@ class Database(metaclass=Singleton):
             results = list(session.scalars(stmt))
         return results
 
+    def get_pack_by_id(self, value: str):
+        stmt = select(Pack).where(Pack.id == value)
+        with self.sessionmaker() as session:
+            result = session.scalar(stmt)
+        return result
+
     def get_charts_in_pack(self, pack: str):
         stmt = (
             select(ChartInfo)

src/arcaea_offline/external/arcaea/songlist.py

@@ -1,7 +1,7 @@
 import json
 from typing import List, Union
 
-from ...models.songs import Chart, ChartLocalized, Song, SongLocalized
+from ...models.songs import Difficulty, DifficultyLocalized, Song, SongLocalized
 from .common import ArcaeaParser, is_localized, set_model_localized_attrs, to_db_value
 
 
@@ -9,7 +9,9 @@ class SonglistParser(ArcaeaParser):
     def __init__(self, filepath):
         super().__init__(filepath)
 
-    def parse(self) -> List[Union[Song, SongLocalized, Chart, ChartLocalized]]:
+    def parse(
+        self,
+    ) -> List[Union[Song, SongLocalized, Difficulty, DifficultyLocalized]]:
         with open(self.filepath, "r", encoding="utf-8") as sl_f:
             songlist_json_root = json.loads(sl_f.read())
 
@@ -63,7 +65,7 @@ class SonglistDifficultiesParser(ArcaeaParser):
     def __init__(self, filepath):
         self.filepath = filepath
 
-    def parse(self) -> List[Union[Chart, ChartLocalized]]:
+    def parse(self) -> List[Union[Difficulty, DifficultyLocalized]]:
         with open(self.filepath, "r", encoding="utf-8") as sl_f:
             songlist_json_root = json.loads(sl_f.read())
 
@@ -74,7 +76,7 @@ class SonglistDifficultiesParser(ArcaeaParser):
                 continue
 
             for item in song_item["difficulties"]:
-                chart = Chart(song_id=song_item["id"])
+                chart = Difficulty(song_id=song_item["id"])
                 chart.rating_class = item["ratingClass"]
                 chart.rating = item["rating"]
                 chart.rating_plus = item.get("ratingPlus") or False
@@ -94,7 +96,7 @@ class SonglistDifficultiesParser(ArcaeaParser):
                 results.append(chart)
 
                 if is_localized(item, "title") or is_localized(item, "artist"):
-                    chart_localized = ChartLocalized(
+                    chart_localized = DifficultyLocalized(
                         song_id=chart.song_id, rating_class=chart.rating_class
                     )
                     set_model_localized_attrs(chart_localized, item, "title")

src/arcaea_offline/models/config.py

@@ -14,7 +14,7 @@ class ConfigBase(DeclarativeBase, ReprHelper):
 
 
 class Property(ConfigBase):
-    __tablename__ = "property"
+    __tablename__ = "properties"
 
     key: Mapped[str] = mapped_column(TEXT(), primary_key=True)
     value: Mapped[str] = mapped_column(TEXT())

src/arcaea_offline/models/scores.py

@@ -5,14 +5,14 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 from sqlalchemy_utils import create_view
 
 from .common import ReprHelper
-from .songs import Chart, ChartInfo
+from .songs import ChartInfo, Difficulty
 
 __all__ = [
     "ScoresBase",
     "Score",
     "ScoresViewBase",
-    "Calculated",
+    "ScoreCalculated",
-    "Best",
+    "ScoreBest",
     "CalculatedPotential",
 ]
 
@@ -22,7 +22,7 @@ class ScoresBase(DeclarativeBase, ReprHelper):
 
 
 class Score(ScoresBase):
-    __tablename__ = "score"
+    __tablename__ = "scores"
 
     id: Mapped[int] = mapped_column(autoincrement=True, primary_key=True)
     song_id: Mapped[str] = mapped_column(TEXT())
@@ -36,6 +36,7 @@ class Score(ScoresBase):
     r10_clear_type: Mapped[Optional[int]] = mapped_column(
         comment="0: LOST, 1: COMPLETE, 2: HARD_LOST"
     )
+    comment: Mapped[Optional[str]] = mapped_column(TEXT())
 
 
 # How to create an SQL View with SQLAlchemy?
@@ -47,35 +48,31 @@ class ScoresViewBase(DeclarativeBase, ReprHelper):
     pass
 
 
-class Calculated(ScoresViewBase):
+class ScoreCalculated(ScoresViewBase):
-    __tablename__ = "calculated"
+    __tablename__ = "scores_calculated"
 
-    score_id: Mapped[str]
+    id: Mapped[int]
     song_id: Mapped[str]
     rating_class: Mapped[int]
     score: Mapped[int]
     pure: Mapped[Optional[int]]
+    shiny_pure: Mapped[Optional[int]]
     far: Mapped[Optional[int]]
     lost: Mapped[Optional[int]]
     date: Mapped[Optional[int]]
     max_recall: Mapped[Optional[int]]
     r10_clear_type: Mapped[Optional[int]]
-    shiny_pure: Mapped[Optional[int]]
     potential: Mapped[float]
+    comment: Mapped[Optional[str]]
 
     __table__ = create_view(
         name=__tablename__,
         selectable=select(
-            Score.id.label("score_id"),
+            Score.id,
-            Chart.song_id,
+            Difficulty.song_id,
-            Chart.rating_class,
+            Difficulty.rating_class,
             Score.score,
             Score.pure,
-            Score.far,
-            Score.lost,
-            Score.date,
-            Score.max_recall,
-            Score.r10_clear_type,
             (
                 Score.score
                 - func.floor(
@@ -83,6 +80,11 @@ class Calculated(ScoresViewBase):
                     + (Score.far * 0.5 * 10000000.0 / ChartInfo.note)
                 )
             ).label("shiny_pure"),
+            Score.far,
+            Score.lost,
+            Score.date,
+            Score.max_recall,
+            Score.r10_clear_type,
             case(
                 (Score.score >= 10000000, ChartInfo.constant / 10.0 + 2),
                 (
@@ -94,48 +96,54 @@ class Calculated(ScoresViewBase):
                     0,
                 ),
             ).label("potential"),
+            Score.comment,
         )
-        .select_from(Chart)
+        .select_from(Difficulty)
         .join(
             ChartInfo,
-            (Chart.song_id == ChartInfo.song_id)
+            (Difficulty.song_id == ChartInfo.song_id)
-            & (Chart.rating_class == ChartInfo.rating_class),
+            & (Difficulty.rating_class == ChartInfo.rating_class),
         )
         .join(
             Score,
-            (Chart.song_id == Score.song_id)
+            (Difficulty.song_id == Score.song_id)
-            & (Chart.rating_class == Score.rating_class),
+            & (Difficulty.rating_class == Score.rating_class),
         ),
         metadata=ScoresViewBase.metadata,
         cascade_on_drop=False,
     )
 
 
-class Best(ScoresViewBase):
+class ScoreBest(ScoresViewBase):
-    __tablename__ = "best"
+    __tablename__ = "scores_best"
 
-    score_id: Mapped[str]
+    id: Mapped[int]
     song_id: Mapped[str]
     rating_class: Mapped[int]
     score: Mapped[int]
     pure: Mapped[Optional[int]]
+    shiny_pure: Mapped[Optional[int]]
     far: Mapped[Optional[int]]
     lost: Mapped[Optional[int]]
     date: Mapped[Optional[int]]
     max_recall: Mapped[Optional[int]]
     r10_clear_type: Mapped[Optional[int]]
-    shiny_pure: Mapped[Optional[int]]
     potential: Mapped[float]
+    comment: Mapped[Optional[str]]
 
     __table__ = create_view(
         name=__tablename__,
         selectable=select(
-            *[col for col in inspect(Calculated).columns if col.name != "potential"],
+            *[
-            func.max(Calculated.potential).label("potential"),
+                col
+                for col in inspect(ScoreCalculated).columns
+                if col.name != "potential"
+            ],
+            func.max(ScoreCalculated.potential).label("potential"),
         )
-        .select_from(Calculated)
+        .select_from(ScoreCalculated)
-        .group_by(Calculated.song_id, Calculated.rating_class)
+        .group_by(ScoreCalculated.song_id, ScoreCalculated.rating_class)
-        .order_by(Calculated.potential.desc()),
+        .order_by(ScoreCalculated.potential.desc()),
         metadata=ScoresViewBase.metadata,
         cascade_on_drop=False,
     )
@@ -147,8 +155,8 @@ class CalculatedPotential(ScoresViewBase):
     b30: Mapped[float]
 
     _select_bests_subquery = (
-        select(Best.potential.label("b30_sum"))
+        select(ScoreBest.potential.label("b30_sum"))
-        .order_by(Best.potential.desc())
+        .order_by(ScoreBest.potential.desc())
         .limit(30)
         .subquery()
     )

src/arcaea_offline/models/songs.py

@@ -1,7 +1,8 @@
 from typing import Optional
 
-from sqlalchemy import TEXT, ForeignKey
+from sqlalchemy import TEXT, ForeignKey, func, select
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+from sqlalchemy_utils import create_view
 
 from .common import ReprHelper
 
@@ -11,9 +12,11 @@ __all__ = [
     "PackLocalized",
     "Song",
     "SongLocalized",
-    "Chart",
+    "Difficulty",
-    "ChartLocalized",
+    "DifficultyLocalized",
     "ChartInfo",
+    "SongsViewBase",
+    "Chart",
 ]
 
 
@@ -22,7 +25,7 @@ class SongsBase(DeclarativeBase, ReprHelper):
 
 
 class Pack(SongsBase):
-    __tablename__ = "pack"
+    __tablename__ = "packs"
 
     id: Mapped[str] = mapped_column(TEXT(), primary_key=True)
     name: Mapped[str] = mapped_column(TEXT())
@@ -30,9 +33,9 @@ class Pack(SongsBase):
 
 
 class PackLocalized(SongsBase):
-    __tablename__ = "pack_localized"
+    __tablename__ = "packs_localized"
 
-    id: Mapped[str] = mapped_column(ForeignKey("pack.id"), primary_key=True)
+    id: Mapped[str] = mapped_column(ForeignKey("packs.id"), primary_key=True)
     name_ja: Mapped[Optional[str]] = mapped_column(TEXT())
     name_ko: Mapped[Optional[str]] = mapped_column(TEXT())
     name_zh_hans: Mapped[Optional[str]] = mapped_column(TEXT())
@@ -44,7 +47,7 @@ class PackLocalized(SongsBase):
 
 
 class Song(SongsBase):
-    __tablename__ = "song"
+    __tablename__ = "songs"
 
     idx: Mapped[int]
     id: Mapped[str] = mapped_column(TEXT(), primary_key=True)
@@ -67,29 +70,41 @@ class Song(SongsBase):
 
 
 class SongLocalized(SongsBase):
-    __tablename__ = "song_localized"
+    __tablename__ = "songs_localized"
 
-    id: Mapped[str] = mapped_column(ForeignKey("song.id"), primary_key=True)
+    id: Mapped[str] = mapped_column(ForeignKey("songs.id"), primary_key=True)
     title_ja: Mapped[Optional[str]] = mapped_column(TEXT())
     title_ko: Mapped[Optional[str]] = mapped_column(TEXT())
     title_zh_hans: Mapped[Optional[str]] = mapped_column(TEXT())
     title_zh_hant: Mapped[Optional[str]] = mapped_column(TEXT())
-    search_title_ja: Mapped[Optional[str]] = mapped_column(TEXT(), comment="json")
+    search_title_ja: Mapped[Optional[str]] = mapped_column(TEXT(), comment="JSON array")
-    search_title_ko: Mapped[Optional[str]] = mapped_column(TEXT(), comment="json")
+    search_title_ko: Mapped[Optional[str]] = mapped_column(TEXT(), comment="JSON array")
-    search_title_zh_hans: Mapped[Optional[str]] = mapped_column(TEXT(), comment="json")
+    search_title_zh_hans: Mapped[Optional[str]] = mapped_column(
-    search_title_zh_hant: Mapped[Optional[str]] = mapped_column(TEXT(), comment="json")
+        TEXT(), comment="JSON array"
-    search_artist_ja: Mapped[Optional[str]] = mapped_column(TEXT(), comment="json")
+    )
-    search_artist_ko: Mapped[Optional[str]] = mapped_column(TEXT(), comment="json")
+    search_title_zh_hant: Mapped[Optional[str]] = mapped_column(
-    search_artist_zh_hans: Mapped[Optional[str]] = mapped_column(TEXT(), comment="json")
+        TEXT(), comment="JSON array"
-    search_artist_zh_hant: Mapped[Optional[str]] = mapped_column(TEXT(), comment="json")
+    )
+    search_artist_ja: Mapped[Optional[str]] = mapped_column(
+        TEXT(), comment="JSON array"
+    )
+    search_artist_ko: Mapped[Optional[str]] = mapped_column(
+        TEXT(), comment="JSON array"
+    )
+    search_artist_zh_hans: Mapped[Optional[str]] = mapped_column(
+        TEXT(), comment="JSON array"
+    )
+    search_artist_zh_hant: Mapped[Optional[str]] = mapped_column(
+        TEXT(), comment="JSON array"
+    )
     source_ja: Mapped[Optional[str]] = mapped_column(TEXT())
     source_ko: Mapped[Optional[str]] = mapped_column(TEXT())
     source_zh_hans: Mapped[Optional[str]] = mapped_column(TEXT())
     source_zh_hant: Mapped[Optional[str]] = mapped_column(TEXT())
 
 
-class Chart(SongsBase):
+class Difficulty(SongsBase):
-    __tablename__ = "chart"
+    __tablename__ = "difficulties"
 
     song_id: Mapped[str] = mapped_column(TEXT(), primary_key=True)
     rating_class: Mapped[int] = mapped_column(primary_key=True)
@@ -110,12 +125,14 @@ class Chart(SongsBase):
     date: Mapped[Optional[int]]
 
 
-class ChartLocalized(SongsBase):
+class DifficultyLocalized(SongsBase):
-    __tablename__ = "chart_localized"
+    __tablename__ = "difficulties_localized"
 
-    song_id: Mapped[str] = mapped_column(ForeignKey("chart.song_id"), primary_key=True)
+    song_id: Mapped[str] = mapped_column(
+        ForeignKey("difficulties.song_id"), primary_key=True
+    )
     rating_class: Mapped[str] = mapped_column(
-        ForeignKey("chart.rating_class"), primary_key=True
+        ForeignKey("difficulties.rating_class"), primary_key=True
     )
     title_ja: Mapped[Optional[str]] = mapped_column(TEXT())
     title_ko: Mapped[Optional[str]] = mapped_column(TEXT())
@@ -128,13 +145,95 @@ class ChartLocalized(SongsBase):
 
 
 class ChartInfo(SongsBase):
-    __tablename__ = "chart_info"
+    __tablename__ = "charts_info"
 
-    song_id: Mapped[str] = mapped_column(ForeignKey("chart.song_id"), primary_key=True)
+    song_id: Mapped[str] = mapped_column(
+        ForeignKey("difficulties.song_id"), primary_key=True
+    )
     rating_class: Mapped[str] = mapped_column(
-        ForeignKey("chart.rating_class"), primary_key=True
+        ForeignKey("difficulties.rating_class"), primary_key=True
     )
     constant: Mapped[int] = mapped_column(
         comment="real_constant * 10. For example, Crimson Throne [FTR] is 10.4, then store 104 here."
     )
     note: Mapped[Optional[int]]
+
+
+class SongsViewBase(DeclarativeBase, ReprHelper):
+    pass
+
+
+class Chart(SongsViewBase):
+    __tablename__ = "charts"
+
+    song_idx: Mapped[int]
+    song_id: Mapped[str]
+    rating_class: Mapped[int]
+    rating: Mapped[int]
+    rating_plus: Mapped[bool]
+    title: Mapped[str]
+    artist: Mapped[str]
+    set: Mapped[str]
+    bpm: Mapped[Optional[str]]
+    bpm_base: Mapped[Optional[float]]
+    audio_preview: Mapped[Optional[int]]
+    audio_preview_end: Mapped[Optional[int]]
+    side: Mapped[Optional[int]]
+    version: Mapped[Optional[str]]
+    date: Mapped[Optional[int]]
+    bg: Mapped[Optional[str]]
+    bg_inverse: Mapped[Optional[str]]
+    bg_day: Mapped[Optional[str]]
+    bg_night: Mapped[Optional[str]]
+    source: Mapped[Optional[str]]
+    source_copyright: Mapped[Optional[str]]
+    chart_designer: Mapped[Optional[str]]
+    jacket_desginer: Mapped[Optional[str]]
+    audio_override: Mapped[bool]
+    jacket_override: Mapped[bool]
+    jacket_night: Mapped[Optional[str]]
+    constant: Mapped[int]
+    note: Mapped[Optional[int]]
+
+    __table__ = create_view(
+        name=__tablename__,
+        selectable=select(
+            Song.idx.label("song_idx"),
+            Difficulty.song_id,
+            Difficulty.rating_class,
+            Difficulty.rating,
+            Difficulty.rating_plus,
+            func.coalesce(Difficulty.title, Song.title).label("title"),
+            func.coalesce(Difficulty.artist, Song.artist).label("artist"),
+            Song.set,
+            func.coalesce(Difficulty.bpm, Song.bpm).label("bpm"),
+            func.coalesce(Difficulty.bpm_base, Song.bpm_base).label("bpm_base"),
+            Song.audio_preview,
+            Song.audio_preview_end,
+            Song.side,
+            func.coalesce(Difficulty.version, Song.version).label("version"),
+            func.coalesce(Difficulty.date, Song.date).label("date"),
+            func.coalesce(Difficulty.bg, Song.bg).label("bg"),
+            func.coalesce(Difficulty.bg_inverse, Song.bg_inverse).label("bg_inverse"),
+            Song.bg_day,
+            Song.bg_night,
+            Song.source,
+            Song.source_copyright,
+            Difficulty.chart_designer,
+            Difficulty.jacket_desginer,
+            Difficulty.audio_override,
+            Difficulty.jacket_override,
+            Difficulty.jacket_night,
+            ChartInfo.constant,
+            ChartInfo.note,
+        )
+        .select_from(Difficulty)
+        .join(
+            ChartInfo,
+            (Difficulty.song_id == ChartInfo.song_id)
+            & (Difficulty.rating_class == ChartInfo.rating_class),
+        )
+        .join(Song, Difficulty.song_id == Song.id),
+        metadata=SongsViewBase.metadata,
+        cascade_on_drop=False,
+    )