mirror of
https://github.com/283375/arcaea-offline.git
synced 2025-06-30 19:56:26 +00:00
feat(db): v1 to v4 migration
This commit is contained in:
@ -0,0 +1,292 @@
|
||||
"""v4
|
||||
|
||||
Revision ID: a3f9d48b7de3
|
||||
Revises:
|
||||
Create Date: 2024-11-24 00:03:07.697165
|
||||
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Mapping, Optional, Sequence, TypedDict, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import context, op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "a3f9d48b7de3"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
class V4DataMigrationOptions(TypedDict):
|
||||
threshold_date: Optional[datetime]
|
||||
|
||||
|
||||
def _data_migration_options(user_input: Optional[Mapping]):
|
||||
options: V4DataMigrationOptions = {
|
||||
"threshold_date": datetime(year=2017, month=1, day=23, tzinfo=timezone.utc),
|
||||
}
|
||||
|
||||
if user_input is None:
|
||||
return options
|
||||
|
||||
if not isinstance(user_input, dict):
|
||||
raise TypeError("v4 migration: data migration options should be a dict object")
|
||||
|
||||
threshold_date = user_input.get("threshold_date")
|
||||
if threshold_date is not None and not isinstance(threshold_date, datetime):
|
||||
raise ValueError(
|
||||
"v4 migration: threshold_date should be None or a datetime.datetime object"
|
||||
)
|
||||
options["threshold_date"] = threshold_date
|
||||
|
||||
return options
|
||||
|
||||
|
||||
def upgrade(
|
||||
*,
|
||||
data_migration: bool = True,
|
||||
data_migration_options: Optional[V4DataMigrationOptions] = None,
|
||||
) -> None:
|
||||
data_migration_options = _data_migration_options(data_migration_options)
|
||||
threshold_date = data_migration_options["threshold_date"]
|
||||
|
||||
op.create_table(
|
||||
"difficulties",
|
||||
sa.Column("song_id", sa.TEXT(), nullable=False),
|
||||
sa.Column("rating_class", sa.Integer(), nullable=False),
|
||||
sa.Column("rating", sa.Integer(), nullable=False),
|
||||
sa.Column("rating_plus", sa.Boolean(), nullable=False),
|
||||
sa.Column("chart_designer", sa.TEXT(), nullable=True),
|
||||
sa.Column("jacket_desginer", sa.TEXT(), nullable=True),
|
||||
sa.Column("audio_override", sa.Boolean(), nullable=False),
|
||||
sa.Column("jacket_override", sa.Boolean(), nullable=False),
|
||||
sa.Column("jacket_night", sa.TEXT(), nullable=True),
|
||||
sa.Column("title", sa.TEXT(), nullable=True),
|
||||
sa.Column("artist", sa.TEXT(), nullable=True),
|
||||
sa.Column("bg", sa.TEXT(), nullable=True),
|
||||
sa.Column("bg_inverse", sa.TEXT(), nullable=True),
|
||||
sa.Column("bpm", sa.TEXT(), nullable=True),
|
||||
sa.Column("bpm_base", sa.Float(), nullable=True),
|
||||
sa.Column("version", sa.TEXT(), nullable=True),
|
||||
sa.Column("date", sa.Integer(), nullable=True),
|
||||
sa.PrimaryKeyConstraint("song_id", "rating_class", name="pk_difficulties"),
|
||||
)
|
||||
op.create_table(
|
||||
"packs",
|
||||
sa.Column("id", sa.TEXT(), nullable=False),
|
||||
sa.Column("name", sa.TEXT(), nullable=False),
|
||||
sa.Column("description", sa.TEXT(), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id", name="fk_packs"),
|
||||
)
|
||||
op.create_table(
|
||||
"songs",
|
||||
sa.Column("idx", sa.Integer(), nullable=False),
|
||||
sa.Column("id", sa.TEXT(), nullable=False),
|
||||
sa.Column("title", sa.TEXT(), nullable=False),
|
||||
sa.Column("artist", sa.TEXT(), nullable=False),
|
||||
sa.Column("set", sa.TEXT(), nullable=False),
|
||||
sa.Column("bpm", sa.TEXT(), nullable=True),
|
||||
sa.Column("bpm_base", sa.Float(), nullable=True),
|
||||
sa.Column("audio_preview", sa.Integer(), nullable=True),
|
||||
sa.Column("audio_preview_end", sa.Integer(), nullable=True),
|
||||
sa.Column("side", sa.Integer(), nullable=True),
|
||||
sa.Column("version", sa.TEXT(), nullable=True),
|
||||
sa.Column("date", sa.Integer(), nullable=True),
|
||||
sa.Column("bg", sa.TEXT(), nullable=True),
|
||||
sa.Column("bg_inverse", sa.TEXT(), nullable=True),
|
||||
sa.Column("bg_day", sa.TEXT(), nullable=True),
|
||||
sa.Column("bg_night", sa.TEXT(), nullable=True),
|
||||
sa.Column("source", sa.TEXT(), nullable=True),
|
||||
sa.Column("source_copyright", sa.TEXT(), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id", name="songs"),
|
||||
)
|
||||
op.create_table(
|
||||
"charts_info",
|
||||
sa.Column("song_id", sa.TEXT(), nullable=False),
|
||||
sa.Column("rating_class", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"constant",
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
comment="real_constant * 10. For example, Crimson Throne [FTR] is 10.4, then store 104.",
|
||||
),
|
||||
sa.Column("notes", sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["rating_class"],
|
||||
["difficulties.rating_class"],
|
||||
name="fk_charts_info_rating_class_difficulties",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["song_id"],
|
||||
["difficulties.song_id"],
|
||||
name="fk_charts_info_song_id_difficulties",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("song_id", "rating_class", name="pk_charts_info"),
|
||||
)
|
||||
op.create_table(
|
||||
"difficulties_localized",
|
||||
sa.Column("song_id", sa.TEXT(), nullable=False),
|
||||
sa.Column("rating_class", sa.Integer(), nullable=False),
|
||||
sa.Column("title_ja", sa.TEXT(), nullable=True),
|
||||
sa.Column("title_ko", sa.TEXT(), nullable=True),
|
||||
sa.Column("title_zh_hans", sa.TEXT(), nullable=True),
|
||||
sa.Column("title_zh_hant", sa.TEXT(), nullable=True),
|
||||
sa.Column("artist_ja", sa.TEXT(), nullable=True),
|
||||
sa.Column("artist_ko", sa.TEXT(), nullable=True),
|
||||
sa.Column("artist_zh_hans", sa.TEXT(), nullable=True),
|
||||
sa.Column("artist_zh_hant", sa.TEXT(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["rating_class"],
|
||||
["difficulties.rating_class"],
|
||||
name="fk_difficulties_localized_rating_class_difficulties",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["song_id"],
|
||||
["difficulties.song_id"],
|
||||
name="fk_difficulties_localized_song_id_difficulties",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"song_id", "rating_class", name="pk_difficulties_localized"
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"packs_localized",
|
||||
sa.Column("id", sa.TEXT(), nullable=False),
|
||||
sa.Column("name_ja", sa.TEXT(), nullable=True),
|
||||
sa.Column("name_ko", sa.TEXT(), nullable=True),
|
||||
sa.Column("name_zh_hans", sa.TEXT(), nullable=True),
|
||||
sa.Column("name_zh_hant", sa.TEXT(), nullable=True),
|
||||
sa.Column("description_ja", sa.TEXT(), nullable=True),
|
||||
sa.Column("description_ko", sa.TEXT(), nullable=True),
|
||||
sa.Column("description_zh_hans", sa.TEXT(), nullable=True),
|
||||
sa.Column("description_zh_hant", sa.TEXT(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["id"],
|
||||
["packs.id"],
|
||||
name="fk_packs_localized_id_packs",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_packs_localized"),
|
||||
)
|
||||
op.create_table(
|
||||
"songs_localized",
|
||||
sa.Column("id", sa.TEXT(), nullable=False),
|
||||
sa.Column("title_ja", sa.TEXT(), nullable=True),
|
||||
sa.Column("title_ko", sa.TEXT(), nullable=True),
|
||||
sa.Column("title_zh_hans", sa.TEXT(), nullable=True),
|
||||
sa.Column("title_zh_hant", sa.TEXT(), nullable=True),
|
||||
sa.Column("search_title_ja", sa.TEXT(), nullable=True, comment="JSON array"),
|
||||
sa.Column("search_title_ko", sa.TEXT(), nullable=True, comment="JSON array"),
|
||||
sa.Column(
|
||||
"search_title_zh_hans", sa.TEXT(), nullable=True, comment="JSON array"
|
||||
),
|
||||
sa.Column(
|
||||
"search_title_zh_hant", sa.TEXT(), nullable=True, comment="JSON array"
|
||||
),
|
||||
sa.Column("search_artist_ja", sa.TEXT(), nullable=True, comment="JSON array"),
|
||||
sa.Column("search_artist_ko", sa.TEXT(), nullable=True, comment="JSON array"),
|
||||
sa.Column(
|
||||
"search_artist_zh_hans", sa.TEXT(), nullable=True, comment="JSON array"
|
||||
),
|
||||
sa.Column(
|
||||
"search_artist_zh_hant", sa.TEXT(), nullable=True, comment="JSON array"
|
||||
),
|
||||
sa.Column("source_ja", sa.TEXT(), nullable=True),
|
||||
sa.Column("source_ko", sa.TEXT(), nullable=True),
|
||||
sa.Column("source_zh_hans", sa.TEXT(), nullable=True),
|
||||
sa.Column("source_zh_hant", sa.TEXT(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["id"],
|
||||
["songs.id"],
|
||||
name="fk_songs_localized_id_songs",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_songs_localized"),
|
||||
)
|
||||
op.drop_table("aliases")
|
||||
op.drop_table("packages")
|
||||
op.execute(sa.text("DROP VIEW IF EXISTS bests"))
|
||||
op.execute(sa.text("DROP VIEW IF EXISTS calculated"))
|
||||
op.execute(sa.text("DROP VIEW IF EXISTS calculated_potential"))
|
||||
op.execute(sa.text("DROP VIEW IF EXISTS song_id_names"))
|
||||
|
||||
op.rename_table("scores", "scores_old")
|
||||
scores_tbl = op.create_table(
|
||||
"scores",
|
||||
sa.Column("id", sa.Integer(), autoincrement=True, primary_key=True),
|
||||
sa.Column("song_id", sa.TEXT(), nullable=False),
|
||||
sa.Column("rating_class", sa.Integer(), nullable=False),
|
||||
sa.Column("score", sa.Integer(), nullable=False),
|
||||
sa.Column("pure", sa.Integer()),
|
||||
sa.Column("far", sa.Integer()),
|
||||
sa.Column("lost", sa.Integer()),
|
||||
sa.Column("date", sa.Integer()),
|
||||
sa.Column("max_recall", sa.Integer()),
|
||||
sa.Column("modifier", sa.Integer(), comment="0: NORMAL, 1: EASY, 2: HARD"),
|
||||
sa.Column(
|
||||
"clear_type",
|
||||
sa.Integer(),
|
||||
comment="0: TRACK LOST, 1: NORMAL CLEAR, 2: FULL RECALL, "
|
||||
"3: PURE MEMORY, 4: EASY CLEAR, 5: HARD CLEAR",
|
||||
),
|
||||
sa.Column("comment", sa.TEXT()),
|
||||
)
|
||||
if data_migration:
|
||||
conn = op.get_bind()
|
||||
query = conn.execute(
|
||||
sa.text(
|
||||
"SELECT id, song_id, rating_class, score, time, pure, far, lost, max_recall, clear_type "
|
||||
"FROM scores_old"
|
||||
)
|
||||
)
|
||||
batch_size = 30
|
||||
|
||||
while True:
|
||||
rows = query.fetchmany(batch_size)
|
||||
if not rows:
|
||||
break
|
||||
|
||||
rows_to_insert = []
|
||||
|
||||
for row in rows:
|
||||
id_ = row[0]
|
||||
song_id = row[1]
|
||||
rating_class = row[2]
|
||||
score = row[3]
|
||||
time = datetime.fromtimestamp(row[4]).astimezone(tz=timezone.utc)
|
||||
pure = row[5]
|
||||
far = row[6]
|
||||
lost = row[7]
|
||||
max_recall = row[8]
|
||||
clear_type = row[9]
|
||||
|
||||
if threshold_date is not None and time <= threshold_date:
|
||||
time = None
|
||||
|
||||
rows_to_insert.append(
|
||||
{
|
||||
"id": id_,
|
||||
"song_id": song_id,
|
||||
"rating_class": rating_class,
|
||||
"score": score,
|
||||
"date": time,
|
||||
"pure": pure,
|
||||
"far": far,
|
||||
"lost": lost,
|
||||
"max_recall": max_recall,
|
||||
"clear_type": clear_type,
|
||||
"modifier": None,
|
||||
"comment": None,
|
||||
}
|
||||
)
|
||||
|
||||
conn.execute(sa.insert(scores_tbl), rows_to_insert)
|
||||
conn.commit()
|
||||
|
||||
op.drop_table("scores_old")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
raise NotImplementedError(
|
||||
f"Downgrade not supported! ({context.get_context().get_current_revision()})"
|
||||
)
|
||||
Reference in New Issue
Block a user