7 Commits

Author SHA1 Message Date
c705fea473 feat: formatter utils 2024-04-03 13:37:23 +08:00
c585e5ec04 feat: score lower limit constants
Add play result score lower limits
2024-04-03 13:36:55 +08:00
09fbebf7a4 refactor: enum naming 2024-04-03 12:51:23 +08:00
bb39a5912b feat: enums 2024-04-03 00:28:08 +08:00
b78040a795 refactor!: sqlalchemy database models 2024-04-02 22:15:21 +08:00
2204338a5e refactor: database base module 2024-04-02 22:02:54 +08:00
55e76ef650 refactor!: remove searcher 2024-04-02 21:54:07 +08:00
24 changed files with 407 additions and 198 deletions

View File

@ -10,10 +10,8 @@ description = "Manage your local Arcaea score database."
readme = "README.md" readme = "README.md"
requires-python = ">=3.8" requires-python = ">=3.8"
dependencies = [ dependencies = [
"beautifulsoup4==4.12.2",
"SQLAlchemy==2.0.20", "SQLAlchemy==2.0.20",
"SQLAlchemy-Utils==0.41.1", "SQLAlchemy-Utils==0.41.1",
"Whoosh==2.7.4",
] ]
classifiers = [ classifiers = [
"Development Status :: 3 - Alpha", "Development Status :: 3 - Alpha",

View File

@ -1,4 +1,2 @@
beautifulsoup4==4.12.2
SQLAlchemy==2.0.20 SQLAlchemy==2.0.20
SQLAlchemy-Utils==0.41.1 SQLAlchemy-Utils==0.41.1
Whoosh==2.7.4

View File

View File

@ -0,0 +1,3 @@
from .clear_type import ArcaeaPlayResultClearType
from .modifier import ArcaeaPlayResultModifier
from .rating_class import ArcaeaRatingClass

View File

@ -0,0 +1,10 @@
from enum import IntEnum
class ArcaeaPlayResultClearType(IntEnum):
TRACK_LOST = 0
NORMAL_CLEAR = 1
FULL_RECALL = 2
PURE_MEMORY = 3
HARD_CLEAR = 4
EASY_CLEAR = 5

View File

@ -0,0 +1,7 @@
from enum import IntEnum
class ArcaeaPlayResultModifier(IntEnum):
NORMAL = 0
EASY = 1
HARD = 2

View File

@ -0,0 +1,9 @@
from enum import IntEnum
class ArcaeaRatingClass(IntEnum):
PAST = 0
PRESENT = 1
FUTURE = 2
BEYOND = 3
ETERNAL = 4

View File

@ -0,0 +1,12 @@
from dataclasses import dataclass
@dataclass(frozen=True)
class ScoreLowerLimits:
EX_PLUS = 9900000
EX = 9800000
AA = 9500000
A = 9200000
B = 8900000
C = 8600000
D = 0

View File

@ -0,0 +1 @@
from .db import Database

View File

@ -5,10 +5,16 @@ from typing import Iterable, List, Optional, Type, Union
from sqlalchemy import Engine, func, inspect, select from sqlalchemy import Engine, func, inspect, select
from sqlalchemy.orm import DeclarativeBase, InstrumentedAttribute, sessionmaker from sqlalchemy.orm import DeclarativeBase, InstrumentedAttribute, sessionmaker
from .external.arcsong.arcsong_json import ArcSongJsonBuilder from arcaea_offline.external.arcsong.arcsong_json import ArcSongJsonBuilder
from .external.exports import ArcaeaOfflineDEFV2_Score, ScoreExport, exporters from arcaea_offline.external.exports import (
from .models.config import ConfigBase, Property ArcaeaOfflineDEFV2_Score,
from .models.scores import ( ScoreExport,
exporters,
)
from arcaea_offline.singleton import Singleton
from .models.v4.config import ConfigBase, Property
from .models.v4.scores import (
CalculatedPotential, CalculatedPotential,
Score, Score,
ScoreBest, ScoreBest,
@ -16,7 +22,7 @@ from .models.scores import (
ScoresBase, ScoresBase,
ScoresViewBase, ScoresViewBase,
) )
from .models.songs import ( from .models.v4.songs import (
Chart, Chart,
ChartInfo, ChartInfo,
Difficulty, Difficulty,
@ -28,7 +34,6 @@ from .models.songs import (
SongsBase, SongsBase,
SongsViewBase, SongsViewBase,
) )
from .singleton import Singleton
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)

View File

@ -1,111 +0,0 @@
from typing import List, Union
from sqlalchemy import select
from sqlalchemy.orm import Session
from whoosh.analysis import NgramFilter, StandardAnalyzer
from whoosh.fields import ID, KEYWORD, TEXT, Schema
from whoosh.filedb.filestore import RamStorage
from whoosh.qparser import FuzzyTermPlugin, MultifieldParser, OrGroup
from .models.songs import Song, SongLocalized
from .utils.search_title import recover_search_title
class Searcher:
def __init__(self):
self.text_analyzer = StandardAnalyzer() | NgramFilter(minsize=2, maxsize=5)
self.song_schema = Schema(
song_id=ID(stored=True, unique=True),
title=TEXT(analyzer=self.text_analyzer, spelling=True),
artist=TEXT(analyzer=self.text_analyzer, spelling=True),
source=TEXT(analyzer=self.text_analyzer, spelling=True),
keywords=KEYWORD(lowercase=True, stored=True, scorable=True),
)
self.storage = RamStorage()
self.index = self.storage.create_index(self.song_schema)
self.default_query_parser = MultifieldParser(
["song_id", "title", "artist", "source", "keywords"],
self.song_schema,
group=OrGroup,
)
self.default_query_parser.add_plugin(FuzzyTermPlugin())
def import_songs(self, session: Session):
writer = self.index.writer()
songs = list(session.scalars(select(Song)))
song_localize_stmt = select(SongLocalized)
for song in songs:
stmt = song_localize_stmt.where(SongLocalized.id == song.id)
sl = session.scalar(stmt)
song_id = song.id
possible_titles: List[Union[str, None]] = [song.title]
possible_artists: List[Union[str, None]] = [song.artist]
possible_sources: List[Union[str, None]] = [song.source]
if sl:
possible_titles.extend(
[sl.title_ja, sl.title_ko, sl.title_zh_hans, sl.title_zh_hant]
)
possible_titles.extend(
recover_search_title(sl.search_title_ja)
+ recover_search_title(sl.search_title_ko)
+ recover_search_title(sl.search_title_zh_hans)
+ recover_search_title(sl.search_title_zh_hant)
)
possible_artists.extend(
recover_search_title(sl.search_artist_ja)
+ recover_search_title(sl.search_artist_ko)
+ recover_search_title(sl.search_artist_zh_hans)
+ recover_search_title(sl.search_artist_zh_hant)
)
possible_sources.extend(
[
sl.source_ja,
sl.source_ko,
sl.source_zh_hans,
sl.source_zh_hant,
]
)
# remove empty items in list
titles = [t for t in possible_titles if t != "" and t is not None]
artists = [t for t in possible_artists if t != "" and t is not None]
sources = [t for t in possible_sources if t != "" and t is not None]
writer.update_document(
song_id=song_id,
title=" ".join(titles),
artist=" ".join(artists),
source=" ".join(sources),
keywords=" ".join([song_id] + titles + artists + sources),
)
writer.commit()
def did_you_mean(self, string: str):
results = set()
with self.index.searcher() as searcher:
corrector_keywords = searcher.corrector("keywords") # type: ignore
corrector_song_id = searcher.corrector("song_id") # type: ignore
corrector_title = searcher.corrector("title") # type: ignore
corrector_artist = searcher.corrector("artist") # type: ignore
corrector_source = searcher.corrector("source") # type: ignore
results.update(corrector_keywords.suggest(string))
results.update(corrector_song_id.suggest(string))
results.update(corrector_title.suggest(string))
results.update(corrector_artist.suggest(string))
results.update(corrector_source.suggest(string))
if string in results:
results.remove(string)
return list(results)
def search(self, string: str, *, limit: int = 10):
query_string = f"{string}"
query = self.default_query_parser.parse(query_string)
with self.index.searcher() as searcher:
results = searcher.search(query, limit=limit)
return [result.get("song_id") for result in results]

View File

@ -0,0 +1,2 @@
from .play_result import PlayResultFormatter
from .rating_class import RatingClassFormatter

View File

@ -0,0 +1,143 @@
from typing import Any, Literal, overload
from arcaea_offline.constants.enums import (
ArcaeaPlayResultClearType,
ArcaeaPlayResultModifier,
)
from arcaea_offline.constants.play_result import ScoreLowerLimits
class PlayResultFormatter:
SCORE_GRADE_FORMAT_RESULTS = Literal["EX+", "EX", "AA", "A", "B", "C", "D"]
@staticmethod
def score_grade(score: int) -> SCORE_GRADE_FORMAT_RESULTS:
"""
Returns the score grade, e.g. EX+.
Raises `ValueError` if the score is negative.
"""
if not isinstance(score, int):
raise TypeError(f"Unsupported type {type(score)}, cannot format")
if score >= ScoreLowerLimits.EX_PLUS:
return "EX+"
elif score >= ScoreLowerLimits.EX:
return "EX"
elif score >= ScoreLowerLimits.AA:
return "AA"
elif score >= ScoreLowerLimits.A:
return "A"
elif score >= ScoreLowerLimits.B:
return "B"
elif score >= ScoreLowerLimits.C:
return "C"
elif score >= ScoreLowerLimits.D:
return "D"
else:
raise ValueError("score cannot be negative")
CLEAR_TYPE_FORMAT_RESULTS = Literal[
"TRACK LOST",
"NORMAL CLEAR",
"FULL RECALL",
"PURE MEMORY",
"EASY CLEAR",
"HARD CLEAR",
"UNKNOWN",
"None",
]
@overload
@classmethod
def clear_type(
cls, clear_type: ArcaeaPlayResultClearType
) -> CLEAR_TYPE_FORMAT_RESULTS:
"""
Returns the uppercased clear type name, e.g. NORMAL CLEAR.
"""
...
@overload
@classmethod
def clear_type(cls, clear_type: int) -> CLEAR_TYPE_FORMAT_RESULTS:
"""
Returns the uppercased clear type name, e.g. NORMAL CLEAR.
The integer will be converted to `ArcaeaPlayResultClearType` enum,
and will return "UNKNOWN" if the convertion fails.
Raises `ValueError` if the integer is negative.
"""
...
@overload
@classmethod
def clear_type(cls, clear_type: None) -> CLEAR_TYPE_FORMAT_RESULTS:
"""
Returns "None"
"""
...
@classmethod
def clear_type(cls, clear_type: Any) -> CLEAR_TYPE_FORMAT_RESULTS:
if clear_type is None:
return "None"
elif isinstance(clear_type, ArcaeaPlayResultClearType):
return clear_type.name.replace("_", " ").upper() # type: ignore
elif isinstance(clear_type, int):
if clear_type < 0:
raise ValueError("clear_type cannot be negative")
try:
return cls.clear_type(ArcaeaPlayResultClearType(clear_type))
except ValueError:
return "UNKNOWN"
else:
raise TypeError(f"Unsupported type {type(clear_type)}, cannot format")
MODIFIER_FORMAT_RESULTS = Literal["NORMAL", "EASY", "HARD", "UNKNOWN", "None"]
@overload
@classmethod
def modifier(cls, modifier: ArcaeaPlayResultModifier) -> MODIFIER_FORMAT_RESULTS:
"""
Returns the uppercased clear type name, e.g. NORMAL CLEAR.
"""
...
@overload
@classmethod
def modifier(cls, modifier: int) -> MODIFIER_FORMAT_RESULTS:
"""
Returns the uppercased clear type name, e.g. NORMAL CLEAR.
The integer will be converted to `ArcaeaPlayResultModifier` enum,
and will return "UNKNOWN" if the convertion fails.
Raises `ValueError` if the integer is negative.
"""
...
@overload
@classmethod
def modifier(cls, modifier: None) -> MODIFIER_FORMAT_RESULTS:
"""
Returns "None"
"""
...
@classmethod
def modifier(cls, modifier: Any) -> MODIFIER_FORMAT_RESULTS:
if modifier is None:
return "None"
elif isinstance(modifier, ArcaeaPlayResultModifier):
return modifier.name
elif isinstance(modifier, int):
if modifier < 0:
raise ValueError("modifier cannot be negative")
try:
return cls.modifier(ArcaeaPlayResultModifier(modifier))
except ValueError:
return "UNKNOWN"
else:
raise TypeError(f"Unsupported type {type(modifier)}, cannot format")

View File

@ -0,0 +1,83 @@
from typing import Any, Literal, overload
from arcaea_offline.constants.enums import ArcaeaRatingClass
class RatingClassFormatter:
abbreviations = {
ArcaeaRatingClass.PAST: "PST",
ArcaeaRatingClass.PRESENT: "PRS",
ArcaeaRatingClass.FUTURE: "FTR",
ArcaeaRatingClass.BEYOND: "BYD",
ArcaeaRatingClass.ETERNAL: "ETR",
}
NAME_FORMAT_RESULTS = Literal[
"Past", "Present", "Future", "Beyond", "Eternal", "Unknown"
]
@overload
@classmethod
def name(cls, rating_class: ArcaeaRatingClass) -> NAME_FORMAT_RESULTS:
"""
Returns the capitalized rating class name, e.g. Future.
"""
...
@overload
@classmethod
def name(cls, rating_class: int) -> NAME_FORMAT_RESULTS:
"""
Returns the capitalized rating class name, e.g. Future.
The integer will be converted to `ArcaeaRatingClass` enum,
and will return "Unknown" if the convertion fails.
"""
...
@classmethod
def name(cls, rating_class: Any) -> NAME_FORMAT_RESULTS:
if isinstance(rating_class, ArcaeaRatingClass):
return rating_class.name.lower().capitalize() # type: ignore
elif isinstance(rating_class, int):
try:
return cls.name(ArcaeaRatingClass(rating_class))
except ValueError:
return "Unknown"
else:
raise TypeError(f"Unsupported type: {type(rating_class)}, cannot format")
ABBREVIATION_FORMAT_RESULTS = Literal["PST", "PRS", "FTR", "BYD", "ETR", "UNK"]
@overload
@classmethod
def abbreviation(
cls, rating_class: ArcaeaRatingClass
) -> ABBREVIATION_FORMAT_RESULTS:
"""
Returns the uppercased rating class name, e.g. FTR.
"""
...
@overload
@classmethod
def abbreviation(cls, rating_class: int) -> ABBREVIATION_FORMAT_RESULTS:
"""
Returns the uppercased rating class name, e.g. FTR.
The integer will be converted to `ArcaeaRatingClass` enum,
and will return "UNK" if the convertion fails.
"""
...
@classmethod
def abbreviation(cls, rating_class: Any) -> ABBREVIATION_FORMAT_RESULTS:
if isinstance(rating_class, ArcaeaRatingClass):
return cls.abbreviations[rating_class] # type: ignore
elif isinstance(rating_class, int):
try:
return cls.abbreviation(ArcaeaRatingClass(rating_class))
except ValueError:
return "UNK"
else:
raise TypeError(f"Unsupported type: {type(rating_class)}, cannot format")

View File

@ -1,25 +0,0 @@
from typing import Optional
RATING_CLASS_TEXT_MAP = {
0: "Past",
1: "Present",
2: "Future",
3: "Beyond",
4: "Eternal",
}
RATING_CLASS_SHORT_TEXT_MAP = {
0: "PST",
1: "PRS",
2: "FTR",
3: "BYD",
4: "ETR",
}
def rating_class_to_text(rating_class: int) -> Optional[str]:
return RATING_CLASS_TEXT_MAP.get(rating_class)
def rating_class_to_short_text(rating_class: int) -> Optional[str]:
return RATING_CLASS_SHORT_TEXT_MAP.get(rating_class)

View File

@ -1,46 +0,0 @@
from typing import Any, Sequence
SCORE_GRADE_FLOOR = [9900000, 9800000, 9500000, 9200000, 8900000, 8600000, 0]
SCORE_GRADE_TEXTS = ["EX+", "EX", "AA", "A", "B", "C", "D"]
MODIFIER_TEXTS = ["NORMAL", "EASY", "HARD"]
CLEAR_TYPE_TEXTS = [
"TRACK LOST",
"NORMAL CLEAR",
"FULL RECALL",
"PURE MEMORY",
"EASY CLEAR",
"HARD CLEAR",
]
def zip_score_grade(score: int, __seq: Sequence, default: Any = "__PRESERVE__"):
"""
zip_score_grade is a simple wrapper that equals to:
```py
for score_floor, val in zip(SCORE_GRADE_FLOOR, __seq):
if score >= score_floor:
return val
return seq[-1] if default == "__PRESERVE__" else default
```
Could be useful in specific cases.
"""
return next(
(
val
for score_floor, val in zip(SCORE_GRADE_FLOOR, __seq)
if score >= score_floor
),
__seq[-1] if default == "__PRESERVE__" else default,
)
def score_to_grade_text(score: int) -> str:
return zip_score_grade(score, SCORE_GRADE_TEXTS)
def modifier_to_text(modifier: int) -> str:
return MODIFIER_TEXTS[modifier]
def clear_type_to_text(clear_type: int) -> str:
return CLEAR_TYPE_TEXTS[clear_type]

View File

@ -1,6 +0,0 @@
import json
from typing import List, Optional
def recover_search_title(db_value: Optional[str]) -> List[str]:
return json.loads(db_value) if db_value else []

View File

@ -0,0 +1,126 @@
import pytest
from arcaea_offline.constants.enums import (
ArcaeaPlayResultClearType,
ArcaeaPlayResultModifier,
ArcaeaRatingClass,
)
from arcaea_offline.utils.formatters.play_result import PlayResultFormatter
from arcaea_offline.utils.formatters.rating_class import RatingClassFormatter
class TestRatingClassFormatter:
def test_name(self):
assert RatingClassFormatter.name(ArcaeaRatingClass.PAST) == "Past"
assert RatingClassFormatter.name(ArcaeaRatingClass.PRESENT) == "Present"
assert RatingClassFormatter.name(ArcaeaRatingClass.FUTURE) == "Future"
assert RatingClassFormatter.name(ArcaeaRatingClass.BEYOND) == "Beyond"
assert RatingClassFormatter.name(ArcaeaRatingClass.ETERNAL) == "Eternal"
assert RatingClassFormatter.name(2) == "Future"
assert RatingClassFormatter.name(100) == "Unknown"
assert RatingClassFormatter.name(-1) == "Unknown"
pytest.raises(TypeError, RatingClassFormatter.name, "2")
pytest.raises(TypeError, RatingClassFormatter.name, [])
pytest.raises(TypeError, RatingClassFormatter.name, None)
def test_abbreviation(self):
assert RatingClassFormatter.abbreviation(ArcaeaRatingClass.PAST) == "PST"
assert RatingClassFormatter.abbreviation(ArcaeaRatingClass.PRESENT) == "PRS"
assert RatingClassFormatter.abbreviation(ArcaeaRatingClass.FUTURE) == "FTR"
assert RatingClassFormatter.abbreviation(ArcaeaRatingClass.BEYOND) == "BYD"
assert RatingClassFormatter.abbreviation(ArcaeaRatingClass.ETERNAL) == "ETR"
assert RatingClassFormatter.abbreviation(2) == "FTR"
assert RatingClassFormatter.abbreviation(100) == "UNK"
assert RatingClassFormatter.abbreviation(-1) == "UNK"
pytest.raises(TypeError, RatingClassFormatter.abbreviation, "2")
pytest.raises(TypeError, RatingClassFormatter.abbreviation, [])
pytest.raises(TypeError, RatingClassFormatter.abbreviation, None)
class TestPlayResultFormatter:
def test_score_grade(self):
assert PlayResultFormatter.score_grade(10001284) == "EX+"
assert PlayResultFormatter.score_grade(9989210) == "EX+"
assert PlayResultFormatter.score_grade(9900000) == "EX+"
assert PlayResultFormatter.score_grade(9899999) == "EX"
assert PlayResultFormatter.score_grade(9843717) == "EX"
assert PlayResultFormatter.score_grade(9800000) == "EX"
assert PlayResultFormatter.score_grade(9799999) == "AA"
assert PlayResultFormatter.score_grade(9794015) == "AA"
assert PlayResultFormatter.score_grade(9750000) == "AA"
assert PlayResultFormatter.score_grade(9499999) == "A"
assert PlayResultFormatter.score_grade(9356855) == "A"
assert PlayResultFormatter.score_grade(9200000) == "A"
assert PlayResultFormatter.score_grade(9199999) == "B"
assert PlayResultFormatter.score_grade(9065785) == "B"
assert PlayResultFormatter.score_grade(8900000) == "B"
assert PlayResultFormatter.score_grade(8899999) == "C"
assert PlayResultFormatter.score_grade(8756211) == "C"
assert PlayResultFormatter.score_grade(8600000) == "C"
assert PlayResultFormatter.score_grade(8599999) == "D"
assert PlayResultFormatter.score_grade(5500000) == "D"
assert PlayResultFormatter.score_grade(0) == "D"
pytest.raises(ValueError, PlayResultFormatter.score_grade, -1)
pytest.raises(TypeError, PlayResultFormatter.score_grade, "10001284")
pytest.raises(TypeError, PlayResultFormatter.score_grade, [])
pytest.raises(TypeError, PlayResultFormatter.score_grade, None)
def test_clear_type(self):
assert (
PlayResultFormatter.clear_type(ArcaeaPlayResultClearType.TRACK_LOST)
== "TRACK LOST"
)
assert (
PlayResultFormatter.clear_type(ArcaeaPlayResultClearType.NORMAL_CLEAR)
== "NORMAL CLEAR"
)
assert (
PlayResultFormatter.clear_type(ArcaeaPlayResultClearType.FULL_RECALL)
== "FULL RECALL"
)
assert (
PlayResultFormatter.clear_type(ArcaeaPlayResultClearType.PURE_MEMORY)
== "PURE MEMORY"
)
assert (
PlayResultFormatter.clear_type(ArcaeaPlayResultClearType.EASY_CLEAR)
== "EASY CLEAR"
)
assert (
PlayResultFormatter.clear_type(ArcaeaPlayResultClearType.HARD_CLEAR)
== "HARD CLEAR"
)
assert PlayResultFormatter.clear_type(None) == "None"
assert PlayResultFormatter.clear_type(1) == "NORMAL CLEAR"
assert PlayResultFormatter.clear_type(6) == "UNKNOWN"
pytest.raises(ValueError, PlayResultFormatter.clear_type, -1)
pytest.raises(TypeError, PlayResultFormatter.clear_type, "1")
pytest.raises(TypeError, PlayResultFormatter.clear_type, [])
def test_modifier(self):
assert PlayResultFormatter.modifier(ArcaeaPlayResultModifier.NORMAL) == "NORMAL"
assert PlayResultFormatter.modifier(ArcaeaPlayResultModifier.EASY) == "EASY"
assert PlayResultFormatter.modifier(ArcaeaPlayResultModifier.HARD) == "HARD"
assert PlayResultFormatter.modifier(None) == "None"
assert PlayResultFormatter.modifier(1) == "EASY"
assert PlayResultFormatter.modifier(6) == "UNKNOWN"
pytest.raises(ValueError, PlayResultFormatter.modifier, -1)
pytest.raises(TypeError, PlayResultFormatter.modifier, "1")
pytest.raises(TypeError, PlayResultFormatter.modifier, [])