4 Commits

Author SHA1 Message Date
bb39a5912b feat: enums 2024-04-03 00:28:08 +08:00
b78040a795 refactor!: sqlalchemy database models 2024-04-02 22:15:21 +08:00
2204338a5e refactor: database base module 2024-04-02 22:02:54 +08:00
55e76ef650 refactor!: remove searcher 2024-04-02 21:54:07 +08:00
22 changed files with 92 additions and 206 deletions

View File

@ -1,40 +0,0 @@
name: test & lint
on:
push:
branches:
- '*'
pull_request:
types: [opened, reopened]
workflow_dispatch:
jobs:
pytest:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install dev dependencies
run: 'pip install .[dev]'
- name: Run tests
run: 'pytest -v'
ruff:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install dev dependencies
run: 'pip install .[dev]'
- name: Run linter
run: 'ruff check'

23
.github/workflows/test.yml vendored Normal file
View File

@ -0,0 +1,23 @@
name: Run tests
on:
push:
branches:
- 'master'
pull_request:
types: [opened, reopened]
workflow_dispatch:
jobs:
pytest:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.8', '3.9', '3.10', '3.11']
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- run: 'pip install -r requirements.dev.txt .'
- run: 'pytest -v'

View File

@ -4,10 +4,11 @@ repos:
hooks:
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.4.4
- repo: https://github.com/psf/black
rev: 23.1.0
hooks:
- id: ruff
args: ["--fix"]
- id: ruff-format
- id: black
- repo: https://github.com/PyCQA/isort
rev: 5.12.0
hooks:
- id: isort

View File

@ -10,14 +10,9 @@ English | [简体中文](./README.zh_Hans.md)
## WIP
> [!CAUTION]
> **Warning**
> This project is under active development, thus it is unstable and API may change frequently.
> [!IMPORTANT]
> v0.3.0 is under development, check out [this branch](https://github.com/283375/arcaea-offline/tree/0.3.0-refactor)!
>
> Once v0.3.0 is ready for release, this repository will be transferred to *[ArcaeaOffline](https://github.com/ArcaeaOffline)/core-python*
## What is this
This is the core library of `Arcaea Offline`, designed to manage player scores, calculate their potential, and provide various useful tools.

View File

@ -8,14 +8,9 @@
## WIP
> [!CAUTION]
> **Warning**
> 该项目正处于早期开发阶段,不能保证稳定性,且 API 可能随时变动。
> [!IMPORTANT]
> v0.3.0 正在[此分支](https://github.com/283375/arcaea-offline/tree/0.3.0-refactor)下开发!
>
> 在 v0.3.0 准备好发布后,此存储库将被迁移至 *[ArcaeaOffline](https://github.com/ArcaeaOffline)/core-python*。
## 这是什么?
这是 `Arcaea Offline` 的核心依赖库,用于维护分数数据库、计算潜力值,并提供一些实用工具。

View File

@ -10,44 +10,32 @@ description = "Manage your local Arcaea score database."
readme = "README.md"
requires-python = ">=3.8"
dependencies = [
"beautifulsoup4==4.12.2",
"SQLAlchemy==2.0.20",
"SQLAlchemy-Utils==0.41.1",
"Whoosh==2.7.4",
]
classifiers = [
"Development Status :: 3 - Alpha",
"Programming Language :: Python :: 3",
]
[project.optional-dependencies]
dev = ["ruff~=0.4", "pre-commit~=3.3", "pytest~=7.4", "tox~=4.11"]
[project.urls]
"Homepage" = "https://github.com/283375/arcaea-offline"
"Bug Tracker" = "https://github.com/283375/arcaea-offline/issues"
[tool.isort]
profile = "black"
src_paths = ["src/arcaea_offline"]
[tool.pyright]
ignore = ["build/"]
[tool.ruff.lint]
# Full list: https://docs.astral.sh/ruff/rules
select = [
"E", # pycodestyle (Error)
"W", # pycodestyle (Warning)
"F", # pyflakes
"I", # isort
"PL", # pylint
"N", # pep8-naming
"FBT", # flake8-boolean-trap
"A", # flake8-builtins
"DTZ", # flake8-datetimez
"LOG", # flake8-logging
"Q", # flake8-quotes
"G", # flake8-logging-format
"PIE", # flake8-pie
"PT", # flake8-pytest-style
]
ignore = [
"E501", # line-too-long
[tool.pylint.main]
jobs = 0
[tool.pylint.logging]
disable = [
"missing-module-docstring",
"missing-class-docstring",
"missing-function-docstring",
"not-callable", # false positive to sqlalchemy `func.*`, remove this when pylint-dev/pylint(#8138) closed
]

View File

@ -1,4 +1,6 @@
ruff~=0.4
pre-commit~=3.3
pytest~=7.4
tox~=4.11
black==23.3.0
isort==5.12.0
pre-commit==3.3.1
pylint==3.0.2
pytest==7.4.3
tox==4.11.3

View File

@ -1,4 +1,2 @@
beautifulsoup4==4.12.2
SQLAlchemy==2.0.20
SQLAlchemy-Utils==0.41.1
Whoosh==2.7.4

View File

View File

@ -0,0 +1,3 @@
from .clear_type import ArcaeaScoreClearType
from .modifier import ArcaeaScoreModifier
from .rating_class import ArcaeaRatingClass

View File

@ -0,0 +1,10 @@
from enum import IntEnum
class ArcaeaScoreClearType(IntEnum):
TRACK_LOST = 0
NORMAL_CLEAR = 1
FULL_RECALL = 2
PURE_MEMORY = 3
HARD_CLEAR = 4
EASY_CLEAR = 5

View File

@ -0,0 +1,7 @@
from enum import IntEnum
class ArcaeaScoreModifier(IntEnum):
NORMAL = 0
EASY = 1
HARD = 2

View File

@ -0,0 +1,9 @@
from enum import IntEnum
class ArcaeaRatingClass(IntEnum):
PAST = 0
PRESENT = 1
FUTURE = 2
BEYOND = 3
ETERNAL = 4

View File

@ -0,0 +1 @@
from .db import Database

View File

@ -5,10 +5,16 @@ from typing import Iterable, List, Optional, Type, Union
from sqlalchemy import Engine, func, inspect, select
from sqlalchemy.orm import DeclarativeBase, InstrumentedAttribute, sessionmaker
from .external.arcsong.arcsong_json import ArcSongJsonBuilder
from .external.exports import ArcaeaOfflineDEFV2_Score, ScoreExport, exporters
from .models.config import ConfigBase, Property
from .models.scores import (
from arcaea_offline.external.arcsong.arcsong_json import ArcSongJsonBuilder
from arcaea_offline.external.exports import (
ArcaeaOfflineDEFV2_Score,
ScoreExport,
exporters,
)
from arcaea_offline.singleton import Singleton
from .models.v4.config import ConfigBase, Property
from .models.v4.scores import (
CalculatedPotential,
Score,
ScoreBest,
@ -16,7 +22,7 @@ from .models.scores import (
ScoresBase,
ScoresViewBase,
)
from .models.songs import (
from .models.v4.songs import (
Chart,
ChartInfo,
Difficulty,
@ -28,7 +34,6 @@ from .models.songs import (
SongsBase,
SongsViewBase,
)
from .singleton import Singleton
logger = logging.getLogger(__name__)

View File

@ -1,111 +0,0 @@
from typing import List, Union
from sqlalchemy import select
from sqlalchemy.orm import Session
from whoosh.analysis import NgramFilter, StandardAnalyzer
from whoosh.fields import ID, KEYWORD, TEXT, Schema
from whoosh.filedb.filestore import RamStorage
from whoosh.qparser import FuzzyTermPlugin, MultifieldParser, OrGroup
from .models.songs import Song, SongLocalized
from .utils.search_title import recover_search_title
class Searcher:
def __init__(self):
self.text_analyzer = StandardAnalyzer() | NgramFilter(minsize=2, maxsize=5)
self.song_schema = Schema(
song_id=ID(stored=True, unique=True),
title=TEXT(analyzer=self.text_analyzer, spelling=True),
artist=TEXT(analyzer=self.text_analyzer, spelling=True),
source=TEXT(analyzer=self.text_analyzer, spelling=True),
keywords=KEYWORD(lowercase=True, stored=True, scorable=True),
)
self.storage = RamStorage()
self.index = self.storage.create_index(self.song_schema)
self.default_query_parser = MultifieldParser(
["song_id", "title", "artist", "source", "keywords"],
self.song_schema,
group=OrGroup,
)
self.default_query_parser.add_plugin(FuzzyTermPlugin())
def import_songs(self, session: Session):
writer = self.index.writer()
songs = list(session.scalars(select(Song)))
song_localize_stmt = select(SongLocalized)
for song in songs:
stmt = song_localize_stmt.where(SongLocalized.id == song.id)
sl = session.scalar(stmt)
song_id = song.id
possible_titles: List[Union[str, None]] = [song.title]
possible_artists: List[Union[str, None]] = [song.artist]
possible_sources: List[Union[str, None]] = [song.source]
if sl:
possible_titles.extend(
[sl.title_ja, sl.title_ko, sl.title_zh_hans, sl.title_zh_hant]
)
possible_titles.extend(
recover_search_title(sl.search_title_ja)
+ recover_search_title(sl.search_title_ko)
+ recover_search_title(sl.search_title_zh_hans)
+ recover_search_title(sl.search_title_zh_hant)
)
possible_artists.extend(
recover_search_title(sl.search_artist_ja)
+ recover_search_title(sl.search_artist_ko)
+ recover_search_title(sl.search_artist_zh_hans)
+ recover_search_title(sl.search_artist_zh_hant)
)
possible_sources.extend(
[
sl.source_ja,
sl.source_ko,
sl.source_zh_hans,
sl.source_zh_hant,
]
)
# remove empty items in list
titles = [t for t in possible_titles if t != "" and t is not None]
artists = [t for t in possible_artists if t != "" and t is not None]
sources = [t for t in possible_sources if t != "" and t is not None]
writer.update_document(
song_id=song_id,
title=" ".join(titles),
artist=" ".join(artists),
source=" ".join(sources),
keywords=" ".join([song_id] + titles + artists + sources),
)
writer.commit()
def did_you_mean(self, string: str):
results = set()
with self.index.searcher() as searcher:
corrector_keywords = searcher.corrector("keywords") # type: ignore
corrector_song_id = searcher.corrector("song_id") # type: ignore
corrector_title = searcher.corrector("title") # type: ignore
corrector_artist = searcher.corrector("artist") # type: ignore
corrector_source = searcher.corrector("source") # type: ignore
results.update(corrector_keywords.suggest(string))
results.update(corrector_song_id.suggest(string))
results.update(corrector_title.suggest(string))
results.update(corrector_artist.suggest(string))
results.update(corrector_source.suggest(string))
if string in results:
results.remove(string)
return list(results)
def search(self, string: str, *, limit: int = 10):
query_string = f"{string}"
query = self.default_query_parser.parse(query_string)
with self.index.searcher() as searcher:
results = searcher.search(query, limit=limit)
return [result.get("song_id") for result in results]