From ef1a65adeb72a76eab1799c3cb37a9f9207c6d7e Mon Sep 17 00:00:00 2001 From: Mathias Scheider Date: Fri, 16 Oct 2026 15:27:41 +0200 Subject: [PATCH] some refactoring --- AGENTS.md | 2 +- alembic/env.py | 2 +- src/ingest/b4w.py | 24 ++---------------------- src/ingest/comdirect.py | 4 ++-- src/ingest/{_common.py => common.py} | 23 ++++++++++++++++++++++- src/ingest/transactions.py | 2 +- src/models/b4w.py | 8 ++++---- src/models/category.py | 11 ++++++++--- src/models/category_budget.py | 2 +- src/models/comdirect.py | 6 +++--- src/models/{_common.py => common.py} | 2 +- src/models/transactions.py | 4 ++-- tests/test_helper_db.py | 2 +- 13 files changed, 49 insertions(+), 43 deletions(-) rename src/ingest/{_common.py => common.py} (67%) rename src/models/{_common.py => common.py} (99%) diff --git a/AGENTS.md b/AGENTS.md index a2c0b04..9459b04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ Two sources with shared patterns: - **comdirect**: `ComdirectGiro`, `ComdirectCreditCard`, `ComdirectDepot`, import to `comdirect_giro` table - **unified**: `Transaction`, `TransactionDb` – both ingest pipelines project their staging rows (`import_b4w`, `comdirect_giro`) into one `transactions` table via `src/ingest/transactions.py` (`map_*` mappers + `consolidate_transactions`). Dedup uses a business-key `hash_id` (source + IBAN + value date + amount + text + counterparty) with a UNIQUE constraint and SQLite `INSERT OR IGNORE`, so re-imports are idempotent. Amounts are canonical signed (negative = money out). Depot/credit-card/tagesgeld are not projected yet. -All models inherit from common base in `src/models/_common.py`: +All models inherit from common base in `src/models/common.py`: - `_hash()`: SHA256-based deduplication key - `_parse_date_german()`: Handles "07.09.2026" format and values like "offen", "--" - `_parse_decimal_german()`: Converts "1,23" to Decimal diff --git a/alembic/env.py b/alembic/env.py index de68b9d..16b49d9 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -6,7 +6,7 @@ from sqlalchemy import engine_from_config, pool from alembic import context from src.db import get_set_engine -from src.models._common import BaseModelDb +from src.models.common import BaseModelDb load_dotenv() # reads .env into os.environ diff --git a/src/ingest/b4w.py b/src/ingest/b4w.py index 46e5f76..dc9feba 100644 --- a/src/ingest/b4w.py +++ b/src/ingest/b4w.py @@ -2,10 +2,9 @@ from pathlib import Path from sqlalchemy.orm import Session -from src.ingest._common import _reader_raw, _upsert +from src.ingest.common import _reader_raw, _upsert, upsert_category from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction -from src.models.b4w import B4wDb, B4wExport, Category -from src.models.category import CategoryDb +from src.models.b4w import B4wDb, B4wExport def reader_b4w(path: Path) -> list[B4wExport]: @@ -13,25 +12,6 @@ def reader_b4w(path: Path) -> list[B4wExport]: return list(map(B4wExport.model_validate, raw)) -def upsert_category( - session: Session, category: Category | None, do_commit=True -) -> CategoryDb | None: - if category is None: - return None - parameters = { - "name": category.name, - "parent": upsert_category(session, category.parent), - } - return _upsert( - session, - CategoryDb, - parameters, - parameters, - do_update=False, - do_commit=do_commit, - ) - - def upsert_b4w( session: Session, data: B4wExport, do_update=False, do_commit=True ) -> B4wDb | None: diff --git a/src/ingest/comdirect.py b/src/ingest/comdirect.py index 361ec6f..50879a0 100644 --- a/src/ingest/comdirect.py +++ b/src/ingest/comdirect.py @@ -6,18 +6,18 @@ from typing import cast from sqlalchemy.orm import Session from settings import BANK_ACCOUNTS -from src.ingest._common import _upsert +from src.ingest.common import _upsert from src.ingest.transactions import ( consolidate_transactions, map_comdirect_giro_to_transaction, ) -from src.models._common import Csv2Dict, CsV2DictMulti from src.models.comdirect import ( ComdirectCreditCard, ComdirectDepot, ComdirectGiro, ComdirectGiroDb, ) +from src.models.common import Csv2Dict, CsV2DictMulti from src.models.settings import Account, Source PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$' diff --git a/src/ingest/_common.py b/src/ingest/common.py similarity index 67% rename from src/ingest/_common.py rename to src/ingest/common.py index 906a5b8..b6318f1 100644 --- a/src/ingest/_common.py +++ b/src/ingest/common.py @@ -4,7 +4,9 @@ from typing import Any from sqlalchemy.orm import Session -from src.models._common import BaseModelDb +from src.models import Category +from src.models.category import CategoryDb +from src.models.common import BaseModelDb def _query_existing( @@ -42,3 +44,22 @@ def _upsert[DbModelT: BaseModelDb]( def _reader_raw(path: Path): with path.open() as fid: return list(csv.DictReader(fid)) + + +def upsert_category( + session: Session, category: Category | None, do_commit=True +) -> CategoryDb | None: + if category is None: + return None + parameters = { + "name": category.name, + "parent": upsert_category(session, category.parent), + } + return _upsert( + session, + CategoryDb, + parameters, + parameters, + do_update=False, + do_commit=do_commit, + ) diff --git a/src/ingest/transactions.py b/src/ingest/transactions.py index fad3354..cf6ed99 100644 --- a/src/ingest/transactions.py +++ b/src/ingest/transactions.py @@ -4,9 +4,9 @@ from sqlalchemy import select from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.orm import Session -from src.models._common import BaseModelDb from src.models.b4w import B4wDb from src.models.comdirect import ComdirectGiroDb +from src.models.common import BaseModelDb from src.models.transactions import Transaction, TransactionDb diff --git a/src/models/b4w.py b/src/models/b4w.py index 0017fc1..21d650f 100644 --- a/src/models/b4w.py +++ b/src/models/b4w.py @@ -6,14 +6,14 @@ from sqlalchemy import Date, ForeignKey, Integer, String from sqlalchemy.orm import Mapped, mapped_column, relationship from src.constants import IBAN_PATTERN, Currency -from src.models._common import ( +from src.models.category import Category +from src.models.common import ( ExportBaseModel, IngestdbBase, Sign, - _hash, date2str, + hashify, ) -from src.models.category import Category class B4wExport(ExportBaseModel): @@ -59,7 +59,7 @@ class B4wExport(ExportBaseModel): @computed_field # type: ignore[prop-decorator] @property def hash_id(self) -> str: - return _hash(self.id) + return hashify(self.id) @property def amount_signed(self) -> Decimal: diff --git a/src/models/category.py b/src/models/category.py index b1ffd37..20dd2d9 100644 --- a/src/models/category.py +++ b/src/models/category.py @@ -1,8 +1,8 @@ from pydantic import Field, computed_field from sqlalchemy import ForeignKey, String, UniqueConstraint -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, Session, mapped_column, relationship -from src.models._common import BaseModelDb, ExportBaseModel, _hash +from src.models.common import BaseModelDb, ExportBaseModel, hashify class Category(ExportBaseModel): @@ -12,7 +12,7 @@ class Category(ExportBaseModel): @computed_field # type: ignore[prop-decorator] @property def hash_id(self) -> str: - return _hash(self.name, self.parent) + return hashify(self.name, self.parent) @classmethod def from_str(cls, s: str) -> Category | None: @@ -54,3 +54,8 @@ class CategoryDb(BaseModelDb): ) children = relationship("CategoryDb", back_populates="parent") budgets = relationship("CategoryBudgetDb", back_populates="category") + + +def add_category(session: Session, category_name: str): + category = Category.from_str(category_name) + session.add(category) diff --git a/src/models/category_budget.py b/src/models/category_budget.py index 81ebc3c..14354fc 100644 --- a/src/models/category_budget.py +++ b/src/models/category_budget.py @@ -4,7 +4,7 @@ from decimal import Decimal from sqlalchemy import Date, ForeignKey, Numeric from sqlalchemy.orm import Mapped, mapped_column, relationship -from src.models._common import BaseModelDb +from src.models.common import BaseModelDb class CategoryBudgetDb(BaseModelDb): diff --git a/src/models/comdirect.py b/src/models/comdirect.py index 7ec89cf..0b38cb9 100644 --- a/src/models/comdirect.py +++ b/src/models/comdirect.py @@ -5,14 +5,14 @@ from pydantic import ConfigDict, Field, computed_field, field_validator from sqlalchemy import Date, String from sqlalchemy.orm import Mapped, mapped_column -from src.models._common import ( +from src.models.common import ( ExportBaseModel, IngestdbBase, - _hash, _parse_date_german, _parse_decimal_german, date2str, decimal2str, + hashify, ) @@ -72,7 +72,7 @@ class ComdirectGiro(ComdirectBase): @computed_field # type: ignore[prop-decorator] @property def hash_id(self) -> str: - return _hash(self.date_val, self.date_book, self.text) + return hashify(self.date_val, self.date_book, self.text) @field_validator("date_book", mode="before") @classmethod diff --git a/src/models/_common.py b/src/models/common.py similarity index 99% rename from src/models/_common.py rename to src/models/common.py index b53e0e1..3edd0c7 100644 --- a/src/models/_common.py +++ b/src/models/common.py @@ -8,7 +8,7 @@ from sqlalchemy import Date, Numeric, String from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column -def _hash(*args): +def hashify(*args): raw = "".join(str(args)) return hashlib.sha256(raw.encode()).hexdigest() diff --git a/src/models/transactions.py b/src/models/transactions.py index cf8a142..23fe48b 100644 --- a/src/models/transactions.py +++ b/src/models/transactions.py @@ -5,7 +5,7 @@ from pydantic import Field, computed_field from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column, relationship -from src.models._common import ExportBaseModel, IngestdbBase, _hash, date2str +from src.models.common import ExportBaseModel, IngestdbBase, date2str, hashify class Transaction(ExportBaseModel): @@ -33,7 +33,7 @@ class Transaction(ExportBaseModel): @computed_field # type: ignore[prop-decorator] @property def hash_id(self) -> str: - return _hash( + return hashify( self.source, self.account_iban, self.date_valid, diff --git a/tests/test_helper_db.py b/tests/test_helper_db.py index 3b80037..cfda3a2 100644 --- a/tests/test_helper_db.py +++ b/tests/test_helper_db.py @@ -1,7 +1,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from src.ingest.b4w import upsert_category +from src.ingest.common import upsert_category from src.models import Category from src.models.category import CategoryDb