some refactoring

This commit is contained in:
2026-10-16 15:27:41 +02:00
parent 696823406e
commit ef1a65adeb
13 changed files with 49 additions and 43 deletions
+1 -1
View File
@@ -71,7 +71,7 @@ Two sources with shared patterns:
- **comdirect**: `ComdirectGiro`, `ComdirectCreditCard`, `ComdirectDepot`, import to `comdirect_giro` table
- **unified**: `Transaction`, `TransactionDb` – both ingest pipelines project their staging rows (`import_b4w`, `comdirect_giro`) into one `transactions` table via `src/ingest/transactions.py` (`map_*` mappers + `consolidate_transactions`). Dedup uses a business-key `hash_id` (source + IBAN + value date + amount + text + counterparty) with a UNIQUE constraint and SQLite `INSERT OR IGNORE`, so re-imports are idempotent. Amounts are canonical signed (negative = money out). Depot/credit-card/tagesgeld are not projected yet.
All models inherit from common base in `src/models/_common.py`:
All models inherit from common base in `src/models/common.py`:
- `_hash()`: SHA256-based deduplication key
- `_parse_date_german()`: Handles "07.09.2026" format and values like "offen", "--"
- `_parse_decimal_german()`: Converts "1,23" to Decimal
+1 -1
View File
@@ -6,7 +6,7 @@ from sqlalchemy import engine_from_config, pool
from alembic import context
from src.db import get_set_engine
from src.models._common import BaseModelDb
from src.models.common import BaseModelDb
load_dotenv() # reads .env into os.environ
+2 -22
View File
@@ -2,10 +2,9 @@ from pathlib import Path
from sqlalchemy.orm import Session
from src.ingest._common import _reader_raw, _upsert
from src.ingest.common import _reader_raw, _upsert, upsert_category
from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
from src.models.b4w import B4wDb, B4wExport, Category
from src.models.category import CategoryDb
from src.models.b4w import B4wDb, B4wExport
def reader_b4w(path: Path) -> list[B4wExport]:
@@ -13,25 +12,6 @@ def reader_b4w(path: Path) -> list[B4wExport]:
return list(map(B4wExport.model_validate, raw))
def upsert_category(
session: Session, category: Category | None, do_commit=True
) -> CategoryDb | None:
if category is None:
return None
parameters = {
"name": category.name,
"parent": upsert_category(session, category.parent),
}
return _upsert(
session,
CategoryDb,
parameters,
parameters,
do_update=False,
do_commit=do_commit,
)
def upsert_b4w(
session: Session, data: B4wExport, do_update=False, do_commit=True
) -> B4wDb | None:
+2 -2
View File
@@ -6,18 +6,18 @@ from typing import cast
from sqlalchemy.orm import Session
from settings import BANK_ACCOUNTS
from src.ingest._common import _upsert
from src.ingest.common import _upsert
from src.ingest.transactions import (
consolidate_transactions,
map_comdirect_giro_to_transaction,
)
from src.models._common import Csv2Dict, CsV2DictMulti
from src.models.comdirect import (
ComdirectCreditCard,
ComdirectDepot,
ComdirectGiro,
ComdirectGiroDb,
)
from src.models.common import Csv2Dict, CsV2DictMulti
from src.models.settings import Account, Source
PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$'
+22 -1
View File
@@ -4,7 +4,9 @@ from typing import Any
from sqlalchemy.orm import Session
from src.models._common import BaseModelDb
from src.models import Category
from src.models.category import CategoryDb
from src.models.common import BaseModelDb
def _query_existing(
@@ -42,3 +44,22 @@ def _upsert[DbModelT: BaseModelDb](
def _reader_raw(path: Path):
with path.open() as fid:
return list(csv.DictReader(fid))
def upsert_category(
session: Session, category: Category | None, do_commit=True
) -> CategoryDb | None:
if category is None:
return None
parameters = {
"name": category.name,
"parent": upsert_category(session, category.parent),
}
return _upsert(
session,
CategoryDb,
parameters,
parameters,
do_update=False,
do_commit=do_commit,
)
+1 -1
View File
@@ -4,9 +4,9 @@ from sqlalchemy import select
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
from sqlalchemy.orm import Session
from src.models._common import BaseModelDb
from src.models.b4w import B4wDb
from src.models.comdirect import ComdirectGiroDb
from src.models.common import BaseModelDb
from src.models.transactions import Transaction, TransactionDb
+4 -4
View File
@@ -6,14 +6,14 @@ from sqlalchemy import Date, ForeignKey, Integer, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.constants import IBAN_PATTERN, Currency
from src.models._common import (
from src.models.category import Category
from src.models.common import (
ExportBaseModel,
IngestdbBase,
Sign,
_hash,
date2str,
hashify,
)
from src.models.category import Category
class B4wExport(ExportBaseModel):
@@ -59,7 +59,7 @@ class B4wExport(ExportBaseModel):
@computed_field # type: ignore[prop-decorator]
@property
def hash_id(self) -> str:
return _hash(self.id)
return hashify(self.id)
@property
def amount_signed(self) -> Decimal:
+8 -3
View File
@@ -1,8 +1,8 @@
from pydantic import Field, computed_field
from sqlalchemy import ForeignKey, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.orm import Mapped, Session, mapped_column, relationship
from src.models._common import BaseModelDb, ExportBaseModel, _hash
from src.models.common import BaseModelDb, ExportBaseModel, hashify
class Category(ExportBaseModel):
@@ -12,7 +12,7 @@ class Category(ExportBaseModel):
@computed_field # type: ignore[prop-decorator]
@property
def hash_id(self) -> str:
return _hash(self.name, self.parent)
return hashify(self.name, self.parent)
@classmethod
def from_str(cls, s: str) -> Category | None:
@@ -54,3 +54,8 @@ class CategoryDb(BaseModelDb):
)
children = relationship("CategoryDb", back_populates="parent")
budgets = relationship("CategoryBudgetDb", back_populates="category")
def add_category(session: Session, category_name: str):
category = Category.from_str(category_name)
session.add(category)
+1 -1
View File
@@ -4,7 +4,7 @@ from decimal import Decimal
from sqlalchemy import Date, ForeignKey, Numeric
from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models._common import BaseModelDb
from src.models.common import BaseModelDb
class CategoryBudgetDb(BaseModelDb):
+3 -3
View File
@@ -5,14 +5,14 @@ from pydantic import ConfigDict, Field, computed_field, field_validator
from sqlalchemy import Date, String
from sqlalchemy.orm import Mapped, mapped_column
from src.models._common import (
from src.models.common import (
ExportBaseModel,
IngestdbBase,
_hash,
_parse_date_german,
_parse_decimal_german,
date2str,
decimal2str,
hashify,
)
@@ -72,7 +72,7 @@ class ComdirectGiro(ComdirectBase):
@computed_field # type: ignore[prop-decorator]
@property
def hash_id(self) -> str:
return _hash(self.date_val, self.date_book, self.text)
return hashify(self.date_val, self.date_book, self.text)
@field_validator("date_book", mode="before")
@classmethod
@@ -8,7 +8,7 @@ from sqlalchemy import Date, Numeric, String
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
def _hash(*args):
def hashify(*args):
raw = "".join(str(args))
return hashlib.sha256(raw.encode()).hexdigest()
+2 -2
View File
@@ -5,7 +5,7 @@ from pydantic import Field, computed_field
from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models._common import ExportBaseModel, IngestdbBase, _hash, date2str
from src.models.common import ExportBaseModel, IngestdbBase, date2str, hashify
class Transaction(ExportBaseModel):
@@ -33,7 +33,7 @@ class Transaction(ExportBaseModel):
@computed_field # type: ignore[prop-decorator]
@property
def hash_id(self) -> str:
return _hash(
return hashify(
self.source,
self.account_iban,
self.date_valid,
+1 -1
View File
@@ -1,7 +1,7 @@
from sqlalchemy import select
from sqlalchemy.orm import Session
from src.ingest.b4w import upsert_category
from src.ingest.common import upsert_category
from src.models import Category
from src.models.category import CategoryDb