some refactoring

This commit is contained in:
2026-10-16 15:27:41 +02:00
parent 696823406e
commit ef1a65adeb
13 changed files with 49 additions and 43 deletions
+1 -1
View File
@@ -71,7 +71,7 @@ Two sources with shared patterns:
- **comdirect**: `ComdirectGiro`, `ComdirectCreditCard`, `ComdirectDepot`, import to `comdirect_giro` table - **comdirect**: `ComdirectGiro`, `ComdirectCreditCard`, `ComdirectDepot`, import to `comdirect_giro` table
- **unified**: `Transaction`, `TransactionDb` – both ingest pipelines project their staging rows (`import_b4w`, `comdirect_giro`) into one `transactions` table via `src/ingest/transactions.py` (`map_*` mappers + `consolidate_transactions`). Dedup uses a business-key `hash_id` (source + IBAN + value date + amount + text + counterparty) with a UNIQUE constraint and SQLite `INSERT OR IGNORE`, so re-imports are idempotent. Amounts are canonical signed (negative = money out). Depot/credit-card/tagesgeld are not projected yet. - **unified**: `Transaction`, `TransactionDb` – both ingest pipelines project their staging rows (`import_b4w`, `comdirect_giro`) into one `transactions` table via `src/ingest/transactions.py` (`map_*` mappers + `consolidate_transactions`). Dedup uses a business-key `hash_id` (source + IBAN + value date + amount + text + counterparty) with a UNIQUE constraint and SQLite `INSERT OR IGNORE`, so re-imports are idempotent. Amounts are canonical signed (negative = money out). Depot/credit-card/tagesgeld are not projected yet.
All models inherit from common base in `src/models/_common.py`: All models inherit from common base in `src/models/common.py`:
- `_hash()`: SHA256-based deduplication key - `_hash()`: SHA256-based deduplication key
- `_parse_date_german()`: Handles "07.09.2026" format and values like "offen", "--" - `_parse_date_german()`: Handles "07.09.2026" format and values like "offen", "--"
- `_parse_decimal_german()`: Converts "1,23" to Decimal - `_parse_decimal_german()`: Converts "1,23" to Decimal
+1 -1
View File
@@ -6,7 +6,7 @@ from sqlalchemy import engine_from_config, pool
from alembic import context from alembic import context
from src.db import get_set_engine from src.db import get_set_engine
from src.models._common import BaseModelDb from src.models.common import BaseModelDb
load_dotenv() # reads .env into os.environ load_dotenv() # reads .env into os.environ
+2 -22
View File
@@ -2,10 +2,9 @@ from pathlib import Path
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from src.ingest._common import _reader_raw, _upsert from src.ingest.common import _reader_raw, _upsert, upsert_category
from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
from src.models.b4w import B4wDb, B4wExport, Category from src.models.b4w import B4wDb, B4wExport
from src.models.category import CategoryDb
def reader_b4w(path: Path) -> list[B4wExport]: def reader_b4w(path: Path) -> list[B4wExport]:
@@ -13,25 +12,6 @@ def reader_b4w(path: Path) -> list[B4wExport]:
return list(map(B4wExport.model_validate, raw)) return list(map(B4wExport.model_validate, raw))
def upsert_category(
session: Session, category: Category | None, do_commit=True
) -> CategoryDb | None:
if category is None:
return None
parameters = {
"name": category.name,
"parent": upsert_category(session, category.parent),
}
return _upsert(
session,
CategoryDb,
parameters,
parameters,
do_update=False,
do_commit=do_commit,
)
def upsert_b4w( def upsert_b4w(
session: Session, data: B4wExport, do_update=False, do_commit=True session: Session, data: B4wExport, do_update=False, do_commit=True
) -> B4wDb | None: ) -> B4wDb | None:
+2 -2
View File
@@ -6,18 +6,18 @@ from typing import cast
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from settings import BANK_ACCOUNTS from settings import BANK_ACCOUNTS
from src.ingest._common import _upsert from src.ingest.common import _upsert
from src.ingest.transactions import ( from src.ingest.transactions import (
consolidate_transactions, consolidate_transactions,
map_comdirect_giro_to_transaction, map_comdirect_giro_to_transaction,
) )
from src.models._common import Csv2Dict, CsV2DictMulti
from src.models.comdirect import ( from src.models.comdirect import (
ComdirectCreditCard, ComdirectCreditCard,
ComdirectDepot, ComdirectDepot,
ComdirectGiro, ComdirectGiro,
ComdirectGiroDb, ComdirectGiroDb,
) )
from src.models.common import Csv2Dict, CsV2DictMulti
from src.models.settings import Account, Source from src.models.settings import Account, Source
PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$' PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$'
+22 -1
View File
@@ -4,7 +4,9 @@ from typing import Any
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from src.models._common import BaseModelDb from src.models import Category
from src.models.category import CategoryDb
from src.models.common import BaseModelDb
def _query_existing( def _query_existing(
@@ -42,3 +44,22 @@ def _upsert[DbModelT: BaseModelDb](
def _reader_raw(path: Path): def _reader_raw(path: Path):
with path.open() as fid: with path.open() as fid:
return list(csv.DictReader(fid)) return list(csv.DictReader(fid))
def upsert_category(
session: Session, category: Category | None, do_commit=True
) -> CategoryDb | None:
if category is None:
return None
parameters = {
"name": category.name,
"parent": upsert_category(session, category.parent),
}
return _upsert(
session,
CategoryDb,
parameters,
parameters,
do_update=False,
do_commit=do_commit,
)
+1 -1
View File
@@ -4,9 +4,9 @@ from sqlalchemy import select
from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from src.models._common import BaseModelDb
from src.models.b4w import B4wDb from src.models.b4w import B4wDb
from src.models.comdirect import ComdirectGiroDb from src.models.comdirect import ComdirectGiroDb
from src.models.common import BaseModelDb
from src.models.transactions import Transaction, TransactionDb from src.models.transactions import Transaction, TransactionDb
+4 -4
View File
@@ -6,14 +6,14 @@ from sqlalchemy import Date, ForeignKey, Integer, String
from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.constants import IBAN_PATTERN, Currency from src.constants import IBAN_PATTERN, Currency
from src.models._common import ( from src.models.category import Category
from src.models.common import (
ExportBaseModel, ExportBaseModel,
IngestdbBase, IngestdbBase,
Sign, Sign,
_hash,
date2str, date2str,
hashify,
) )
from src.models.category import Category
class B4wExport(ExportBaseModel): class B4wExport(ExportBaseModel):
@@ -59,7 +59,7 @@ class B4wExport(ExportBaseModel):
@computed_field # type: ignore[prop-decorator] @computed_field # type: ignore[prop-decorator]
@property @property
def hash_id(self) -> str: def hash_id(self) -> str:
return _hash(self.id) return hashify(self.id)
@property @property
def amount_signed(self) -> Decimal: def amount_signed(self) -> Decimal:
+8 -3
View File
@@ -1,8 +1,8 @@
from pydantic import Field, computed_field from pydantic import Field, computed_field
from sqlalchemy import ForeignKey, String, UniqueConstraint from sqlalchemy import ForeignKey, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm import Mapped, Session, mapped_column, relationship
from src.models._common import BaseModelDb, ExportBaseModel, _hash from src.models.common import BaseModelDb, ExportBaseModel, hashify
class Category(ExportBaseModel): class Category(ExportBaseModel):
@@ -12,7 +12,7 @@ class Category(ExportBaseModel):
@computed_field # type: ignore[prop-decorator] @computed_field # type: ignore[prop-decorator]
@property @property
def hash_id(self) -> str: def hash_id(self) -> str:
return _hash(self.name, self.parent) return hashify(self.name, self.parent)
@classmethod @classmethod
def from_str(cls, s: str) -> Category | None: def from_str(cls, s: str) -> Category | None:
@@ -54,3 +54,8 @@ class CategoryDb(BaseModelDb):
) )
children = relationship("CategoryDb", back_populates="parent") children = relationship("CategoryDb", back_populates="parent")
budgets = relationship("CategoryBudgetDb", back_populates="category") budgets = relationship("CategoryBudgetDb", back_populates="category")
def add_category(session: Session, category_name: str):
category = Category.from_str(category_name)
session.add(category)
+1 -1
View File
@@ -4,7 +4,7 @@ from decimal import Decimal
from sqlalchemy import Date, ForeignKey, Numeric from sqlalchemy import Date, ForeignKey, Numeric
from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models._common import BaseModelDb from src.models.common import BaseModelDb
class CategoryBudgetDb(BaseModelDb): class CategoryBudgetDb(BaseModelDb):
+3 -3
View File
@@ -5,14 +5,14 @@ from pydantic import ConfigDict, Field, computed_field, field_validator
from sqlalchemy import Date, String from sqlalchemy import Date, String
from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.orm import Mapped, mapped_column
from src.models._common import ( from src.models.common import (
ExportBaseModel, ExportBaseModel,
IngestdbBase, IngestdbBase,
_hash,
_parse_date_german, _parse_date_german,
_parse_decimal_german, _parse_decimal_german,
date2str, date2str,
decimal2str, decimal2str,
hashify,
) )
@@ -72,7 +72,7 @@ class ComdirectGiro(ComdirectBase):
@computed_field # type: ignore[prop-decorator] @computed_field # type: ignore[prop-decorator]
@property @property
def hash_id(self) -> str: def hash_id(self) -> str:
return _hash(self.date_val, self.date_book, self.text) return hashify(self.date_val, self.date_book, self.text)
@field_validator("date_book", mode="before") @field_validator("date_book", mode="before")
@classmethod @classmethod
@@ -8,7 +8,7 @@ from sqlalchemy import Date, Numeric, String
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
def _hash(*args): def hashify(*args):
raw = "".join(str(args)) raw = "".join(str(args))
return hashlib.sha256(raw.encode()).hexdigest() return hashlib.sha256(raw.encode()).hexdigest()
+2 -2
View File
@@ -5,7 +5,7 @@ from pydantic import Field, computed_field
from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models._common import ExportBaseModel, IngestdbBase, _hash, date2str from src.models.common import ExportBaseModel, IngestdbBase, date2str, hashify
class Transaction(ExportBaseModel): class Transaction(ExportBaseModel):
@@ -33,7 +33,7 @@ class Transaction(ExportBaseModel):
@computed_field # type: ignore[prop-decorator] @computed_field # type: ignore[prop-decorator]
@property @property
def hash_id(self) -> str: def hash_id(self) -> str:
return _hash( return hashify(
self.source, self.source,
self.account_iban, self.account_iban,
self.date_valid, self.date_valid,
+1 -1
View File
@@ -1,7 +1,7 @@
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from src.ingest.b4w import upsert_category from src.ingest.common import upsert_category
from src.models import Category from src.models import Category
from src.models.category import CategoryDb from src.models.category import CategoryDb