Transactions table (#7)

Reviewed-on: #7
Co-authored-by: Mathias <mathias2784@web.de>

src/ingest/b4w.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from sqlalchemy.orm import Session
 
 from src.ingest._common import _reader_raw, _upsert
+from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
 from src.models.b4w import B4wDb, B4wExport, Category, CategoryDb
 
 
@@ -67,4 +68,11 @@ def ingest_b4w(path: Path, session: Session):
     data = reader_b4w(path)
     for row in data:
         upsert_b4w(session, row, do_commit=False)
+    consolidate_transactions(
+        session,
+        source="b4w",
+        StagingDb=B4wDb,
+        mapper=map_b4w_to_transaction,
+        do_commit=False,
+    )
     session.commit()

src/ingest/comdirect.py

@@ -1,11 +1,16 @@
 import re
 from enum import Enum, auto
 from pathlib import Path
+from typing import cast
 
 from sqlalchemy.orm import Session
 
 from settings import BANK_ACCOUNTS
 from src.ingest._common import _upsert
+from src.ingest.transactions import (
+    consolidate_transactions,
+    map_comdirect_giro_to_transaction,
+)
 from src.models._common import Csv2Dict, CsV2DictMulti
 from src.models.comdirect import (
     ComdirectCreditCard,
@@ -40,18 +45,18 @@ COMDIRECT_ACCOUNT_PARSERS = {a.name: a.model.model_validate for a in ACCOUNTS}
 
 
 def reader_comdirect(path: Path) -> dict[str, list[ComdirectExportModel]]:
-    result = {}
+    result: dict[str, list[ComdirectExportModel]] = {}
     raw = _reader_raw_comdirect(path)
     for account in ACCOUNTS:
         assert account.name in raw
         data_raw: Csv2Dict = raw.pop(account.name)
-        data: list = list(map(account.model.model_validate, data_raw))
-        result[account.name] = data
+        data = [account.model.model_validate(row) for row in data_raw]
+        result[account.name] = cast(list[ComdirectExportModel], data)
     assert not raw
     return result
 
 
-def _open_and_readlines(path: Path, encoding: str) -> list:
+def _open_and_readlines(path: Path, encoding: str) -> list[str]:
     with path.open("r", encoding=encoding) as fid:
         lines = fid.readlines()
     return lines
@@ -65,14 +70,15 @@ def _reader_raw_comdirect(path: Path) -> CsV2DictMulti:
         lines = _open_and_readlines(path, encoding="latin-1")
 
     expect_next = EXPECT_LINE.ACCOUNT_NAME
-    account_name = None
-    result = {}
+    account_name: str | None = None
+    header: list[str] = []
+    result: CsV2DictMulti = {}
     for line in lines:
         if expect_next == EXPECT_LINE.ACCOUNT_NAME:
-            account_name = re.findall(PATTERN_ACCOUNT_NAME, line)
-            if account_name:
-                account_name = account_name[0]
-                result[account_name] = {}
+            matches = re.findall(PATTERN_ACCOUNT_NAME, line)
+            if matches:
+                account_name = matches[0]
+                result[account_name] = []
                 expect_next = EXPECT_LINE.BLANK
         elif expect_next == EXPECT_LINE.BLANK:
             if line == "\n":
@@ -80,11 +86,13 @@ def _reader_raw_comdirect(path: Path) -> CsV2DictMulti:
         elif expect_next == EXPECT_LINE.HEADER:
             header = re.findall(PATTERN_HEADER, line)
             if header:
+                assert account_name is not None
                 result[account_name] = []
                 expect_next = EXPECT_LINE.DATA
         elif expect_next == EXPECT_LINE.DATA:
             data = re.findall(PATTERN_DATA, line)
             if data:
+                assert account_name is not None
                 result[account_name].append(
                     {column: value for column, value in zip(header, data)}
                 )
@@ -106,7 +114,7 @@ def upsert_comdirect_giro(
     parameters_existing = {"hash_id": data.hash_id}
 
     if not data.date_val:
-        return
+        return None
 
     parameters_new = {
         "hash_id": data.hash_id,
@@ -138,18 +146,25 @@ def ingest_comdirect(
     for account_name, data in data_collection.items():
         account = SOURCE_COMDIRECT.get_account(account_name)
         assert isinstance(account, Account)
-        if account_name == "Girokonto":
-            db_load_function = lambda session_, row_, account_: upsert_comdirect_giro(
-                session=session_,
-                data=row_,
-                account_iban=account_.iban,
-                account_currency=account_.currency,
-                do_commit=False,
-            )
-        else:
+        if account_name != "Girokonto":
+            # Aktuell speist nur das Girokonto die Tabelle `comdirect_giro`
+            # (Kreditkarte, Tagesgeld und Depot sind vorerst ausgeklammert).
             continue
         for row in data:
             data_validated = account.model.model_validate(row)
-            assert isinstance(data_validated, account.model)
-            db_load_function(session, row, account)
-            session.commit()
+            assert isinstance(data_validated, ComdirectGiro)
+            upsert_comdirect_giro(
+                session=session,
+                data=data_validated,
+                account_iban=account_iban,
+                account_currency=account_currency,
+                do_commit=False,
+            )
+    consolidate_transactions(
+        session,
+        source="comdirect_giro",
+        StagingDb=ComdirectGiroDb,
+        mapper=map_comdirect_giro_to_transaction,
+        do_commit=False,
+    )
+    session.commit()

src/ingest/transactions.py

@@ -0,0 +1,97 @@
+from collections.abc import Callable
+
+from sqlalchemy import select
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+from sqlalchemy.orm import Session
+
+from src.models._common import BaseModelDb
+from src.models.b4w import B4wDb
+from src.models.comdirect import ComdirectGiroDb
+from src.models.transactions import Transaction, TransactionDb
+
+
+def map_b4w_to_transaction(row: B4wDb) -> Transaction:
+    """Project one staging row of ``import_b4w`` onto a canonical Transaction."""
+    return Transaction(
+        source="b4w",
+        source_row_id=row.id,
+        account_iban=row.account_iban,
+        account_currency=row.account_currency,
+        date_valid=row.date_valid,
+        date_book=row.date_book,
+        amount=row.amount,
+        amount_currency=row.amount_currency,
+        text=row.text,
+        other_name=row.other_name,
+        other_iban=row.other_iban,
+        category_id=row.category_id,
+    )
+
+
+def map_comdirect_giro_to_transaction(row: ComdirectGiroDb) -> Transaction:
+    """Project one staging row of ``comdirect_giro`` onto a canonical Transaction."""
+    return Transaction(
+        source="comdirect_giro",
+        source_row_id=row.id,
+        account_iban=row.account_iban,
+        account_currency=row.account_currency,
+        date_valid=row.date_valid,
+        date_book=row.date_book,
+        amount=row.amount,
+        amount_currency=row.amount_currency,
+        text=row.text,
+    )
+
+
+def upsert_transaction(
+    session: Session, transaction: Transaction, do_commit: bool = True
+) -> None:
+    """Insert one canonical transaction, ignoring duplicates on ``hash_id``.
+
+    Uses a DB-native ``INSERT OR IGNORE`` (SQLite). For PostgreSQL, swap for
+    ``postgresql_insert(TransactionDb).values(...).on_conflict_do_nothing(
+    constraint="uq_transactions_hash")``.
+    """
+    parameters = {
+        "hash_id": transaction.hash_id,
+        "source": transaction.source,
+        "source_row_id": transaction.source_row_id,
+        "account_iban": transaction.account_iban,
+        "account_currency": transaction.account_currency,
+        "date_valid": transaction.date_valid,
+        "date_book": transaction.date_book,
+        "amount": transaction.amount,
+        "amount_currency": transaction.amount_currency,
+        "text": transaction.text,
+        "other_name": transaction.other_name,
+        "other_iban": transaction.other_iban,
+        "category_id": transaction.category_id,
+    }
+    stmt = sqlite_insert(TransactionDb).values(parameters)
+    _ = session.execute(stmt.prefix_with("OR IGNORE"))
+    if do_commit:
+        session.commit()
+
+
+def consolidate_transactions[StagingDbT: BaseModelDb](
+    session: Session,
+    source: str,
+    StagingDb: type[StagingDbT],
+    mapper: Callable[[StagingDbT], Transaction],
+    do_commit: bool = True,
+) -> None:
+    """Project staging rows of one source into the unified ``transactions`` table.
+
+    Idempotent: rows whose business-key hash already exists for this source
+    are skipped, and ``INSERT OR IGNORE`` protects against in-batch duplicates.
+    """
+    existing_hashes = set(
+        session.scalars(select(TransactionDb.hash_id).filter_by(source=source))
+    )
+    for staging_row in session.query(StagingDb).all():
+        transaction = mapper(staging_row)
+        if transaction.hash_id in existing_hashes:
+            continue
+        upsert_transaction(session, transaction, do_commit=False)
+    if do_commit:
+        session.commit()

src/main.py

@@ -1,6 +1,7 @@
 import argparse
 from pathlib import Path
 
+from src.db import get_session
 from src.ingest.b4w import ingest_b4w
 from src.ingest.comdirect import ingest_comdirect
 from src.models.settings import Account
@@ -19,17 +20,22 @@ def main():
     args = parser.parse_args()
 
     if args.input_b4w:
-        ingest_b4w(Path(args.input_b4w))
+        with get_session() as session:
+            ingest_b4w(Path(args.input_b4w), session)
     if args.input_comdirect:
         import settings
 
         giro_account = settings.BANK_ACCOUNTS["comdirect"].get_account("Girokonto")
         assert isinstance(giro_account, Account)
-        ingest_comdirect(
-            Path(args.input_comdirect),
-            account_iban=giro_account.iban or "",
-            account_currency=giro_account.currency or "",
-        )
+        with get_session() as session:
+            ingest_comdirect(
+                session,
+                Path(args.input_comdirect),
+                account_iban=giro_account.iban or "",
+                account_currency=(
+                    giro_account.currency.value if giro_account.currency else ""
+                ),
+            )
 
 
 if __name__ == "__main__":

src/models/__init__.py

@@ -5,3 +5,5 @@ from src.models.comdirect import ComdirectDepot as ComdirectDepot
 from src.models.comdirect import ComdirectGiro as ComdirectGiro
 from src.models.settings import Account as Account
 from src.models.settings import Source as Source
+from src.models.transactions import Transaction as Transaction
+from src.models.transactions import TransactionDb as TransactionDb

src/models/_common.py

@@ -4,8 +4,8 @@ from decimal import Decimal
 from enum import Enum
 
 from pydantic import BaseModel
-from sqlalchemy import Column, Date, Integer, Numeric, String
-from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy import Date, Numeric, String
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 
 
 def _hash(*args):
@@ -89,11 +89,11 @@ class BaseModelDb(DeclarativeBase):
 class IngestdbBase(BaseModelDb):
     __abstract__ = True
 
-    id = Column(Integer, primary_key=True, autoincrement=True)
-    hash_id = Column(String, nullable=False)
-    account_iban = Column(String, nullable=False)
-    account_currency = Column(String(3), nullable=False)
-    date_valid = Column(Date, nullable=False)
-    amount = Column(Numeric(12, 2), nullable=False)
-    amount_currency = Column(String(3), nullable=False)
-    text = Column(String, nullable=True)
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    hash_id: Mapped[str] = mapped_column(String, nullable=False)
+    account_iban: Mapped[str] = mapped_column(String, nullable=False)
+    account_currency: Mapped[str] = mapped_column(String(3), nullable=False)
+    date_valid: Mapped[date] = mapped_column(Date, nullable=False)
+    amount: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
+    amount_currency: Mapped[str] = mapped_column(String(3), nullable=False)
+    text: Mapped[str | None] = mapped_column(String, nullable=True)

src/models/b4w.py

@@ -2,8 +2,8 @@ from datetime import date
 from decimal import Decimal
 
 from pydantic import Field, computed_field, field_validator
-from sqlalchemy import Column, Date, ForeignKey, Integer, String, UniqueConstraint
-from sqlalchemy.orm import relationship
+from sqlalchemy import Date, ForeignKey, Integer, String, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from src.constants import IBAN_PATTERN, Currency
 from src.models._common import (
@@ -56,7 +56,7 @@ class B4wExport(ExportBaseModel):
             "Notes": f"{self.notes}",
         }
 
-    @computed_field
+    @computed_field  # type: ignore[prop-decorator]
     @property
     def hash_id(self) -> str:
         return _hash(self.id)
@@ -91,7 +91,7 @@ class Category(ExportBaseModel):
     name: str = Field(pattern=r"[^:\r\n]+")
     parent: Category | None = None
 
-    @computed_field
+    @computed_field  # type: ignore[prop-decorator]
     @property
     def hash_id(self) -> str:
         return _hash(self.name, self.parent)
@@ -99,7 +99,7 @@ class Category(ExportBaseModel):
     @classmethod
     def from_str(cls, s: str) -> Category | None:
         if not s:
-            return
+            return None
         components = s.split(":")
         name = components[-1]
         rest = ":".join(components[:-1])
@@ -124,9 +124,11 @@ class CategoryDb(BaseModelDb):
         UniqueConstraint("name", "parent_id", name="uq_category_name_parent"),
     )
 
-    id = Column(Integer, primary_key=True, autoincrement=True)
-    name = Column(String, nullable=False)
-    parent_id = Column(Integer, ForeignKey("category.id"), nullable=True)
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    name: Mapped[str] = mapped_column(String, nullable=False)
+    parent_id: Mapped[int | None] = mapped_column(
+        ForeignKey("category.id"), nullable=True
+    )
 
     # Relationships
     parent = relationship(
@@ -134,18 +136,18 @@ class CategoryDb(BaseModelDb):
     )
     children = relationship("CategoryDb", back_populates="parent")
 
-    UniqueConstraint("name", "parent_id", name="uq_category_name_parent")
-
 
 class B4wDb(IngestdbBase):
     __tablename__ = "import_b4w"
 
-    id_b4w = Column(Integer, nullable=False, unique=True)
-    date_book = Column(Date, nullable=True)
-    other_name = Column(String, nullable=True)
-    other_iban = Column(String, nullable=True)
-    category_id = Column(Integer, ForeignKey("category.id"), nullable=True)
-    notes = Column(String, nullable=True)
+    id_b4w: Mapped[int] = mapped_column(Integer, nullable=False, unique=True)
+    date_book: Mapped[date | None] = mapped_column(Date, nullable=True)
+    other_name: Mapped[str | None] = mapped_column(String, nullable=True)
+    other_iban: Mapped[str | None] = mapped_column(String, nullable=True)
+    category_id: Mapped[int | None] = mapped_column(
+        ForeignKey("category.id"), nullable=True
+    )
+    notes: Mapped[str | None] = mapped_column(String, nullable=True)
 
     # relationship
     category = relationship("CategoryDb")

src/models/comdirect.py

@@ -2,7 +2,8 @@ from datetime import date
 from decimal import Decimal
 
 from pydantic import ConfigDict, Field, computed_field, field_validator
-from sqlalchemy import Column, Date, String
+from sqlalchemy import Date, String
+from sqlalchemy.orm import Mapped, mapped_column
 
 from src.models._common import (
     ExportBaseModel,
@@ -26,7 +27,7 @@ class ComdirectBase(ExportBaseModel):
     @classmethod
     def parse_date_val(cls, date_str: str | None) -> date | None:
         if date_str is None:
-            return
+            return None
         return _parse_date_german(date_str)
 
     @field_validator("amount_eur", mode="before")
@@ -68,7 +69,7 @@ class ComdirectGiro(ComdirectBase):
     )
     text: str = Field(alias="Buchungstext")
 
-    @computed_field
+    @computed_field  # type: ignore[prop-decorator]
     @property
     def hash_id(self) -> str:
         return _hash(self.date_val, self.date_book, self.text)
@@ -103,9 +104,6 @@ class ComdirectCreditCard(ComdirectGiro):
     )
     reference: str = Field(alias="Referenz", default="")
     text: str = Field(alias="Buchungstext", default="")
-    amount_eur: Decimal | None = Field(
-        alias="Umsatz in EUR", examples=["32,43"], default=None
-    )
 
     def parameters_to_dict(self, by_name: bool = True) -> dict[str, str]:
         parameters = dict(**super().parameters_to_dict(by_name=by_name))
@@ -155,5 +153,5 @@ class ComdirectBaseDb(IngestdbBase):
 class ComdirectGiroDb(ComdirectBaseDb):
     __tablename__ = "comdirect_giro"
 
-    type = Column(String, nullable=True)
-    date_book = Column(Date, nullable=True)
+    type: Mapped[str | None] = mapped_column(String, nullable=True)
+    date_book: Mapped[date | None] = mapped_column(Date, nullable=True)

src/models/transactions.py

@@ -0,0 +1,78 @@
+from datetime import date
+from decimal import Decimal
+
+from pydantic import Field, computed_field
+from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from src.models._common import ExportBaseModel, IngestdbBase, _hash, date2str
+
+
+class Transaction(ExportBaseModel):
+    """Canonical, source-agnostic representation of a bank transaction.
+
+    Both input sources (b4w, comdirect giro) are projected into this model.
+    The deduplication key is a business-key hash of the transaction
+    attributes, so re-importing an export is idempotent even if
+    source-internal ids change between exports.
+    """
+
+    source: str = Field(description="Source name", examples=["b4w", "comdirect_giro"])
+    source_row_id: int = Field(description="Primary key of the staging table row")
+    account_iban: str
+    account_currency: str
+    date_valid: date
+    amount: Decimal = Field(description="Signed amount; negative = money out")
+    amount_currency: str
+    date_book: date | None = None
+    text: str | None = None
+    other_name: str | None = None
+    other_iban: str | None = None
+    category_id: int | None = None
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def hash_id(self) -> str:
+        return _hash(
+            self.source,
+            self.account_iban,
+            self.date_valid,
+            f"{self.amount:.2f}",
+            self.text,
+            self.other_name,
+        )
+
+    def parameters_to_dict(self) -> dict[str, str]:
+        return {
+            "source": f"{self.source}",
+            "source_row_id": f"{self.source_row_id}",
+            "account_iban": f"{self.account_iban}",
+            "account_currency": f"{self.account_currency}",
+            "date_valid": date2str(self.date_valid),
+            "amount": f"{self.amount}",
+            "amount_currency": f"{self.amount_currency}",
+            "date_book": date2str(self.date_book),
+            "text": f"{self.text or ''}",
+            "other_name": f"{self.other_name or ''}",
+            "other_iban": f"{self.other_iban or ''}",
+            "category_id": f"{self.category_id}",
+        }
+
+
+class TransactionDb(IngestdbBase):
+    __tablename__ = "transactions"
+    __table_args__ = (
+        UniqueConstraint("hash_id", name="uq_transactions_hash"),
+        Index("ix_transactions_source_row_id", "source", "source_row_id"),
+    )
+
+    source: Mapped[str] = mapped_column(String, nullable=False)
+    source_row_id: Mapped[int] = mapped_column(Integer, nullable=False)
+    date_book: Mapped[date | None] = mapped_column(Date, nullable=True)
+    other_name: Mapped[str | None] = mapped_column(String, nullable=True)
+    other_iban: Mapped[str | None] = mapped_column(String, nullable=True)
+    category_id: Mapped[int | None] = mapped_column(
+        ForeignKey("category.id"), nullable=True
+    )
+
+    category = relationship("CategoryDb")