heavy refactor

src/ingest/_common.py

@@ -0,0 +1,44 @@
+import csv
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy.orm import Session
+
+from src.models._common import BaseModelDb
+
+
+def _query_existing(
+    session: Session, DbModel: type[BaseModelDb], query_parameters: dict[str, Any]
+) -> BaseModelDb | None:
+    query = session.query(DbModel).filter_by(**query_parameters)
+    return query.first()
+
+
+def _upsert[DbModelT: BaseModelDb](
+    session: Session,
+    DbModel: type[DbModelT],
+    parameters_existing: dict[str, Any],
+    parameters_new: dict[str, Any],
+    do_update: bool = False,
+    do_commit: bool = True,
+) -> DbModelT:
+    existing_db_row = _query_existing(session, DbModel, parameters_existing)
+
+    if existing_db_row:
+        if do_update:
+            for key, value in parameters_new.items():
+                setattr(existing_db_row, key, value)
+        assert isinstance(existing_db_row, DbModel)
+        return existing_db_row
+
+    new_db_row = DbModel(**parameters_new)
+    session.add(new_db_row)
+    session.flush()
+    if do_commit:
+        session.commit()
+    return new_db_row
+
+
+def _reader_raw(path: Path):
+    with path.open() as fid:
+        return list(csv.DictReader(fid))

src/ingest/b4w.py

@@ -0,0 +1,72 @@
+from pathlib import Path
+
+from sqlalchemy.orm import Session
+
+from src.db import get_session
+from src.ingest._common import _reader_raw, _upsert
+from src.models.b4w import B4wDb, B4wExport, Category, CategoryDb
+
+
+def upsert_category(
+    session: Session, category: Category | None, do_commit=True
+) -> CategoryDb | None:
+    if category is None:
+        return None
+    parameters = {
+        "name": category.name,
+        "parent": upsert_category(session, category.parent),
+    }
+    return _upsert(
+        session,
+        CategoryDb,
+        parameters,
+        parameters,
+        do_update=False,
+        do_commit=do_commit,
+    )
+
+
+def upsert_b4w(
+    session: Session, data: B4wExport, do_update=False, do_commit=True
+) -> B4wDb | None:
+    category_db = upsert_category(session, data.category)
+
+    parameters_existing = {"hash_id": data.hash_id}
+
+    parameters_new = {
+        "id_b4w": data.id,
+        "hash_id": data.hash_id,
+        "account_iban": data.account_iban,
+        "account_currency": data.account_currency.value,
+        "date_valid": data.date_valid,
+        "date_book": data.date_book,
+        "amount": data.amount_signed,
+        "amount_currency": data.amount_currency.value,
+        "text": data.reference,
+        "other_name": data.other_name,
+        "other_iban": data.other_iban,
+        "category": category_db,
+        "notes": data.notes,
+    }
+
+    return _upsert(
+        session,
+        B4wDb,
+        parameters_existing,
+        parameters_new,
+        do_update=do_update,
+        do_commit=do_commit,
+    )
+
+
+def ingest_b4w(path: Path):
+    data = reader_b4w(path)
+    with get_session() as session:
+        for row in data:
+            upsert_b4w(session, row, do_commit=False)
+        session.commit()
+
+
+def reader_b4w(path: Path) -> list[B4wExport]:
+    raw = _reader_raw(path)
+    return list(map(B4wExport.model_validate, raw))

src/ingest/comdirect.py

@@ -0,0 +1,130 @@
+import re
+from enum import Enum, auto
+from pathlib import Path
+
+from sqlalchemy.orm import Session
+
+from src.db import get_session
+from src.ingest._common import _upsert
+from src.models.comdirect import (
+    ComdirectCreditCard,
+    ComdirectDepot,
+    ComdirectGiro,
+    ComdirectGiroDb,
+)
+
+PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$'
+PATTERN_HEADER = r'"([^"]+)"'
+PATTERN_DATA = r'"([^"]+)"'
+
+
+class EXPECT_LINE(Enum):
+    ACCOUNT_NAME = auto()
+    BLANK = auto()
+    HEADER = auto()
+    DATA = auto()
+
+
+COMDIRECT_ACCOUNT_PARSERS = {
+    "Girokonto": lambda row: ComdirectGiro.model_validate(row),
+    "Visa-Karte (Kreditkarte)": lambda row: ComdirectCreditCard.model_validate(row),
+    "Tagesgeld PLUS-Konto": lambda row: ComdirectGiro.model_validate(row),
+    "Depot": lambda row: ComdirectDepot.model_validate(row),
+}
+
+
+def upsert_comdirect_giro(
+    session: Session,
+    data: ComdirectGiro,
+    account_iban: str,
+    account_currency: str,
+    do_update=False,
+    do_commit=True,
+) -> ComdirectGiroDb | None:
+    parameters_existing = {"hash_id": data.hash_id}
+
+    if not data.date_val:
+        return
+
+    parameters_new = {
+        "hash_id": data.hash_id,
+        "account_iban": account_iban,
+        "account_currency": account_currency,
+        "amount_currency": account_currency,
+        "date_valid": data.date_val,
+        "amount": data.amount_eur,
+        "date_book": data.date_book,
+        "type": data.type,
+        "text": data.text,
+    }
+
+    return _upsert(
+        session,
+        ComdirectGiroDb,
+        parameters_existing,
+        parameters_new,
+        do_update=do_update,
+        do_commit=do_commit,
+    )
+
+
+def ingest_comdirect(path: Path, account_iban: str, account_currency: str):
+    data = reader_comdirect(path).get("Girokonto")
+    assert isinstance(data, list)
+    with get_session() as session:
+        for row in data:
+            upsert_comdirect_giro(
+                session,
+                row,
+                account_iban,
+                account_currency,
+                do_commit=False,
+            )
+        session.commit()
+
+
+def reader_comdirect(path: Path) -> dict[str, list]:
+    result = {}
+    raw = _reader_raw_comdirect(path)
+    for account_name, account_data in raw.items():
+        parser = COMDIRECT_ACCOUNT_PARSERS.get(account_name)
+        if parser is None:
+            continue
+        result[account_name] = list(map(parser, account_data))
+    return result
+
+
+def _reader_raw_comdirect(path: Path):
+
+    with path.open(encoding="latin1") as fid:
+        lines = fid.readlines()
+
+    expect_next = EXPECT_LINE.ACCOUNT_NAME
+    account_name = None
+    result = {}
+    for line in lines:
+        if expect_next == EXPECT_LINE.ACCOUNT_NAME:
+            account_name = re.findall(PATTERN_ACCOUNT_NAME, line)
+            if account_name:
+                account_name = account_name[0]
+                result[account_name] = {}
+                expect_next = EXPECT_LINE.BLANK
+        elif expect_next == EXPECT_LINE.BLANK:
+            if line == "\n":
+                expect_next = EXPECT_LINE.HEADER
+        elif expect_next == EXPECT_LINE.HEADER:
+            header = re.findall(PATTERN_HEADER, line)
+            if header:
+                result[account_name] = []
+                expect_next = EXPECT_LINE.DATA
+        elif expect_next == EXPECT_LINE.DATA:
+            data = re.findall(PATTERN_DATA, line)
+            if data:
+                result[account_name].append(
+                    {column: value for column, value in zip(header, data)}
+                )
+            else:
+                expect_next = EXPECT_LINE.ACCOUNT_NAME
+        else:
+            expect_next = EXPECT_LINE.ACCOUNT_NAME
+    return result