mathias2784/finanz-cockpit
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

central transactions table

Browse Source
This commit is contained in:
mathias2784
2026-10-10 17:25:00 +01:00
parent fdc66037ce
commit 3fab51fe70
15 changed files with 715 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ingest/b4w.py
+8
View File
@@ -3,6 +3,7 @@ from pathlib import Path
from sqlalchemy.orm import Session
from src.ingest._common import _reader_raw, _upsert
from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
from src.models.b4w import B4wDb, B4wExport, Category, CategoryDb
@@ -67,4 +68,11 @@ def ingest_b4w(path: Path, session: Session):
data = reader_b4w(path)
for row in data:
upsert_b4w(session, row, do_commit=False)
consolidate_transactions(
session,
source="b4w",
StagingDb=B4wDb,
mapper=map_b4w_to_transaction,
do_commit=False,
)
session.commit()
src/ingest/comdirect.py
+38 -23
View File
@@ -1,11 +1,16 @@
import re
from enum import Enum, auto
from pathlib import Path
from typing import cast
from sqlalchemy.orm import Session
from settings import BANK_ACCOUNTS
from src.ingest._common import _upsert
from src.ingest.transactions import (
consolidate_transactions,
map_comdirect_giro_to_transaction,
)
from src.models._common import Csv2Dict, CsV2DictMulti
from src.models.comdirect import (
ComdirectCreditCard,
@@ -40,18 +45,18 @@ COMDIRECT_ACCOUNT_PARSERS = {a.name: a.model.model_validate for a in ACCOUNTS}
def reader_comdirect(path: Path) -> dict[str, list[ComdirectExportModel]]:
result = {}
result: dict[str, list[ComdirectExportModel]] = {}
raw = _reader_raw_comdirect(path)
for account in ACCOUNTS:
assert account.name in raw
data_raw: Csv2Dict = raw.pop(account.name)
data: list = list(map(account.model.model_validate, data_raw))
result[account.name] = data
data = [account.model.model_validate(row) for row in data_raw]
result[account.name] = cast(list[ComdirectExportModel], data)
assert not raw
return result
def _open_and_readlines(path: Path, encoding: str) -> list:
def _open_and_readlines(path: Path, encoding: str) -> list[str]:
with path.open("r", encoding=encoding) as fid:
lines = fid.readlines()
return lines
@@ -65,14 +70,15 @@ def _reader_raw_comdirect(path: Path) -> CsV2DictMulti:
lines = _open_and_readlines(path, encoding="latin-1")
expect_next = EXPECT_LINE.ACCOUNT_NAME
account_name = None
result = {}
account_name: str | None = None
header: list[str] = []
result: CsV2DictMulti = {}
for line in lines:
if expect_next == EXPECT_LINE.ACCOUNT_NAME:
account_name = re.findall(PATTERN_ACCOUNT_NAME, line)
if account_name:
account_name = account_name[0]
result[account_name] = {}
matches = re.findall(PATTERN_ACCOUNT_NAME, line)
if matches:
account_name = matches[0]
result[account_name] = []
expect_next = EXPECT_LINE.BLANK
elif expect_next == EXPECT_LINE.BLANK:
if line == "\n":
@@ -80,11 +86,13 @@ def _reader_raw_comdirect(path: Path) -> CsV2DictMulti:
elif expect_next == EXPECT_LINE.HEADER:
header = re.findall(PATTERN_HEADER, line)
if header:
assert account_name is not None
result[account_name] = []
expect_next = EXPECT_LINE.DATA
elif expect_next == EXPECT_LINE.DATA:
data = re.findall(PATTERN_DATA, line)
if data:
assert account_name is not None
result[account_name].append(
{column: value for column, value in zip(header, data)}
)
@@ -106,7 +114,7 @@ def upsert_comdirect_giro(
parameters_existing = {"hash_id": data.hash_id}
if not data.date_val:
return
return None
parameters_new = {
"hash_id": data.hash_id,
@@ -138,18 +146,25 @@ def ingest_comdirect(
for account_name, data in data_collection.items():
account = SOURCE_COMDIRECT.get_account(account_name)
assert isinstance(account, Account)
if account_name == "Girokonto":
db_load_function = lambda session_, row_, account_: upsert_comdirect_giro(
session=session_,
data=row_,
account_iban=account_.iban,
account_currency=account_.currency,
do_commit=False,
)
else:
if account_name != "Girokonto":
# Aktuell speist nur das Girokonto die Tabelle `comdirect_giro`
# (Kreditkarte, Tagesgeld und Depot sind vorerst ausgeklammert).
continue
for row in data:
data_validated = account.model.model_validate(row)
assert isinstance(data_validated, account.model)
db_load_function(session, row, account)
session.commit()
assert isinstance(data_validated, ComdirectGiro)
upsert_comdirect_giro(
session=session,
data=data_validated,
account_iban=account_iban,
account_currency=account_currency,
do_commit=False,
)
consolidate_transactions(
session,
source="comdirect_giro",
StagingDb=ComdirectGiroDb,
mapper=map_comdirect_giro_to_transaction,
do_commit=False,
)
session.commit()
src/ingest/transactions.py
+97
View File
@@ -0,0 +1,97 @@
from collections.abc import Callable
from sqlalchemy import select
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
from sqlalchemy.orm import Session
from src.models._common import BaseModelDb
from src.models.b4w import B4wDb
from src.models.comdirect import ComdirectGiroDb
from src.models.transactions import Transaction, TransactionDb
def map_b4w_to_transaction(row: B4wDb) -> Transaction:
"""Project one staging row of ``import_b4w`` onto a canonical Transaction."""
return Transaction(
source="b4w",
source_row_id=row.id,
account_iban=row.account_iban,
account_currency=row.account_currency,
date_valid=row.date_valid,
date_book=row.date_book,
amount=row.amount,
amount_currency=row.amount_currency,
text=row.text,
other_name=row.other_name,
other_iban=row.other_iban,
category_id=row.category_id,
)
def map_comdirect_giro_to_transaction(row: ComdirectGiroDb) -> Transaction:
"""Project one staging row of ``comdirect_giro`` onto a canonical Transaction."""
return Transaction(
source="comdirect_giro",
source_row_id=row.id,
account_iban=row.account_iban,
account_currency=row.account_currency,
date_valid=row.date_valid,
date_book=row.date_book,
amount=row.amount,
amount_currency=row.amount_currency,
text=row.text,
)
def upsert_transaction(
session: Session, transaction: Transaction, do_commit: bool = True
) -> None:
"""Insert one canonical transaction, ignoring duplicates on ``hash_id``.
Uses a DB-native ``INSERT OR IGNORE`` (SQLite). For PostgreSQL, swap for
``postgresql_insert(TransactionDb).values(...).on_conflict_do_nothing(
constraint="uq_transactions_hash")``.
"""
parameters = {
"hash_id": transaction.hash_id,
"source": transaction.source,
"source_row_id": transaction.source_row_id,
"account_iban": transaction.account_iban,
"account_currency": transaction.account_currency,
"date_valid": transaction.date_valid,
"date_book": transaction.date_book,
"amount": transaction.amount,
"amount_currency": transaction.amount_currency,
"text": transaction.text,
"other_name": transaction.other_name,
"other_iban": transaction.other_iban,
"category_id": transaction.category_id,
}
stmt = sqlite_insert(TransactionDb).values(parameters)
_ = session.execute(stmt.prefix_with("OR IGNORE"))
if do_commit:
session.commit()
def consolidate_transactions[StagingDbT: BaseModelDb](
session: Session,
source: str,
StagingDb: type[StagingDbT],
mapper: Callable[[StagingDbT], Transaction],
do_commit: bool = True,
) -> None:
"""Project staging rows of one source into the unified ``transactions`` table.
Idempotent: rows whose business-key hash already exists for this source
are skipped, and ``INSERT OR IGNORE`` protects against in-batch duplicates.
"""
existing_hashes = set(
session.scalars(select(TransactionDb.hash_id).filter_by(source=source))
)
for staging_row in session.query(StagingDb).all():
transaction = mapper(staging_row)
if transaction.hash_id in existing_hashes:
continue
upsert_transaction(session, transaction, do_commit=False)
if do_commit:
session.commit()