mathias2784/finanz-cockpit
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

comdirect ingest: overhaul

Browse Source
This commit is contained in:
mathias2784
2026-09-12 10:13:59 +02:00
parent 69c13f93ac
commit c354523ae9
7 changed files with 125 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ingest/b4w.py
+5 -5
View File
@@ -7,6 +7,11 @@ from src.ingest._common import _reader_raw, _upsert
from src.models.b4w import B4wDb, B4wExport, Category, CategoryDb
def reader_b4w(path: Path) -> list[B4wExport]:
raw = _reader_raw(path)
return list(map(B4wExport.model_validate, raw))
def upsert_category(
session: Session, category: Category | None, do_commit=True
) -> CategoryDb | None:
@@ -65,8 +70,3 @@ def ingest_b4w(path: Path):
for row in data:
upsert_b4w(session, row, do_commit=False)
session.commit()
def reader_b4w(path: Path) -> list[B4wExport]:
raw = _reader_raw(path)
return list(map(B4wExport.model_validate, raw))
src/ingest/comdirect.py
+76 -62
View File
@@ -4,14 +4,17 @@ from pathlib import Path
from sqlalchemy.orm import Session
from settings import BANK_ACCOUNTS
from src.db import get_session
from src.ingest._common import _upsert
from src.models._common import Csv2Dict, CsV2DictMulti
from src.models.comdirect import (
ComdirectCreditCard,
ComdirectDepot,
ComdirectGiro,
ComdirectGiroDb,
)
from src.models.settings import Account, Source
PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$'
PATTERN_HEADER = r'"([^"]+)"'
@@ -25,12 +28,61 @@ class EXPECT_LINE(Enum):
DATA = auto()
COMDIRECT_ACCOUNT_PARSERS = {
"Girokonto": lambda row: ComdirectGiro.model_validate(row),
"Visa-Karte (Kreditkarte)": lambda row: ComdirectCreditCard.model_validate(row),
"Tagesgeld PLUS-Konto": lambda row: ComdirectGiro.model_validate(row),
"Depot": lambda row: ComdirectDepot.model_validate(row),
}
type ComdirectExportModel = ComdirectGiro | ComdirectDepot | ComdirectCreditCard
SOURCE_COMDIRECT = BANK_ACCOUNTS.get("Comdirect")
assert isinstance(SOURCE_COMDIRECT, Source)
SOURCE_PATH = SOURCE_COMDIRECT.source_path
ACCOUNTS = SOURCE_COMDIRECT.accounts
def reader_comdirect(path: Path) -> dict[str, list[ComdirectExportModel]]:
result = {}
raw = _reader_raw_comdirect(path)
for account in ACCOUNTS:
data_raw: Csv2Dict = raw.pop(account.name)
data: list = list(map(account.model.model_validate, data_raw))
result[account.name] = data
assert not raw
return result
def _reader_raw_comdirect(path: Path) -> CsV2DictMulti:
with path.open(encoding="latin1") as fid:
lines = fid.readlines()
expect_next = EXPECT_LINE.ACCOUNT_NAME
account_name = None
result = {}
for line in lines:
if expect_next == EXPECT_LINE.ACCOUNT_NAME:
account_name = re.findall(PATTERN_ACCOUNT_NAME, line)
if account_name:
account_name = account_name[0]
result[account_name] = {}
expect_next = EXPECT_LINE.BLANK
elif expect_next == EXPECT_LINE.BLANK:
if line == "\n":
expect_next = EXPECT_LINE.HEADER
elif expect_next == EXPECT_LINE.HEADER:
header = re.findall(PATTERN_HEADER, line)
if header:
result[account_name] = []
expect_next = EXPECT_LINE.DATA
elif expect_next == EXPECT_LINE.DATA:
data = re.findall(PATTERN_DATA, line)
if data:
result[account_name].append(
{column: value for column, value in zip(header, data)}
)
else:
expect_next = EXPECT_LINE.ACCOUNT_NAME
else:
expect_next = EXPECT_LINE.ACCOUNT_NAME
return result
def upsert_comdirect_giro(
@@ -69,62 +121,24 @@ def upsert_comdirect_giro(
def ingest_comdirect(path: Path, account_iban: str, account_currency: str):
data = reader_comdirect(path).get("Girokonto")
assert isinstance(data, list)
with get_session() as session:
for row in data:
upsert_comdirect_giro(
session,
row,
account_iban,
account_currency,
assert SOURCE_COMDIRECT
data_collection = reader_comdirect(SOURCE_PATH)
for account_name, data in data_collection.items():
account = SOURCE_COMDIRECT.get_account(account_name)
assert isinstance(account, Account)
if account_name == "Comdirect":
db_load_function = lambda session_, row_, account_: upsert_comdirect_giro(
session=session_,
data=row_,
account_iban=account_.iban,
account_currency=account_.currency,
do_commit=False,
)
session.commit()
def reader_comdirect(path: Path) -> dict[str, list]:
result = {}
raw = _reader_raw_comdirect(path)
for account_name, account_data in raw.items():
parser = COMDIRECT_ACCOUNT_PARSERS.get(account_name)
if parser is None:
continue
result[account_name] = list(map(parser, account_data))
return result
def _reader_raw_comdirect(path: Path):
with path.open(encoding="latin1") as fid:
lines = fid.readlines()
expect_next = EXPECT_LINE.ACCOUNT_NAME
account_name = None
result = {}
for line in lines:
if expect_next == EXPECT_LINE.ACCOUNT_NAME:
account_name = re.findall(PATTERN_ACCOUNT_NAME, line)
if account_name:
account_name = account_name[0]
result[account_name] = {}
expect_next = EXPECT_LINE.BLANK
elif expect_next == EXPECT_LINE.BLANK:
if line == "\n":
expect_next = EXPECT_LINE.HEADER
elif expect_next == EXPECT_LINE.HEADER:
header = re.findall(PATTERN_HEADER, line)
if header:
result[account_name] = []
expect_next = EXPECT_LINE.DATA
elif expect_next == EXPECT_LINE.DATA:
data = re.findall(PATTERN_DATA, line)
if data:
result[account_name].append(
{column: value for column, value in zip(header, data)}
)
else:
expect_next = EXPECT_LINE.ACCOUNT_NAME
else:
expect_next = EXPECT_LINE.ACCOUNT_NAME
return result
continue
with get_session() as session:
for row in data:
data_validated = account.model.model_validate(row)
assert isinstance(data_validated, account.model)
db_load_function(session, row, account)
session.commit()