Category overhaul #20
@@ -71,7 +71,7 @@ Two sources with shared patterns:
|
|||||||
- **comdirect**: `ComdirectGiro`, `ComdirectCreditCard`, `ComdirectDepot`, import to `comdirect_giro` table
|
- **comdirect**: `ComdirectGiro`, `ComdirectCreditCard`, `ComdirectDepot`, import to `comdirect_giro` table
|
||||||
- **unified**: `Transaction`, `TransactionDb` – both ingest pipelines project their staging rows (`import_b4w`, `comdirect_giro`) into one `transactions` table via `src/ingest/transactions.py` (`map_*` mappers + `consolidate_transactions`). Dedup uses a business-key `hash_id` (source + IBAN + value date + amount + text + counterparty) with a UNIQUE constraint and SQLite `INSERT OR IGNORE`, so re-imports are idempotent. Amounts are canonical signed (negative = money out). Depot/credit-card/tagesgeld are not projected yet.
|
- **unified**: `Transaction`, `TransactionDb` – both ingest pipelines project their staging rows (`import_b4w`, `comdirect_giro`) into one `transactions` table via `src/ingest/transactions.py` (`map_*` mappers + `consolidate_transactions`). Dedup uses a business-key `hash_id` (source + IBAN + value date + amount + text + counterparty) with a UNIQUE constraint and SQLite `INSERT OR IGNORE`, so re-imports are idempotent. Amounts are canonical signed (negative = money out). Depot/credit-card/tagesgeld are not projected yet.
|
||||||
|
|
||||||
All models inherit from common base in `src/models/_common.py`:
|
All models inherit from common base in `src/models/common.py`:
|
||||||
- `_hash()`: SHA256-based deduplication key
|
- `_hash()`: SHA256-based deduplication key
|
||||||
- `_parse_date_german()`: Handles "07.09.2026" format and values like "offen", "--"
|
- `_parse_date_german()`: Handles "07.09.2026" format and values like "offen", "--"
|
||||||
- `_parse_decimal_german()`: Converts "1,23" to Decimal
|
- `_parse_decimal_german()`: Converts "1,23" to Decimal
|
||||||
|
|||||||
+1
-1
@@ -6,7 +6,7 @@ from sqlalchemy import engine_from_config, pool
|
|||||||
|
|
||||||
from alembic import context
|
from alembic import context
|
||||||
from src.db import get_set_engine
|
from src.db import get_set_engine
|
||||||
from src.models._common import BaseModelDb
|
from src.models.common import BaseModelDb
|
||||||
|
|
||||||
load_dotenv() # reads .env into os.environ
|
load_dotenv() # reads .env into os.environ
|
||||||
|
|
||||||
|
|||||||
+2
-22
@@ -2,10 +2,9 @@ from pathlib import Path
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from src.ingest._common import _reader_raw, _upsert
|
from src.ingest.common import _reader_raw, _upsert, upsert_category
|
||||||
from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
|
from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
|
||||||
from src.models.b4w import B4wDb, B4wExport, Category
|
from src.models.b4w import B4wDb, B4wExport
|
||||||
from src.models.category import CategoryDb
|
|
||||||
|
|
||||||
|
|
||||||
def reader_b4w(path: Path) -> list[B4wExport]:
|
def reader_b4w(path: Path) -> list[B4wExport]:
|
||||||
@@ -13,25 +12,6 @@ def reader_b4w(path: Path) -> list[B4wExport]:
|
|||||||
return list(map(B4wExport.model_validate, raw))
|
return list(map(B4wExport.model_validate, raw))
|
||||||
|
|
||||||
|
|
||||||
def upsert_category(
|
|
||||||
session: Session, category: Category | None, do_commit=True
|
|
||||||
) -> CategoryDb | None:
|
|
||||||
if category is None:
|
|
||||||
return None
|
|
||||||
parameters = {
|
|
||||||
"name": category.name,
|
|
||||||
"parent": upsert_category(session, category.parent),
|
|
||||||
}
|
|
||||||
return _upsert(
|
|
||||||
session,
|
|
||||||
CategoryDb,
|
|
||||||
parameters,
|
|
||||||
parameters,
|
|
||||||
do_update=False,
|
|
||||||
do_commit=do_commit,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def upsert_b4w(
|
def upsert_b4w(
|
||||||
session: Session, data: B4wExport, do_update=False, do_commit=True
|
session: Session, data: B4wExport, do_update=False, do_commit=True
|
||||||
) -> B4wDb | None:
|
) -> B4wDb | None:
|
||||||
|
|||||||
@@ -6,18 +6,18 @@ from typing import cast
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from settings import BANK_ACCOUNTS
|
from settings import BANK_ACCOUNTS
|
||||||
from src.ingest._common import _upsert
|
from src.ingest.common import _upsert
|
||||||
from src.ingest.transactions import (
|
from src.ingest.transactions import (
|
||||||
consolidate_transactions,
|
consolidate_transactions,
|
||||||
map_comdirect_giro_to_transaction,
|
map_comdirect_giro_to_transaction,
|
||||||
)
|
)
|
||||||
from src.models._common import Csv2Dict, CsV2DictMulti
|
|
||||||
from src.models.comdirect import (
|
from src.models.comdirect import (
|
||||||
ComdirectCreditCard,
|
ComdirectCreditCard,
|
||||||
ComdirectDepot,
|
ComdirectDepot,
|
||||||
ComdirectGiro,
|
ComdirectGiro,
|
||||||
ComdirectGiroDb,
|
ComdirectGiroDb,
|
||||||
)
|
)
|
||||||
|
from src.models.common import Csv2Dict, CsV2DictMulti
|
||||||
from src.models.settings import Account, Source
|
from src.models.settings import Account, Source
|
||||||
|
|
||||||
PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$'
|
PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$'
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ from typing import Any
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from src.models._common import BaseModelDb
|
from src.models import Category
|
||||||
|
from src.models.category import CategoryDb
|
||||||
|
from src.models.common import BaseModelDb
|
||||||
|
|
||||||
|
|
||||||
def _query_existing(
|
def _query_existing(
|
||||||
@@ -42,3 +44,22 @@ def _upsert[DbModelT: BaseModelDb](
|
|||||||
def _reader_raw(path: Path):
|
def _reader_raw(path: Path):
|
||||||
with path.open() as fid:
|
with path.open() as fid:
|
||||||
return list(csv.DictReader(fid))
|
return list(csv.DictReader(fid))
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_category(
|
||||||
|
session: Session, category: Category | None, do_commit=True
|
||||||
|
) -> CategoryDb | None:
|
||||||
|
if category is None:
|
||||||
|
return None
|
||||||
|
parameters = {
|
||||||
|
"name": category.name,
|
||||||
|
"parent": upsert_category(session, category.parent),
|
||||||
|
}
|
||||||
|
return _upsert(
|
||||||
|
session,
|
||||||
|
CategoryDb,
|
||||||
|
parameters,
|
||||||
|
parameters,
|
||||||
|
do_update=False,
|
||||||
|
do_commit=do_commit,
|
||||||
|
)
|
||||||
@@ -4,9 +4,9 @@ from sqlalchemy import select
|
|||||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from src.models._common import BaseModelDb
|
|
||||||
from src.models.b4w import B4wDb
|
from src.models.b4w import B4wDb
|
||||||
from src.models.comdirect import ComdirectGiroDb
|
from src.models.comdirect import ComdirectGiroDb
|
||||||
|
from src.models.common import BaseModelDb
|
||||||
from src.models.transactions import Transaction, TransactionDb
|
from src.models.transactions import Transaction, TransactionDb
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+4
-4
@@ -6,14 +6,14 @@ from sqlalchemy import Date, ForeignKey, Integer, String
|
|||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from src.constants import IBAN_PATTERN, Currency
|
from src.constants import IBAN_PATTERN, Currency
|
||||||
from src.models._common import (
|
from src.models.category import Category
|
||||||
|
from src.models.common import (
|
||||||
ExportBaseModel,
|
ExportBaseModel,
|
||||||
IngestdbBase,
|
IngestdbBase,
|
||||||
Sign,
|
Sign,
|
||||||
_hash,
|
|
||||||
date2str,
|
date2str,
|
||||||
|
hashify,
|
||||||
)
|
)
|
||||||
from src.models.category import Category
|
|
||||||
|
|
||||||
|
|
||||||
class B4wExport(ExportBaseModel):
|
class B4wExport(ExportBaseModel):
|
||||||
@@ -59,7 +59,7 @@ class B4wExport(ExportBaseModel):
|
|||||||
@computed_field # type: ignore[prop-decorator]
|
@computed_field # type: ignore[prop-decorator]
|
||||||
@property
|
@property
|
||||||
def hash_id(self) -> str:
|
def hash_id(self) -> str:
|
||||||
return _hash(self.id)
|
return hashify(self.id)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def amount_signed(self) -> Decimal:
|
def amount_signed(self) -> Decimal:
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
from pydantic import Field, computed_field
|
from pydantic import Field, computed_field
|
||||||
from sqlalchemy import ForeignKey, String, UniqueConstraint
|
from sqlalchemy import ForeignKey, String, UniqueConstraint
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, Session, mapped_column, relationship
|
||||||
|
|
||||||
from src.models._common import BaseModelDb, ExportBaseModel, _hash
|
from src.models.common import BaseModelDb, ExportBaseModel, hashify
|
||||||
|
|
||||||
|
|
||||||
class Category(ExportBaseModel):
|
class Category(ExportBaseModel):
|
||||||
@@ -12,7 +12,7 @@ class Category(ExportBaseModel):
|
|||||||
@computed_field # type: ignore[prop-decorator]
|
@computed_field # type: ignore[prop-decorator]
|
||||||
@property
|
@property
|
||||||
def hash_id(self) -> str:
|
def hash_id(self) -> str:
|
||||||
return _hash(self.name, self.parent)
|
return hashify(self.name, self.parent)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_str(cls, s: str) -> Category | None:
|
def from_str(cls, s: str) -> Category | None:
|
||||||
@@ -54,3 +54,8 @@ class CategoryDb(BaseModelDb):
|
|||||||
)
|
)
|
||||||
children = relationship("CategoryDb", back_populates="parent")
|
children = relationship("CategoryDb", back_populates="parent")
|
||||||
budgets = relationship("CategoryBudgetDb", back_populates="category")
|
budgets = relationship("CategoryBudgetDb", back_populates="category")
|
||||||
|
|
||||||
|
|
||||||
|
def add_category(session: Session, category_name: str):
|
||||||
|
category = Category.from_str(category_name)
|
||||||
|
session.add(category)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from decimal import Decimal
|
|||||||
from sqlalchemy import Date, ForeignKey, Numeric
|
from sqlalchemy import Date, ForeignKey, Numeric
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from src.models._common import BaseModelDb
|
from src.models.common import BaseModelDb
|
||||||
|
|
||||||
|
|
||||||
class CategoryBudgetDb(BaseModelDb):
|
class CategoryBudgetDb(BaseModelDb):
|
||||||
|
|||||||
@@ -5,14 +5,14 @@ from pydantic import ConfigDict, Field, computed_field, field_validator
|
|||||||
from sqlalchemy import Date, String
|
from sqlalchemy import Date, String
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from src.models._common import (
|
from src.models.common import (
|
||||||
ExportBaseModel,
|
ExportBaseModel,
|
||||||
IngestdbBase,
|
IngestdbBase,
|
||||||
_hash,
|
|
||||||
_parse_date_german,
|
_parse_date_german,
|
||||||
_parse_decimal_german,
|
_parse_decimal_german,
|
||||||
date2str,
|
date2str,
|
||||||
decimal2str,
|
decimal2str,
|
||||||
|
hashify,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -72,7 +72,7 @@ class ComdirectGiro(ComdirectBase):
|
|||||||
@computed_field # type: ignore[prop-decorator]
|
@computed_field # type: ignore[prop-decorator]
|
||||||
@property
|
@property
|
||||||
def hash_id(self) -> str:
|
def hash_id(self) -> str:
|
||||||
return _hash(self.date_val, self.date_book, self.text)
|
return hashify(self.date_val, self.date_book, self.text)
|
||||||
|
|
||||||
@field_validator("date_book", mode="before")
|
@field_validator("date_book", mode="before")
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from sqlalchemy import Date, Numeric, String
|
|||||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||||
|
|
||||||
|
|
||||||
def _hash(*args):
|
def hashify(*args):
|
||||||
raw = "".join(str(args))
|
raw = "".join(str(args))
|
||||||
return hashlib.sha256(raw.encode()).hexdigest()
|
return hashlib.sha256(raw.encode()).hexdigest()
|
||||||
|
|
||||||
@@ -5,7 +5,7 @@ from pydantic import Field, computed_field
|
|||||||
from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint
|
from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from src.models._common import ExportBaseModel, IngestdbBase, _hash, date2str
|
from src.models.common import ExportBaseModel, IngestdbBase, date2str, hashify
|
||||||
|
|
||||||
|
|
||||||
class Transaction(ExportBaseModel):
|
class Transaction(ExportBaseModel):
|
||||||
@@ -33,7 +33,7 @@ class Transaction(ExportBaseModel):
|
|||||||
@computed_field # type: ignore[prop-decorator]
|
@computed_field # type: ignore[prop-decorator]
|
||||||
@property
|
@property
|
||||||
def hash_id(self) -> str:
|
def hash_id(self) -> str:
|
||||||
return _hash(
|
return hashify(
|
||||||
self.source,
|
self.source,
|
||||||
self.account_iban,
|
self.account_iban,
|
||||||
self.date_valid,
|
self.date_valid,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from src.ingest.b4w import upsert_category
|
from src.ingest.common import upsert_category
|
||||||
from src.models import Category
|
from src.models import Category
|
||||||
from src.models.category import CategoryDb
|
from src.models.category import CategoryDb
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user