Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dc98ea910f | ||
|
|
da0a78d2d0 | ||
|
|
5cea645062 | ||
|
|
ef1a65adeb | ||
|
|
696823406e | ||
|
|
c7e4b68dfb | ||
|
|
435f7b92bf |
@@ -71,7 +71,7 @@ Two sources with shared patterns:
|
|||||||
- **comdirect**: `ComdirectGiro`, `ComdirectCreditCard`, `ComdirectDepot`, import to `comdirect_giro` table
|
- **comdirect**: `ComdirectGiro`, `ComdirectCreditCard`, `ComdirectDepot`, import to `comdirect_giro` table
|
||||||
- **unified**: `Transaction`, `TransactionDb` – both ingest pipelines project their staging rows (`import_b4w`, `comdirect_giro`) into one `transactions` table via `src/ingest/transactions.py` (`map_*` mappers + `consolidate_transactions`). Dedup uses a business-key `hash_id` (source + IBAN + value date + amount + text + counterparty) with a UNIQUE constraint and SQLite `INSERT OR IGNORE`, so re-imports are idempotent. Amounts are canonical signed (negative = money out). Depot/credit-card/tagesgeld are not projected yet.
|
- **unified**: `Transaction`, `TransactionDb` – both ingest pipelines project their staging rows (`import_b4w`, `comdirect_giro`) into one `transactions` table via `src/ingest/transactions.py` (`map_*` mappers + `consolidate_transactions`). Dedup uses a business-key `hash_id` (source + IBAN + value date + amount + text + counterparty) with a UNIQUE constraint and SQLite `INSERT OR IGNORE`, so re-imports are idempotent. Amounts are canonical signed (negative = money out). Depot/credit-card/tagesgeld are not projected yet.
|
||||||
|
|
||||||
All models inherit from common base in `src/models/_common.py`:
|
All models inherit from common base in `src/models/common.py`:
|
||||||
- `_hash()`: SHA256-based deduplication key
|
- `_hash()`: SHA256-based deduplication key
|
||||||
- `_parse_date_german()`: Handles "07.09.2026" format and values like "offen", "--"
|
- `_parse_date_german()`: Handles "07.09.2026" format and values like "offen", "--"
|
||||||
- `_parse_decimal_german()`: Converts "1,23" to Decimal
|
- `_parse_decimal_german()`: Converts "1,23" to Decimal
|
||||||
|
|||||||
+1
-1
@@ -6,7 +6,7 @@ from sqlalchemy import engine_from_config, pool
|
|||||||
|
|
||||||
from alembic import context
|
from alembic import context
|
||||||
from src.db import get_set_engine
|
from src.db import get_set_engine
|
||||||
from src.models._common import BaseModelDb
|
from src.models.common import BaseModelDb
|
||||||
|
|
||||||
load_dotenv() # reads .env into os.environ
|
load_dotenv() # reads .env into os.environ
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""create category_budget table
|
||||||
|
|
||||||
|
Revision ID: fe23064de1c5
|
||||||
|
Revises: 78ec55b9e1d1
|
||||||
|
Create Date: 2026-10-16 10:43:24.952699
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = "fe23064de1c5"
|
||||||
|
down_revision: str | Sequence[str] | None = "78ec55b9e1d1"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""Upgrade schema."""
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.create_table(
|
||||||
|
"category_budget",
|
||||||
|
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
|
||||||
|
sa.Column("category_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("amount_eur", sa.Numeric(precision=12, scale=2), nullable=False),
|
||||||
|
sa.Column("valid_from", sa.Date(), nullable=True),
|
||||||
|
sa.Column("valid_to", sa.Date(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["category_id"],
|
||||||
|
["category.id"],
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_category_budget_category_id",
|
||||||
|
"category_budget",
|
||||||
|
["category_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
# ### end Alembic commands ###
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""Downgrade schema."""
|
||||||
|
# ### commands auto generated by Alembic - please adjust! ###
|
||||||
|
op.drop_index("ix_category_budget_category_id", table_name="category_budget")
|
||||||
|
op.drop_table("category_budget")
|
||||||
|
# ### end Alembic commands ###
|
||||||
@@ -1,20 +1,16 @@
|
|||||||
import csv
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import DeclarativeBase, Session
|
||||||
|
|
||||||
from src.models._common import BaseModelDb
|
|
||||||
|
|
||||||
|
|
||||||
def _query_existing(
|
def _query_existing[DbModelT: DeclarativeBase](
|
||||||
session: Session, DbModel: type[BaseModelDb], query_parameters: dict[str, Any]
|
session: Session, DbModel: type[DbModelT], query_parameters: dict[str, Any]
|
||||||
) -> BaseModelDb | None:
|
) -> DbModelT | None:
|
||||||
query = session.query(DbModel).filter_by(**query_parameters)
|
query = session.query(DbModel).filter_by(**query_parameters)
|
||||||
return query.first()
|
return query.first()
|
||||||
|
|
||||||
|
|
||||||
def _upsert[DbModelT: BaseModelDb](
|
def upsert[DbModelT: DeclarativeBase](
|
||||||
session: Session,
|
session: Session,
|
||||||
DbModel: type[DbModelT],
|
DbModel: type[DbModelT],
|
||||||
parameters_existing: dict[str, Any],
|
parameters_existing: dict[str, Any],
|
||||||
@@ -28,7 +24,6 @@ def _upsert[DbModelT: BaseModelDb](
|
|||||||
if do_update:
|
if do_update:
|
||||||
for key, value in parameters_new.items():
|
for key, value in parameters_new.items():
|
||||||
setattr(existing_db_row, key, value)
|
setattr(existing_db_row, key, value)
|
||||||
assert isinstance(existing_db_row, DbModel)
|
|
||||||
return existing_db_row
|
return existing_db_row
|
||||||
|
|
||||||
new_db_row = DbModel(**parameters_new)
|
new_db_row = DbModel(**parameters_new)
|
||||||
@@ -37,8 +32,3 @@ def _upsert[DbModelT: BaseModelDb](
|
|||||||
if do_commit:
|
if do_commit:
|
||||||
session.commit()
|
session.commit()
|
||||||
return new_db_row
|
return new_db_row
|
||||||
|
|
||||||
|
|
||||||
def _reader_raw(path: Path):
|
|
||||||
with path.open() as fid:
|
|
||||||
return list(csv.DictReader(fid))
|
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
from datetime import date, timedelta
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from src.models.category import Category, CategoryBudget, CategoryBudgetDb, CategoryDb
|
||||||
|
|
||||||
|
|
||||||
|
def find_category_db(session: Session, category: Category | None) -> CategoryDb | None:
|
||||||
|
if category is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def find_existing(node: Category) -> CategoryDb | None:
|
||||||
|
if node.parent is None:
|
||||||
|
parent_id = None
|
||||||
|
else:
|
||||||
|
parent_db = find_existing(node.parent)
|
||||||
|
if parent_db is None:
|
||||||
|
return None
|
||||||
|
parent_id = parent_db.id
|
||||||
|
|
||||||
|
return (
|
||||||
|
session.query(CategoryDb)
|
||||||
|
.filter_by(name=node.name, parent_id=parent_id)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
|
||||||
|
return find_existing(category)
|
||||||
|
|
||||||
|
|
||||||
|
def set_category_budget(session: Session, category_budget: CategoryBudget):
|
||||||
|
category_db: CategoryDb | None = find_category_db(session, category_budget.category)
|
||||||
|
if category_db is None:
|
||||||
|
raise ValueError(
|
||||||
|
f"No matching category in database for category: {category_budget.category.to_str()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
new_start = category_budget.valid_from or date.min
|
||||||
|
new_end = category_budget.valid_to or date.max
|
||||||
|
|
||||||
|
existing_budgets = (
|
||||||
|
session.query(CategoryBudgetDb).filter_by(category_id=category_db.id).all()
|
||||||
|
)
|
||||||
|
|
||||||
|
def _add_row(amount_eur: Decimal, valid_from: date | None, valid_to: date | None):
|
||||||
|
session.add(
|
||||||
|
CategoryBudgetDb(
|
||||||
|
category_id=category_db.id,
|
||||||
|
amount_eur=amount_eur,
|
||||||
|
valid_from=valid_from,
|
||||||
|
valid_to=valid_to,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for existing_budget in existing_budgets:
|
||||||
|
existing_start = existing_budget.valid_from or date.min
|
||||||
|
existing_end = existing_budget.valid_to or date.max
|
||||||
|
|
||||||
|
overlap_start = max(new_start, existing_start)
|
||||||
|
overlap_end = min(new_end, existing_end)
|
||||||
|
|
||||||
|
if overlap_start > overlap_end:
|
||||||
|
continue # no overlap
|
||||||
|
|
||||||
|
session.delete(existing_budget)
|
||||||
|
|
||||||
|
if existing_start < new_start:
|
||||||
|
# Keep existing_start but cut existing end before new range
|
||||||
|
_add_row(
|
||||||
|
amount_eur=existing_budget.amount_eur,
|
||||||
|
valid_from=existing_budget.valid_from,
|
||||||
|
valid_to=overlap_start - timedelta(days=1),
|
||||||
|
)
|
||||||
|
if existing_end > new_end:
|
||||||
|
# Cut exsisting start to right after the new range and keep existing_end
|
||||||
|
_add_row(
|
||||||
|
amount_eur=existing_budget.amount_eur,
|
||||||
|
valid_from=overlap_end + timedelta(days=1),
|
||||||
|
valid_to=existing_budget.valid_to,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Add the new budget
|
||||||
|
_add_row(
|
||||||
|
amount_eur=category_budget.amount_eur,
|
||||||
|
valid_from=category_budget.valid_from,
|
||||||
|
valid_to=category_budget.valid_to,
|
||||||
|
)
|
||||||
+5
-22
@@ -2,9 +2,11 @@ from pathlib import Path
|
|||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from src.ingest._common import _reader_raw, _upsert
|
from src.db_utils import upsert
|
||||||
|
from src.ingest.common import _reader_raw
|
||||||
from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
|
from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
|
||||||
from src.models.b4w import B4wDb, B4wExport, Category, CategoryDb
|
from src.models.b4w import B4wDb, B4wExport
|
||||||
|
from src.models.category import upsert_category
|
||||||
|
|
||||||
|
|
||||||
def reader_b4w(path: Path) -> list[B4wExport]:
|
def reader_b4w(path: Path) -> list[B4wExport]:
|
||||||
@@ -12,25 +14,6 @@ def reader_b4w(path: Path) -> list[B4wExport]:
|
|||||||
return list(map(B4wExport.model_validate, raw))
|
return list(map(B4wExport.model_validate, raw))
|
||||||
|
|
||||||
|
|
||||||
def upsert_category(
|
|
||||||
session: Session, category: Category | None, do_commit=True
|
|
||||||
) -> CategoryDb | None:
|
|
||||||
if category is None:
|
|
||||||
return None
|
|
||||||
parameters = {
|
|
||||||
"name": category.name,
|
|
||||||
"parent": upsert_category(session, category.parent),
|
|
||||||
}
|
|
||||||
return _upsert(
|
|
||||||
session,
|
|
||||||
CategoryDb,
|
|
||||||
parameters,
|
|
||||||
parameters,
|
|
||||||
do_update=False,
|
|
||||||
do_commit=do_commit,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def upsert_b4w(
|
def upsert_b4w(
|
||||||
session: Session, data: B4wExport, do_update=False, do_commit=True
|
session: Session, data: B4wExport, do_update=False, do_commit=True
|
||||||
) -> B4wDb | None:
|
) -> B4wDb | None:
|
||||||
@@ -54,7 +37,7 @@ def upsert_b4w(
|
|||||||
"notes": data.notes,
|
"notes": data.notes,
|
||||||
}
|
}
|
||||||
|
|
||||||
return _upsert(
|
return upsert(
|
||||||
session,
|
session,
|
||||||
B4wDb,
|
B4wDb,
|
||||||
parameters_existing,
|
parameters_existing,
|
||||||
|
|||||||
+18
-9
@@ -6,18 +6,18 @@ from typing import cast
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from settings import BANK_ACCOUNTS
|
from settings import BANK_ACCOUNTS
|
||||||
from src.ingest._common import _upsert
|
from src.db_utils import upsert
|
||||||
from src.ingest.transactions import (
|
from src.ingest.transactions import (
|
||||||
consolidate_transactions,
|
consolidate_transactions,
|
||||||
map_comdirect_giro_to_transaction,
|
map_comdirect_giro_to_transaction,
|
||||||
)
|
)
|
||||||
from src.models._common import Csv2Dict, CsV2DictMulti
|
|
||||||
from src.models.comdirect import (
|
from src.models.comdirect import (
|
||||||
ComdirectCreditCard,
|
ComdirectCreditCard,
|
||||||
ComdirectDepot,
|
ComdirectDepot,
|
||||||
ComdirectGiro,
|
ComdirectGiro,
|
||||||
ComdirectGiroDb,
|
ComdirectGiroDb,
|
||||||
)
|
)
|
||||||
|
from src.models.common import Csv2Dict, CsV2DictMulti
|
||||||
from src.models.settings import Account, Source
|
from src.models.settings import Account, Source
|
||||||
|
|
||||||
PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$'
|
PATTERN_ACCOUNT_NAME = r'^"Umsätze ([^"]+)";"Zeitraum: .*$'
|
||||||
@@ -73,6 +73,10 @@ def _reader_raw_comdirect(path: Path) -> CsV2DictMulti:
|
|||||||
account_name: str | None = None
|
account_name: str | None = None
|
||||||
header: list[str] = []
|
header: list[str] = []
|
||||||
result: CsV2DictMulti = {}
|
result: CsV2DictMulti = {}
|
||||||
|
# Comdirect groups "new" transactions (marked with "neu") under a header
|
||||||
|
# line that carries only their Buchungstag; remember it until the next
|
||||||
|
# transaction row for the same account.
|
||||||
|
pending_book_date: dict[str, str] = {}
|
||||||
for line in lines:
|
for line in lines:
|
||||||
if expect_next == EXPECT_LINE.ACCOUNT_NAME:
|
if expect_next == EXPECT_LINE.ACCOUNT_NAME:
|
||||||
matches = re.findall(PATTERN_ACCOUNT_NAME, line)
|
matches = re.findall(PATTERN_ACCOUNT_NAME, line)
|
||||||
@@ -91,13 +95,18 @@ def _reader_raw_comdirect(path: Path) -> CsV2DictMulti:
|
|||||||
expect_next = EXPECT_LINE.DATA
|
expect_next = EXPECT_LINE.DATA
|
||||||
elif expect_next == EXPECT_LINE.DATA:
|
elif expect_next == EXPECT_LINE.DATA:
|
||||||
data = re.findall(PATTERN_DATA, line)
|
data = re.findall(PATTERN_DATA, line)
|
||||||
if data:
|
if not data:
|
||||||
assert account_name is not None
|
|
||||||
result[account_name].append(
|
|
||||||
{column: value for column, value in zip(header, data)}
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
expect_next = EXPECT_LINE.ACCOUNT_NAME
|
expect_next = EXPECT_LINE.ACCOUNT_NAME
|
||||||
|
elif len(data) == 1:
|
||||||
|
# Group header line (single Buchungstag for the "neu" rows below).
|
||||||
|
assert account_name is not None
|
||||||
|
pending_book_date[account_name] = data[0]
|
||||||
|
else:
|
||||||
|
assert account_name is not None
|
||||||
|
row = {column: value for column, value in zip(header, data)}
|
||||||
|
if data[0] == "neu" and account_name in pending_book_date:
|
||||||
|
row["Buchungstag"] = pending_book_date.pop(account_name)
|
||||||
|
result[account_name].append(row)
|
||||||
else:
|
else:
|
||||||
expect_next = EXPECT_LINE.ACCOUNT_NAME
|
expect_next = EXPECT_LINE.ACCOUNT_NAME
|
||||||
return result
|
return result
|
||||||
@@ -128,7 +137,7 @@ def upsert_comdirect_giro(
|
|||||||
"text": data.text,
|
"text": data.text,
|
||||||
}
|
}
|
||||||
|
|
||||||
return _upsert(
|
return upsert(
|
||||||
session,
|
session,
|
||||||
ComdirectGiroDb,
|
ComdirectGiroDb,
|
||||||
parameters_existing,
|
parameters_existing,
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
import csv
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _reader_raw(path: Path):
|
||||||
|
with path.open() as fid:
|
||||||
|
return list(csv.DictReader(fid))
|
||||||
@@ -4,9 +4,9 @@ from sqlalchemy import select
|
|||||||
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from src.models._common import BaseModelDb
|
|
||||||
from src.models.b4w import B4wDb
|
from src.models.b4w import B4wDb
|
||||||
from src.models.comdirect import ComdirectGiroDb
|
from src.models.comdirect import ComdirectGiroDb
|
||||||
|
from src.models.common import BaseModelDb
|
||||||
from src.models.transactions import Transaction, TransactionDb
|
from src.models.transactions import Transaction, TransactionDb
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+35
-1
@@ -1,5 +1,39 @@
|
|||||||
|
from importlib import import_module
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
_EXPORTS = {
|
||||||
|
"B4wExport": ".b4w",
|
||||||
|
"Category": ".category",
|
||||||
|
"CategoryBudgetDb": ".category_budget",
|
||||||
|
"ComdirectCreditCard": ".comdirect",
|
||||||
|
"ComdirectDepot": ".comdirect",
|
||||||
|
"ComdirectGiro": ".comdirect",
|
||||||
|
"Account": ".settings",
|
||||||
|
"Source": ".settings",
|
||||||
|
"Transaction": ".transactions",
|
||||||
|
"TransactionDb": ".transactions",
|
||||||
|
}
|
||||||
|
__all__ = list(_EXPORTS)
|
||||||
|
|
||||||
|
|
||||||
|
def __getattr__(name: str) -> Any:
|
||||||
|
module_name = _EXPORTS.get(name)
|
||||||
|
if module_name is None:
|
||||||
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||||
|
|
||||||
|
value = getattr(import_module(module_name, __name__), name)
|
||||||
|
globals()[name] = value
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def __dir__() -> list[str]:
|
||||||
|
return sorted([*globals(), *__all__])
|
||||||
|
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
from src.models.b4w import B4wExport as B4wExport
|
from src.models.b4w import B4wExport as B4wExport
|
||||||
from src.models.b4w import Category as Category
|
from src.models.category import Category as Category
|
||||||
|
from src.models.category import CategoryBudgetDb as CategoryBudgetDb
|
||||||
from src.models.comdirect import ComdirectCreditCard as ComdirectCreditCard
|
from src.models.comdirect import ComdirectCreditCard as ComdirectCreditCard
|
||||||
from src.models.comdirect import ComdirectDepot as ComdirectDepot
|
from src.models.comdirect import ComdirectDepot as ComdirectDepot
|
||||||
from src.models.comdirect import ComdirectGiro as ComdirectGiro
|
from src.models.comdirect import ComdirectGiro as ComdirectGiro
|
||||||
|
|||||||
+5
-55
@@ -2,17 +2,17 @@ from datetime import date
|
|||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
|
|
||||||
from pydantic import Field, computed_field, field_validator
|
from pydantic import Field, computed_field, field_validator
|
||||||
from sqlalchemy import Date, ForeignKey, Integer, String, UniqueConstraint
|
from sqlalchemy import Date, ForeignKey, Integer, String
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from src.constants import IBAN_PATTERN, Currency
|
from src.constants import IBAN_PATTERN, Currency
|
||||||
from src.models._common import (
|
from src.models.category import Category
|
||||||
BaseModelDb,
|
from src.models.common import (
|
||||||
ExportBaseModel,
|
ExportBaseModel,
|
||||||
IngestdbBase,
|
IngestdbBase,
|
||||||
Sign,
|
Sign,
|
||||||
_hash,
|
|
||||||
date2str,
|
date2str,
|
||||||
|
hashify,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -59,7 +59,7 @@ class B4wExport(ExportBaseModel):
|
|||||||
@computed_field # type: ignore[prop-decorator]
|
@computed_field # type: ignore[prop-decorator]
|
||||||
@property
|
@property
|
||||||
def hash_id(self) -> str:
|
def hash_id(self) -> str:
|
||||||
return _hash(self.id)
|
return hashify(self.id)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def amount_signed(self) -> Decimal:
|
def amount_signed(self) -> Decimal:
|
||||||
@@ -87,56 +87,6 @@ class B4wExport(ExportBaseModel):
|
|||||||
return category
|
return category
|
||||||
|
|
||||||
|
|
||||||
class Category(ExportBaseModel):
|
|
||||||
name: str = Field(pattern=r"[^:\r\n]+")
|
|
||||||
parent: Category | None = None
|
|
||||||
|
|
||||||
@computed_field # type: ignore[prop-decorator]
|
|
||||||
@property
|
|
||||||
def hash_id(self) -> str:
|
|
||||||
return _hash(self.name, self.parent)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_str(cls, s: str) -> Category | None:
|
|
||||||
if not s:
|
|
||||||
return None
|
|
||||||
components = s.split(":")
|
|
||||||
name = components[-1]
|
|
||||||
rest = ":".join(components[:-1])
|
|
||||||
return cls(name=name, parent=cls.from_str(rest))
|
|
||||||
|
|
||||||
def to_str(self) -> str | None:
|
|
||||||
if not self.parent:
|
|
||||||
return self.name
|
|
||||||
else:
|
|
||||||
return f"{self.parent.to_str()}:{self.name}"
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
return self.to_str()
|
|
||||||
|
|
||||||
def __repr__(self):
|
|
||||||
return f'Category.from_str("{self.__str__()}")'
|
|
||||||
|
|
||||||
|
|
||||||
class CategoryDb(BaseModelDb):
|
|
||||||
__tablename__ = "category"
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint("name", "parent_id", name="uq_category_name_parent"),
|
|
||||||
)
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
|
||||||
name: Mapped[str] = mapped_column(String, nullable=False)
|
|
||||||
parent_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("category.id"), nullable=True
|
|
||||||
)
|
|
||||||
|
|
||||||
# Relationships
|
|
||||||
parent = relationship(
|
|
||||||
"CategoryDb", remote_side="CategoryDb.id", back_populates="children"
|
|
||||||
)
|
|
||||||
children = relationship("CategoryDb", back_populates="parent")
|
|
||||||
|
|
||||||
|
|
||||||
class B4wDb(IngestdbBase):
|
class B4wDb(IngestdbBase):
|
||||||
__tablename__ = "import_b4w"
|
__tablename__ = "import_b4w"
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
from datetime import date
|
||||||
|
from decimal import Decimal
|
||||||
|
from typing import Self
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field, computed_field, model_validator
|
||||||
|
from sqlalchemy import Date, ForeignKey, Numeric, String, UniqueConstraint
|
||||||
|
from sqlalchemy.orm import Mapped, Session, mapped_column, relationship
|
||||||
|
|
||||||
|
from src.db_utils import upsert
|
||||||
|
from src.models.common import BaseModelDb, ExportBaseModel, hashify
|
||||||
|
|
||||||
|
|
||||||
|
class Category(ExportBaseModel):
|
||||||
|
name: str = Field(pattern=r"[^:\r\n]+")
|
||||||
|
parent: Category | None = None
|
||||||
|
|
||||||
|
@computed_field # type: ignore[prop-decorator]
|
||||||
|
@property
|
||||||
|
def hash_id(self) -> str:
|
||||||
|
return hashify(self.name, self.parent)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_str(cls, s: str) -> Category | None:
|
||||||
|
if not s:
|
||||||
|
return None
|
||||||
|
components = s.split(":")
|
||||||
|
name = components[-1]
|
||||||
|
rest = ":".join(components[:-1])
|
||||||
|
return cls(name=name, parent=cls.from_str(rest))
|
||||||
|
|
||||||
|
def to_str(self) -> str | None:
|
||||||
|
if not self.parent:
|
||||||
|
return self.name
|
||||||
|
else:
|
||||||
|
return f"{self.parent.to_str()}:{self.name}"
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return self.to_str()
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return f'Category.from_str("{self.__str__()}")'
|
||||||
|
|
||||||
|
|
||||||
|
class CategoryDb(BaseModelDb):
|
||||||
|
__tablename__ = "category"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("name", "parent_id", name="uq_category_name_parent"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||||
|
name: Mapped[str] = mapped_column(String, nullable=False)
|
||||||
|
parent_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("category.id"), nullable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# Relationships
|
||||||
|
parent = relationship(
|
||||||
|
"CategoryDb", remote_side="CategoryDb.id", back_populates="children"
|
||||||
|
)
|
||||||
|
children = relationship("CategoryDb", back_populates="parent")
|
||||||
|
budgets: Mapped[list[CategoryBudgetDb]] = relationship(back_populates="category")
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_category(
|
||||||
|
session: Session, category: Category | None, do_commit: bool = True
|
||||||
|
) -> CategoryDb | None:
|
||||||
|
if category is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
parent = upsert_category(session, category.parent, do_commit=False)
|
||||||
|
parameters = {"name": category.name, "parent": parent}
|
||||||
|
return upsert(
|
||||||
|
session,
|
||||||
|
CategoryDb,
|
||||||
|
parameters,
|
||||||
|
parameters,
|
||||||
|
do_update=False,
|
||||||
|
do_commit=do_commit,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class CategoryBudget(BaseModel):
|
||||||
|
category: Category
|
||||||
|
amount_eur: Decimal = Field(gt=0, decimal_places=2)
|
||||||
|
valid_from: date | None = Field(
|
||||||
|
description="Start of validity. None means unbounded"
|
||||||
|
)
|
||||||
|
valid_to: date | None = Field(description="End of validity. None means unbounded")
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_date_range(self) -> Self:
|
||||||
|
if (
|
||||||
|
self.valid_from is not None
|
||||||
|
and self.valid_to is not None
|
||||||
|
and self.valid_from > self.valid_to
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
f"`valid_from` ({self.valid_from}) and valid_to ({self.valid_to}) are nor in order"
|
||||||
|
)
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class CategoryBudgetDb(BaseModelDb):
|
||||||
|
"""Budget amount for a category, optionally bounded to a time window.
|
||||||
|
|
||||||
|
Satellite table to ``category``: one category may have several budgets
|
||||||
|
(e.g. one per month). ``valid_from``/``valid_to`` are nullable; ``None``
|
||||||
|
means unbounded on that side.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "category_budget"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||||
|
category_id: Mapped[int] = mapped_column(ForeignKey("category.id"), nullable=False)
|
||||||
|
amount_eur: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
|
||||||
|
valid_from: Mapped[date | None] = mapped_column(Date, nullable=True, default=None)
|
||||||
|
valid_to: Mapped[date | None] = mapped_column(Date, nullable=True, default=None)
|
||||||
|
|
||||||
|
# relationship
|
||||||
|
category = relationship("CategoryDb", back_populates="budgets")
|
||||||
@@ -5,14 +5,14 @@ from pydantic import ConfigDict, Field, computed_field, field_validator
|
|||||||
from sqlalchemy import Date, String
|
from sqlalchemy import Date, String
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from src.models._common import (
|
from src.models.common import (
|
||||||
ExportBaseModel,
|
ExportBaseModel,
|
||||||
IngestdbBase,
|
IngestdbBase,
|
||||||
_hash,
|
|
||||||
_parse_date_german,
|
_parse_date_german,
|
||||||
_parse_decimal_german,
|
_parse_decimal_german,
|
||||||
date2str,
|
date2str,
|
||||||
decimal2str,
|
decimal2str,
|
||||||
|
hashify,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -72,7 +72,7 @@ class ComdirectGiro(ComdirectBase):
|
|||||||
@computed_field # type: ignore[prop-decorator]
|
@computed_field # type: ignore[prop-decorator]
|
||||||
@property
|
@property
|
||||||
def hash_id(self) -> str:
|
def hash_id(self) -> str:
|
||||||
return _hash(self.date_val, self.date_book, self.text)
|
return hashify(self.date_val, self.date_book, self.text)
|
||||||
|
|
||||||
@field_validator("date_book", mode="before")
|
@field_validator("date_book", mode="before")
|
||||||
@classmethod
|
@classmethod
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from sqlalchemy import Date, Numeric, String
|
|||||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||||
|
|
||||||
|
|
||||||
def _hash(*args):
|
def hashify(*args):
|
||||||
raw = "".join(str(args))
|
raw = "".join(str(args))
|
||||||
return hashlib.sha256(raw.encode()).hexdigest()
|
return hashlib.sha256(raw.encode()).hexdigest()
|
||||||
|
|
||||||
@@ -5,7 +5,7 @@ from pydantic import Field, computed_field
|
|||||||
from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint
|
from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from src.models._common import ExportBaseModel, IngestdbBase, _hash, date2str
|
from src.models.common import ExportBaseModel, IngestdbBase, date2str, hashify
|
||||||
|
|
||||||
|
|
||||||
class Transaction(ExportBaseModel):
|
class Transaction(ExportBaseModel):
|
||||||
@@ -33,7 +33,7 @@ class Transaction(ExportBaseModel):
|
|||||||
@computed_field # type: ignore[prop-decorator]
|
@computed_field # type: ignore[prop-decorator]
|
||||||
@property
|
@property
|
||||||
def hash_id(self) -> str:
|
def hash_id(self) -> str:
|
||||||
return _hash(
|
return hashify(
|
||||||
self.source,
|
self.source,
|
||||||
self.account_iban,
|
self.account_iban,
|
||||||
self.date_valid,
|
self.date_valid,
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
from datetime import date
|
||||||
|
from decimal import Decimal
|
||||||
|
|
||||||
|
from sqlalchemy import inspect
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from src.models.category import CategoryBudgetDb, CategoryDb
|
||||||
|
|
||||||
|
|
||||||
|
def test_category_budget_schema(db_session: Session):
|
||||||
|
inspector = inspect(db_session.get_bind())
|
||||||
|
columns = {c["name"]: c for c in inspector.get_columns("category_budget")}
|
||||||
|
|
||||||
|
assert set(columns) == {
|
||||||
|
"id",
|
||||||
|
"category_id",
|
||||||
|
"amount_eur",
|
||||||
|
"valid_from",
|
||||||
|
"valid_to",
|
||||||
|
}
|
||||||
|
assert columns["id"]["nullable"] is False
|
||||||
|
assert columns["category_id"]["nullable"] is False
|
||||||
|
assert columns["amount_eur"]["nullable"] is False
|
||||||
|
assert columns["valid_from"]["nullable"] is True
|
||||||
|
assert columns["valid_to"]["nullable"] is True
|
||||||
|
assert str(columns["amount_eur"]["type"]) == "NUMERIC(12, 2)"
|
||||||
|
|
||||||
|
foreign_keys = inspector.get_foreign_keys("category_budget")
|
||||||
|
assert len(foreign_keys) == 1
|
||||||
|
assert foreign_keys[0]["constrained_columns"] == ["category_id"]
|
||||||
|
assert foreign_keys[0]["referred_table"] == "category"
|
||||||
|
|
||||||
|
|
||||||
|
def test_category_budget_defaults_null_dates(db_session: Session):
|
||||||
|
category = CategoryDb(name="Lebenshaltung")
|
||||||
|
db_session.add(category)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
budget = CategoryBudgetDb(category_id=category.id, amount_eur=Decimal("150.00"))
|
||||||
|
db_session.add(budget)
|
||||||
|
db_session.commit()
|
||||||
|
db_session.refresh(budget)
|
||||||
|
|
||||||
|
assert budget.id is not None
|
||||||
|
assert budget.category_id == category.id
|
||||||
|
assert budget.amount_eur == Decimal("150.00")
|
||||||
|
# Null means unbounded in either direction.
|
||||||
|
assert budget.valid_from is None
|
||||||
|
assert budget.valid_to is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_category_budget_date_window(db_session: Session):
|
||||||
|
category = CategoryDb(name="Freizeit")
|
||||||
|
db_session.add(category)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
budget = CategoryBudgetDb(
|
||||||
|
category_id=category.id,
|
||||||
|
amount_eur=Decimal("200.50"),
|
||||||
|
valid_from=date(2026, 10, 1),
|
||||||
|
valid_to=date(2026, 10, 31),
|
||||||
|
)
|
||||||
|
db_session.add(budget)
|
||||||
|
db_session.commit()
|
||||||
|
db_session.refresh(budget)
|
||||||
|
|
||||||
|
assert budget.valid_from == date(2026, 10, 1)
|
||||||
|
assert budget.valid_to == date(2026, 10, 31)
|
||||||
|
|
||||||
|
|
||||||
|
def test_category_budget_relationship_both_ways(db_session: Session):
|
||||||
|
category = CategoryDb(name="Beruf")
|
||||||
|
db_session.add(category)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
budget = CategoryBudgetDb(category_id=category.id, amount_eur=Decimal("1.00"))
|
||||||
|
db_session.add(budget)
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
# Force re-read from the database so the relationship is loaded, not
|
||||||
|
# just resolved from the session identity map.
|
||||||
|
db_session.expire_all()
|
||||||
|
assert [b.id for b in category.budgets] == [budget.id]
|
||||||
|
assert budget.category.id == category.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_category_has_multiple_budgets(db_session: Session):
|
||||||
|
category = CategoryDb(name="Wohnen")
|
||||||
|
db_session.add(category)
|
||||||
|
db_session.flush()
|
||||||
|
|
||||||
|
budgets = [
|
||||||
|
CategoryBudgetDb(
|
||||||
|
category_id=category.id,
|
||||||
|
amount_eur=Decimal("1200.00"),
|
||||||
|
valid_from=date(2026, 10, 1),
|
||||||
|
valid_to=date(2026, 10, 31),
|
||||||
|
),
|
||||||
|
CategoryBudgetDb(
|
||||||
|
category_id=category.id,
|
||||||
|
amount_eur=Decimal("1300.00"),
|
||||||
|
valid_from=date(2026, 11, 1),
|
||||||
|
valid_to=date(2026, 11, 30),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for budget in budgets:
|
||||||
|
db_session.add(budget)
|
||||||
|
db_session.commit()
|
||||||
|
db_session.expire_all()
|
||||||
|
|
||||||
|
assert [b.id for b in category.budgets] == [b.id for b in budgets]
|
||||||
@@ -1,47 +0,0 @@
|
|||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from src.ingest.b4w import upsert_category
|
|
||||||
from src.models import Category
|
|
||||||
from src.models.b4w import CategoryDb
|
|
||||||
|
|
||||||
|
|
||||||
def test_upsert_category_returns_orm_object(db_session: Session):
|
|
||||||
result = upsert_category(
|
|
||||||
db_session,
|
|
||||||
Category.from_str("dummy_parent_1:dummy_parent_2:dummy"),
|
|
||||||
do_commit=False,
|
|
||||||
)
|
|
||||||
assert result
|
|
||||||
assert isinstance(result, CategoryDb)
|
|
||||||
assert str(result.name) == "dummy"
|
|
||||||
assert str(result.parent.name) == "dummy_parent_2"
|
|
||||||
assert str(result.parent.parent.name) == "dummy_parent_1"
|
|
||||||
|
|
||||||
|
|
||||||
def test_upsert_category_writes_new_to_db(db_session: Session):
|
|
||||||
upsert_category(
|
|
||||||
db_session,
|
|
||||||
Category.from_str("grand_father:father:new_category"),
|
|
||||||
do_commit=True,
|
|
||||||
)
|
|
||||||
result = db_session.execute(
|
|
||||||
select(CategoryDb).filter_by(name="new_category")
|
|
||||||
).scalar_one()
|
|
||||||
assert result
|
|
||||||
assert str(result.name) == "new_category"
|
|
||||||
assert str(result.parent.name) == "father"
|
|
||||||
assert str(result.parent.parent.name) == "grand_father"
|
|
||||||
|
|
||||||
|
|
||||||
def test_upsert_category_skips_existing(db_session: Session):
|
|
||||||
for _ in range(3):
|
|
||||||
upsert_category(
|
|
||||||
db_session,
|
|
||||||
Category.from_str("grand_father:father:try_again_and_again"),
|
|
||||||
do_commit=True,
|
|
||||||
)
|
|
||||||
result = db_session.execute(
|
|
||||||
select(CategoryDb).filter_by(name="try_again_and_again")
|
|
||||||
).all()
|
|
||||||
assert len(result) == 1
|
|
||||||
@@ -1,56 +1,45 @@
|
|||||||
from src.models import Category
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from src.models.category import Category, CategoryDb, upsert_category
|
||||||
|
|
||||||
|
|
||||||
def test_single_level_category():
|
def test_upsert_category_returns_orm_object(db_session: Session):
|
||||||
c = Category.from_str("single")
|
result = upsert_category(
|
||||||
assert c
|
db_session,
|
||||||
assert c.name == "single"
|
Category.from_str("dummy_parent_1:dummy_parent_2:dummy"),
|
||||||
assert c.parent is None
|
do_commit=False,
|
||||||
|
|
||||||
|
|
||||||
def test_two_level_category():
|
|
||||||
c = Category.from_str("first:second")
|
|
||||||
assert c
|
|
||||||
assert c.name == "second"
|
|
||||||
assert isinstance(c.parent, Category)
|
|
||||||
assert c.parent.name == "first"
|
|
||||||
assert c.parent.parent is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_three_level_category():
|
|
||||||
c = Category.from_str("first:second:third")
|
|
||||||
assert c
|
|
||||||
assert c.name == "third"
|
|
||||||
assert isinstance(c.parent, Category)
|
|
||||||
assert c.parent.name == "second"
|
|
||||||
assert isinstance(c.parent.parent, Category)
|
|
||||||
assert c.parent.parent.name == "first"
|
|
||||||
assert c.parent.parent.parent is None
|
|
||||||
|
|
||||||
|
|
||||||
def test_single_level_category_to_string():
|
|
||||||
c = Category.from_str("single")
|
|
||||||
assert c
|
|
||||||
assert str(c) == "single"
|
|
||||||
|
|
||||||
|
|
||||||
def test_two_level_category_to_string():
|
|
||||||
c = Category(name="second", parent=Category(name="first"))
|
|
||||||
assert c.to_str() == "first:second"
|
|
||||||
|
|
||||||
|
|
||||||
def test_three_level_category_to_string():
|
|
||||||
c = Category(
|
|
||||||
name="third", parent=Category(name="second", parent=Category(name="first"))
|
|
||||||
)
|
)
|
||||||
assert c.to_str() == "first:second:third"
|
assert result
|
||||||
|
assert isinstance(result, CategoryDb)
|
||||||
|
assert str(result.name) == "dummy"
|
||||||
|
assert str(result.parent.name) == "dummy_parent_2"
|
||||||
|
assert str(result.parent.parent.name) == "dummy_parent_1"
|
||||||
|
|
||||||
|
|
||||||
def test_category_str():
|
def test_upsert_category_writes_new_to_db(db_session: Session):
|
||||||
c = Category.from_str("first:second:third")
|
upsert_category(
|
||||||
assert str(c) == "first:second:third"
|
db_session,
|
||||||
|
Category.from_str("grand_father:father:new_category"),
|
||||||
|
do_commit=True,
|
||||||
|
)
|
||||||
|
result = db_session.execute(
|
||||||
|
select(CategoryDb).filter_by(name="new_category")
|
||||||
|
).scalar_one()
|
||||||
|
assert result
|
||||||
|
assert str(result.name) == "new_category"
|
||||||
|
assert str(result.parent.name) == "father"
|
||||||
|
assert str(result.parent.parent.name) == "grand_father"
|
||||||
|
|
||||||
|
|
||||||
def test_category_repr():
|
def test_upsert_category_skips_existing(db_session: Session):
|
||||||
c = Category.from_str("first:second:third")
|
for _ in range(3):
|
||||||
assert repr(c) == 'Category.from_str("first:second:third")'
|
upsert_category(
|
||||||
|
db_session,
|
||||||
|
Category.from_str("grand_father:father:try_again_and_again"),
|
||||||
|
do_commit=True,
|
||||||
|
)
|
||||||
|
result = db_session.execute(
|
||||||
|
select(CategoryDb).filter_by(name="try_again_and_again")
|
||||||
|
).all()
|
||||||
|
assert len(result) == 1
|
||||||
|
|||||||
Reference in New Issue
Block a user