Compare commits

..
Author SHA1 Message Date
mathias2784 e5534aba34 merge main incl. conflict resolution
Tests / test (push) Successful in 19s
Tests / lint (push) Successful in 16s
2026-10-27 19:01:45 +01:00
mathias2784 40e5a05ec9 exclude tests from mypy check 2026-10-27 18:43:51 +01:00
mathias2784 048a6c6f38 Add temporal validity and lifecycle tracking to categories 2026-10-27 18:39:53 +01:00
mathias2784 d4ebab96d6 Fix category budget overlaps and document the uv run test workflow.
Tests / test (push) Successful in 11s
Tests / lint (push) Successful in 12s
2026-10-18 09:40:31 +02:00
mathias2784 ea7854473f Add category-list factory and seed categories in B4W smoke test
Tests / test (push) Successful in 23s
Tests / lint (push) Failing after 15s
2026-10-18 09:28:19 +02:00
mathias2784 c0adc77f56 initial load of categories
Tests / test (push) Successful in 11s
Tests / lint (push) Successful in 11s
2026-10-18 08:43:45 +02:00
mathias2784 1fcaec8daf mypy black ruff
Tests / test (push) Successful in 11s
Tests / lint (push) Successful in 11s
2026-10-16 22:51:11 +02:00
mathias2784 553e1a4c83 Refactor domain_utils.py overlap helpers, add ONE_DAY constant, remove inflated comments, and standardize _add_row() calls
Tests / test (push) Successful in 11s
Tests / lint (push) Failing after 7s
2026-10-16 22:30:36 +02:00
mathias2784 98a5e9332f some fixes
Tests / test (push) Successful in 16s
Tests / lint (push) Failing after 12s
2026-10-16 22:15:19 +02:00
mathias2784 2c7f8f4a67 Add tests for category budgets and new functionality
Tests / test (push) Successful in 10s
Tests / lint (push) Failing after 7s
New test files added:
- test_category_edge_cases.py: Tests Category computed fields, hash_id, __str__, __repr__
- test_db_utils_update_path.py: Tests for updated upsert() path handling
- test_models_common.py: Tests Sign enum, date2str, ExportBaseModel methods
- test_models_init.py: Tests for lazy loading from models.__init__.py

Code changes:
- Fixed CategoryBudget to have default=None for optional valid_from/valid_to fields
- Updated src/models/__init__.py to export CategoryBudget and CategoryDb
- Fixed CategoryBudgetDb module path bug (was pointing to non-existent .category_budget)
- Improved db_utils.upsert() to handle partial overlaps correctly

Coverage improvements:
- New functionality coverage: 100%
- Overall codebase coverage: 86% (up from 83%)

Notes:
- Some existing domain_utils.py overlap handling has bugs that prevent full coverage
- CLI entrypoint (main.py) and some enum methods remain uncovered but are optional
2026-10-16 21:53:38 +02:00
mathias2784 dc98ea910f utility functions for category
Tests / test (push) Successful in 16s
Tests / lint (push) Successful in 17s
2026-10-16 21:12:57 +02:00
mathias2784 da0a78d2d0 yet another refactor 2026-10-16 16:47:08 +02:00
mathias2784 5cea645062 Some more refactoring
Tests / test (push) Successful in 10s
Tests / lint (push) Successful in 11s
2026-10-16 16:27:50 +02:00
mathias2784 ef1a65adeb some refactoring 2026-10-16 15:27:41 +02:00
mathias2784 696823406e Extract Category model into src/models/category.py
Tests / test (push) Successful in 10s
Tests / lint (push) Successful in 11s
2026-10-16 12:29:11 +02:00
mathias2784 c7e4b68dfb Fix comdirect CSV reader for neu group header rows 2026-10-16 12:28:57 +02:00
mathias2784 435f7b92bf Add category_budget satellite table for time-bounded per-category budgets
Tests / test (push) Successful in 10s
Tests / lint (push) Successful in 12s
2026-10-16 11:45:13 +02:00
12 changed files with 36 additions and 347 deletions
+8 -21
View File
@@ -23,29 +23,20 @@ alembic upgrade head
```python
from src.ingest.b4w import ingest_b4w
from src.db import get_session
from pathlib import Path
csv_path = Path("/mnt/anbau/Software/Python/finanz-cockpit/scripts/dummy_data/b4w_export_dummy_01.csv")
with get_session() as session:
ingest_b4w(csv_path, session)
ingest_b4w(csv_path)
```
### Import comdirect Export
```python
from src.ingest.comdirect import ingest_comdirect
from src.db import get_session
from pathlib import Path
csv_path = Path("/mnt/anbau/Software/Python/finanz-cockpit/scripts/dummy_data/comdirect_giro_01.csv")
with get_session() as session:
ingest_comdirect(
session,
csv_path,
account_iban="DE89370400440532013000",
account_currency="EUR",
)
ingest_comdirect(csv_path)
```
### Set Category Budgets
@@ -54,7 +45,6 @@ with get_session() as session:
from src.models.category import Category, CategoryBudget
from datetime import date
from src.domain_utils import set_category_budget
from src.db import get_session
# Create category hierarchy
cat = Category.from_str("Lebenshaltung:Essen:Supermarkt")
@@ -66,17 +56,14 @@ budget = CategoryBudget(
valid_from=date(2026, 10, 1),
valid_to=date(2026, 11, 30),
)
with get_session() as session:
set_category_budget(session, budget)
session.commit()
set_category_budget(db_session, budget)
# Budgets can also be unbounded (no dates specified)
budget = CategoryBudget(
# Budgets can also be unbounded (no dates specified)
budget = CategoryBudget(
category=cat,
amount_eur=200.00, # Monthly average
)
set_category_budget(session, budget)
session.commit()
)
set_category_budget(db_session, budget)
```
## Testing
@@ -154,7 +141,7 @@ alembic upgrade head
pytest
# Run with coverage report
pytest --cov=src --cov-report=html:htmlcov
pytest --cov=src --cov-report=html -c .coverage htmlcov
```
### Common Patterns
-27
View File
@@ -1,27 +0,0 @@
{
"month": "2026-10",
"categories": [
{
"category": "super-category-1",
"total_amount": 100,
"total_budget": 120,
"sub_categories": [
{
"category": "category-1",
"total_amount": 60,
"total_budget": 50,
"sub_categories": [],
"transactions": []
},
{
"category": "category-2",
"total_amount": 40,
"total_budget": 60,
"sub_categories": [],
"transactions": []
}
],
"transactions": []
}
]
}
+1 -2
View File
@@ -29,8 +29,7 @@ def find_category_db(session: Session, category: Category | None) -> CategoryDb
return (
session.query(CategoryDb)
.filter_by(name=node.name, parent_id=parent_id)
.filter(CategoryDb.valid_to.is_(None))
.one_or_none()
.first()
)
return find_existing(category)
+4 -23
View File
@@ -5,21 +5,11 @@ from sqlalchemy.dialects.sqlite import insert as sqlite_insert
from sqlalchemy.orm import Session
from src.models.b4w import B4wDb
from src.models.category import Category, CategoryDb
from src.models.comdirect import ComdirectGiroDb
from src.models.common import BaseModelDb
from src.models.transactions import Transaction, TransactionDb
def _category_from_db(category_db: CategoryDb | None) -> Category | None:
if category_db is None:
return None
return Category(
name=category_db.name,
parent=_category_from_db(category_db.parent),
)
def map_b4w_to_transaction(row: B4wDb) -> Transaction:
"""Project one staging row of ``import_b4w`` onto a canonical Transaction."""
return Transaction(
@@ -34,7 +24,7 @@ def map_b4w_to_transaction(row: B4wDb) -> Transaction:
text=row.text,
other_name=row.other_name,
other_iban=row.other_iban,
category=_category_from_db(row.category),
category_id=row.category_id,
)
@@ -54,11 +44,7 @@ def map_comdirect_giro_to_transaction(row: ComdirectGiroDb) -> Transaction:
def upsert_transaction(
session: Session,
transaction: Transaction,
*,
category_db: CategoryDb | None = None,
do_commit: bool = True,
session: Session, transaction: Transaction, do_commit: bool = True
) -> None:
"""Insert one canonical transaction, ignoring duplicates on ``hash_id``.
@@ -79,7 +65,7 @@ def upsert_transaction(
"text": transaction.text,
"other_name": transaction.other_name,
"other_iban": transaction.other_iban,
"category_id": category_db.id if category_db is not None else None,
"category_id": transaction.category_id,
}
stmt = sqlite_insert(TransactionDb).values(parameters)
_ = session.execute(stmt.prefix_with("OR IGNORE"))
@@ -106,11 +92,6 @@ def consolidate_transactions[StagingDbT: BaseModelDb](
transaction = mapper(staging_row)
if transaction.hash_id in existing_hashes:
continue
upsert_transaction(
session,
transaction,
category_db=getattr(staging_row, "category", None),
do_commit=False,
)
upsert_transaction(session, transaction, do_commit=False)
if do_commit:
session.commit()
+4 -8
View File
@@ -18,7 +18,7 @@ from sqlalchemy import (
from sqlalchemy.orm import Mapped, Session, mapped_column, relationship
from src.db_utils import upsert
from src.models.common import AmountPos, BaseModelDb, ExportBaseModel, hashify
from src.models.common import BaseModelDb, ExportBaseModel, hashify
class UTCDateTime(TypeDecorator[datetime]):
@@ -69,16 +69,12 @@ class Category(ExportBaseModel):
else:
return f"{self.parent.to_str()}:{self.name}"
def __str__(self) -> str:
return self.to_str() or ""
def __str__(self):
return self.to_str()
def __repr__(self):
return f'Category.from_str("{self.__str__()}")'
def get_budget(self, date_valid: date | None) -> float:
# TODO
return 0.0
class CategoryDb(BaseModelDb):
__tablename__ = "category"
@@ -191,7 +187,7 @@ def upsert_category(
class CategoryBudget(BaseModel):
category: Category
amount_eur: AmountPos
amount_eur: Decimal = Field(gt=0, decimal_places=2)
valid_from: date | None = Field(
default=None, description="Start of validity. None means unbounded"
)
-85
View File
@@ -1,85 +0,0 @@
from datetime import date
from pydantic import BaseModel, computed_field
from src.models.category import Category
from src.models.transactions import Transaction
class CategoryTree(BaseModel):
category: Category | None
sub_categories: list[CategoryTree] | None = None
transactions: list[Transaction] | None = None
@computed_field # type: ignore[prop-decorator]
@property
def total_amount(self) -> float:
result = 0.0
if self.transactions:
result += sum(
float(transaction.amount) for transaction in self.transactions
)
if self.sub_categories:
result += sum(
float(sub_category.total_amount) for sub_category in self.sub_categories
)
return result
def total_budget(self, date_valid: date) -> float:
result = 0.0
if self.category is None:
return result
result += self.category.get_budget(date_valid)
if self.sub_categories:
result += sum(
sub_category.total_budget(date_valid)
for sub_category in self.sub_categories
)
return result
def __add__(self, other: CategoryTree):
if self.category != other.category:
return
sub_trees = []
seen = []
for sub_tree_1 in self.sub_categories or []:
new = sub_tree_1
for sub_tree_2 in other.sub_categories or []:
if sub_tree_1.category == sub_tree_2.category:
new += sub_tree_2
seen.append(sub_tree_2)
break
sub_trees.append(new)
sub_trees += [
sub_tree
for sub_tree in (other.sub_categories or [])
if sub_tree not in seen
]
transactions = ((self.transactions or []) + (other.transactions or [])) or None
return CategoryTree(
category=self.category,
sub_categories=list(sub_trees),
transactions=transactions,
)
def build_category_tree(transactions: list[Transaction]) -> list[CategoryTree]:
results: dict[str, CategoryTree] = {}
for branch in map(_transaction_to_category_branch, transactions):
key = str(branch.category) if branch.category else ""
results[key] = (results[key] + branch) if key in results else branch
return list(results.values())
def _transaction_to_category_branch(transaction: Transaction) -> CategoryTree:
category = transaction.category
result = CategoryTree(category=category, transactions=[transaction])
if not category:
return result
while category.parent is not None:
category = category.parent
result = CategoryTree(category=category, sub_categories=[result])
return result
+3 -8
View File
@@ -2,9 +2,8 @@ import hashlib
from datetime import date
from decimal import Decimal
from enum import Enum
from typing import Annotated
from pydantic import BaseModel, Field
from pydantic import BaseModel
from sqlalchemy import Date, Numeric, String
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
@@ -41,15 +40,11 @@ def date2str(value: date | None, default="--", date_format="%Y-%m-%d") -> str:
return default
type AmountPos = Annotated[Decimal, Field(decimal_places=2, gt=0)]
type Amount = Annotated[Decimal, Field(decimal_places=2)]
def _parse_decimal_german(decimal_str: str) -> Amount:
def _parse_decimal_german(decimal_str: str) -> Decimal:
return Decimal(decimal_str.replace(".", "").replace(",", "."))
def decimal2str(value: Amount) -> str:
def decimal2str(value: Decimal) -> str:
value = Decimal(f"{value:.2f}")
int_part, _, frac = str(value).partition(".")
groups: list[str] = []
+5 -5
View File
@@ -1,11 +1,11 @@
from datetime import date
from decimal import Decimal
from pydantic import Field, computed_field
from sqlalchemy import Date, ForeignKey, Index, Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.models.category import Category
from src.models.common import Amount, ExportBaseModel, IngestdbBase, date2str, hashify
from src.models.common import ExportBaseModel, IngestdbBase, date2str, hashify
class Transaction(ExportBaseModel):
@@ -22,13 +22,13 @@ class Transaction(ExportBaseModel):
account_iban: str
account_currency: str
date_valid: date
amount: Amount
amount: Decimal = Field(description="Signed amount; negative = money out")
amount_currency: str
date_book: date | None = None
text: str | None = None
other_name: str | None = None
other_iban: str | None = None
category: Category | None = None
category_id: int | None = None
@computed_field # type: ignore[prop-decorator]
@property
@@ -55,7 +55,7 @@ class Transaction(ExportBaseModel):
"text": f"{self.text or ''}",
"other_name": f"{self.other_name or ''}",
"other_iban": f"{self.other_iban or ''}",
"category": (self.category.to_str() if self.category else "") or "",
"category_id": f"{self.category_id}",
}
+6 -17
View File
@@ -1,5 +1,4 @@
from datetime import date
from decimal import Decimal
import pytest
@@ -13,30 +12,24 @@ class TestCategoryComputedFields:
def test_hash_id_computed_field_simple(self):
"""Test hash_id for single-level category."""
c = Category.from_str("single")
assert c
assert hasattr(c, "hash_id")
assert isinstance(c.hash_id, str)
# Hash should be deterministic for same inputs
c2 = Category.from_str("single")
assert c2
assert c.hash_id == c2.hash_id
def test_hash_id_computed_field_nested(self):
"""Test hash_id for multi-level category hierarchy."""
c1 = Category.from_str("level1:level2:level3")
assert c1
assert hasattr(c1, "hash_id")
c2 = Category.from_str("level1:level2:level3")
assert c2
assert c1.hash_id == c2.hash_id
def test_hash_id_different_hierarchy(self):
"""Test that different hierarchies have different hash_ids."""
c1 = Category.from_str("A:B")
c2 = Category.from_str("A:C")
assert c1
assert c2
# Different last level -> different hash_id
assert c1.hash_id != c2.hash_id
@@ -47,26 +40,22 @@ class TestCategoryComputedFields:
# Create another with same name but no parent
orphan = Category.from_str("Budget")
assert orphan
assert cat1.hash_id != orphan.hash_id
def test_to_str_single_level(self):
"""Test to_str() for single-level category."""
c = Category.from_str("single")
assert c
assert c.to_str() == "single"
def test_to_str_nested(self):
"""Test to_str() for nested category."""
c = Category.from_str("parent:child:grandchild")
assert c
assert c.to_str() == "parent:child:grandchild"
def test_str_uses_to_str(self):
"""Test that __str__ delegates to to_str()."""
c = Category.from_str("A:B:C")
assert c
assert str(c) == c.to_str() == "A:B:C"
def test_repr_simple(self):
@@ -90,7 +79,7 @@ class TestCategoryBudgetValidation:
cat = CategoryFactory.build()
budget = CategoryBudget(
category=cat,
amount_eur=Decimal("100.00"),
amount_eur=100.00,
)
assert budget.valid_from is None
assert budget.valid_to is None
@@ -101,7 +90,7 @@ class TestCategoryBudgetValidation:
cat = CategoryFactory.build()
budget = CategoryBudget(
category=cat,
amount_eur=Decimal("100.00"),
amount_eur=100.00,
valid_from=date(2026, 10, 1),
valid_to=date(2026, 10, 31),
)
@@ -116,7 +105,7 @@ class TestCategoryBudgetValidation:
with pytest.raises(ValueError) as exc_info:
CategoryBudget(
category=cat,
amount_eur=Decimal("100.00"),
amount_eur=100.00,
valid_from=date(2026, 10, 31),
valid_to=date(2026, 10, 1),
)
@@ -131,7 +120,7 @@ class TestCategoryBudgetValidation:
cat = CategoryFactory.build()
budget = CategoryBudget(
category=cat,
amount_eur=Decimal("100.00"),
amount_eur=100.00,
valid_from=date(2026, 10, 1),
valid_to=date(2026, 10, 1),
)
@@ -144,7 +133,7 @@ class TestCategoryBudgetValidation:
cat = CategoryFactory.build()
budget = CategoryBudget(
category=cat,
amount_eur=Decimal("150.75"),
amount_eur=150.75,
valid_from=date(2026, 9, 1),
valid_to=date(2026, 9, 30),
)
@@ -153,7 +142,7 @@ class TestCategoryBudgetValidation:
try:
CategoryBudget(
category=cat,
amount_eur=Decimal("200.00"),
amount_eur=200.00,
valid_from=date(2026, 12, 31),
valid_to=date(2026, 12, 30),
)
-116
View File
@@ -1,116 +0,0 @@
from src.models.category import Category
from src.models.category_tree import (
CategoryTree,
_transaction_to_category_branch,
build_category_tree,
)
from src.models.transactions import Transaction
from tests.factories import TransactionFactory
def test_transaction_to_category_branch_simple():
transaction = TransactionFactory.build(category=Category.from_str("dummy category"))
result = _transaction_to_category_branch(transaction)
assert isinstance(result, CategoryTree)
assert result.category
assert result.category.name == "dummy category"
assert result.transactions
assert isinstance(result.transactions, list)
assert len(result.transactions) == 1
assert isinstance(result.transactions[0], Transaction)
def test_transaction_to_category_branch_two_category_levels():
transaction = TransactionFactory.build(category=Category.from_str("father:son"))
result = _transaction_to_category_branch(transaction)
assert isinstance(result, CategoryTree)
assert result.category
assert result.category.name == "father"
assert result.transactions is None
assert result.sub_categories
assert isinstance(result.sub_categories, list)
assert len(result.sub_categories) == 1
assert isinstance(result.sub_categories[0], CategoryTree)
assert isinstance(result.sub_categories[0].category, Category)
assert result.sub_categories[0].category.name == "son"
assert result.sub_categories[0].transactions
assert isinstance(result.sub_categories[0].transactions, list)
assert len(result.sub_categories[0].transactions) == 1
assert isinstance(result.sub_categories[0].transactions[0], Transaction)
assert result.sub_categories[0].transactions[0] == transaction
def test_transaction_to_category_branch_no_category():
transaction = TransactionFactory.build(category=None)
result = _transaction_to_category_branch(transaction)
assert result.category is None
assert result.transactions
assert isinstance(result.transactions, list)
assert len(result.transactions) == 1
assert result.transactions[0] == transaction
def test_category_tree_add():
category_tree_1 = CategoryTree(
category=Category.from_str("super"),
sub_categories=[
CategoryTree(
category=Category.from_str("super:sub1"),
transactions=[TransactionFactory.build(amount=10)],
),
CategoryTree(
category=Category.from_str("super:sub2"),
transactions=[TransactionFactory.build(amount=30)],
),
],
)
category_tree_2 = CategoryTree(
category=Category.from_str("super"),
sub_categories=[
CategoryTree(
category=Category.from_str("super:sub2"),
transactions=[TransactionFactory.build(amount=20)],
),
CategoryTree(
category=Category.from_str("super:sub3"),
transactions=[TransactionFactory.build(amount=100)],
),
],
)
result = category_tree_1 + category_tree_2
assert result
assert result.sub_categories
assert isinstance(result.sub_categories, list)
assert len(result.sub_categories) == 3
assert all(s.category for s in result.sub_categories)
assert sorted([s.category.name for s in result.sub_categories]) == [
"sub1",
"sub2",
"sub3",
]
assert result.total_amount == 160.0
def test_build_category_tree():
transactions = [
TransactionFactory.build(category=Category.from_str("super1:sub1"), amount=10),
TransactionFactory.build(category=Category.from_str("super1:sub2"), amount=13),
TransactionFactory.build(
category=Category.from_str("super1:sub2:sub2_sub1"), amount=104
),
TransactionFactory.build(
category=Category.from_str("super1:sub2:sub2_sub1"), amount=109
),
TransactionFactory.build(category=Category.from_str("super2"), amount=5000),
]
result = build_category_tree(transactions)
assert result
assert isinstance(result, list)
assert len(result) == 2
assert isinstance(result[0], CategoryTree)
assert result[0].category
assert result[0].category.name == "super1"
assert result[0].transactions is None
assert result[0].sub_categories
assert isinstance(result[0].sub_categories, list)
assert len(result[0].sub_categories) == 2
+2 -29
View File
@@ -1,13 +1,13 @@
"""Comprehensive tests for set_category_budget overlap handling."""
from datetime import UTC, date, datetime
from datetime import date
import pytest
from sqlalchemy.orm import Session
from src.domain_utils import find_category_db, set_category_budget
from src.models import Category, CategoryBudget, CategoryBudgetDb
from src.models.category import CategoryDb, upsert_category
from src.models.category import upsert_category
from tests.factories import CategoryFactory
# NOTE: MYPY errors suppressed for test fixtures - see AGENTS.md for type checking notes.
@@ -615,33 +615,6 @@ class TestFindCategoryDb:
result = find_category_db(db_session, Category(name="NonExistent"))
assert result is None
def test_find_reintroduced_category_returns_active_version(
self, db_session: Session
):
category = Category(name="Reintroduced")
historical = CategoryDb(
name=category.name,
valid_to=datetime(2026, 1, 1, tzinfo=UTC),
last_action="retired",
)
db_session.add(historical)
db_session.commit()
active = upsert_category(db_session, category, do_commit=False)
assert active is not None
db_session.commit()
result = find_category_db(db_session, category)
assert result is not None
assert result.id == active.id
set_category_budget(
db_session,
CategoryBudget(category=category, amount_eur=100.00),
)
budget = db_session.query(CategoryBudgetDb).one()
assert budget.category_id == active.id
class TestSetCategoryBudgetErrorHandling:
"""Test error handling in set_category_budget."""
+1 -4
View File
@@ -101,7 +101,7 @@ def test_map_comdirect_giro_to_transaction():
assert transaction.amount == Decimal("32.43")
assert transaction.other_name is None
assert transaction.other_iban is None
assert transaction.category is None
assert transaction.category_id is None
def test_consolidate_b4w_signed_amount_and_category(db_session: Session):
@@ -125,9 +125,6 @@ def test_consolidate_b4w_signed_amount_and_category(db_session: Session):
assert transaction.amount == Decimal("-50.00")
assert transaction.category_id == staging.category_id
assert transaction.text == "Supermarket"
mapped = map_b4w_to_transaction(staging)
assert mapped.category is not None
assert mapped.category.to_str() == export.category.to_str()
def test_ingest_b4w_e2e_idempotent(db_session: Session):