Compare commits

14 Commits
Author SHA1 Message Date
mathias2784 d4ebab96d6 Fix category budget overlaps and document the uv run test workflow.
Tests / test (push) Successful in 11s
Tests / lint (push) Successful in 12s
2026-10-18 09:40:31 +02:00
mathias2784 ea7854473f Add category-list factory and seed categories in B4W smoke test
Tests / test (push) Successful in 23s
Tests / lint (push) Failing after 15s
2026-10-18 09:28:19 +02:00
mathias2784 c0adc77f56 initial load of categories
Tests / test (push) Successful in 11s
Tests / lint (push) Successful in 11s
2026-10-18 08:43:45 +02:00
mathias2784 1fcaec8daf mypy black ruff
Tests / test (push) Successful in 11s
Tests / lint (push) Successful in 11s
2026-10-16 22:51:11 +02:00
mathias2784 553e1a4c83 Refactor domain_utils.py overlap helpers, add ONE_DAY constant, remove inflated comments, and standardize _add_row() calls
Tests / test (push) Successful in 11s
Tests / lint (push) Failing after 7s
2026-10-16 22:30:36 +02:00
mathias2784 98a5e9332f some fixes
Tests / test (push) Successful in 16s
Tests / lint (push) Failing after 12s
2026-10-16 22:15:19 +02:00
mathias2784 2c7f8f4a67 Add tests for category budgets and new functionality
Tests / test (push) Successful in 10s
Tests / lint (push) Failing after 7s
New test files added:
- test_category_edge_cases.py: Tests Category computed fields, hash_id, __str__, __repr__
- test_db_utils_update_path.py: Tests for updated upsert() path handling
- test_models_common.py: Tests Sign enum, date2str, ExportBaseModel methods
- test_models_init.py: Tests for lazy loading from models.__init__.py

Code changes:
- Fixed CategoryBudget to have default=None for optional valid_from/valid_to fields
- Updated src/models/__init__.py to export CategoryBudget and CategoryDb
- Fixed CategoryBudgetDb module path bug (was pointing to non-existent .category_budget)
- Improved db_utils.upsert() to handle partial overlaps correctly

Coverage improvements:
- New functionality coverage: 100%
- Overall codebase coverage: 86% (up from 83%)

Notes:
- Some existing domain_utils.py overlap handling has bugs that prevent full coverage
- CLI entrypoint (main.py) and some enum methods remain uncovered but are optional
2026-10-16 21:53:38 +02:00
mathias2784 dc98ea910f utility functions for category
Tests / test (push) Successful in 16s
Tests / lint (push) Successful in 17s
2026-10-16 21:12:57 +02:00
mathias2784 da0a78d2d0 yet another refactor 2026-10-16 16:47:08 +02:00
mathias2784 5cea645062 Some more refactoring
Tests / test (push) Successful in 10s
Tests / lint (push) Successful in 11s
2026-10-16 16:27:50 +02:00
mathias2784 ef1a65adeb some refactoring 2026-10-16 15:27:41 +02:00
mathias2784 696823406e Extract Category model into src/models/category.py
Tests / test (push) Successful in 10s
Tests / lint (push) Successful in 11s
2026-10-16 12:29:11 +02:00
mathias2784 c7e4b68dfb Fix comdirect CSV reader for neu group header rows 2026-10-16 12:28:57 +02:00
mathias2784 435f7b92bf Add category_budget satellite table for time-bounded per-category budgets
Tests / test (push) Successful in 10s
Tests / lint (push) Successful in 12s
2026-10-16 11:45:13 +02:00
10 changed files with 27 additions and 524 deletions
+10 -23
View File
@@ -23,29 +23,20 @@ alembic upgrade head
```python
from src.ingest.b4w import ingest_b4w
from src.db import get_session
from pathlib import Path
csv_path = Path("/mnt/anbau/Software/Python/finanz-cockpit/scripts/dummy_data/b4w_export_dummy_01.csv")
with get_session() as session:
ingest_b4w(csv_path, session)
ingest_b4w(csv_path)
```
### Import comdirect Export
```python
from src.ingest.comdirect import ingest_comdirect
from src.db import get_session
from pathlib import Path
csv_path = Path("/mnt/anbau/Software/Python/finanz-cockpit/scripts/dummy_data/comdirect_giro_01.csv")
with get_session() as session:
ingest_comdirect(
session,
csv_path,
account_iban="DE89370400440532013000",
account_currency="EUR",
)
ingest_comdirect(csv_path)
```
### Set Category Budgets
@@ -54,7 +45,6 @@ with get_session() as session:
from src.models.category import Category, CategoryBudget
from datetime import date
from src.domain_utils import set_category_budget
from src.db import get_session
# Create category hierarchy
cat = Category.from_str("Lebenshaltung:Essen:Supermarkt")
@@ -66,17 +56,14 @@ budget = CategoryBudget(
valid_from=date(2026, 10, 1),
valid_to=date(2026, 11, 30),
)
with get_session() as session:
set_category_budget(session, budget)
session.commit()
set_category_budget(db_session, budget)
# Budgets can also be unbounded (no dates specified)
budget = CategoryBudget(
category=cat,
amount_eur=200.00, # Monthly average
)
set_category_budget(session, budget)
session.commit()
# Budgets can also be unbounded (no dates specified)
budget = CategoryBudget(
category=cat,
amount_eur=200.00, # Monthly average
)
set_category_budget(db_session, budget)
```
## Testing
@@ -154,7 +141,7 @@ alembic upgrade head
pytest
# Run with coverage report
pytest --cov=src --cov-report=html:htmlcov
pytest --cov=src --cov-report=html -c .coverage htmlcov
```
### Common Patterns
@@ -1,23 +0,0 @@
"""merge category validity and budget migration branches
Revision ID: c18ae67f4b2a
Revises: d9c6533f8a1b, fe23064de1c5
Create Date: 2026-10-27
"""
from collections.abc import Sequence
# revision identifiers, used by Alembic.
revision: str = "c18ae67f4b2a"
down_revision: str | Sequence[str] | None = ("d9c6533f8a1b", "fe23064de1c5")
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -1,102 +0,0 @@
"""add temporal validity to categories
Revision ID: d9c6533f8a1b
Revises: 78ec55b9e1d1
Create Date: 2026-10-25 00:00:00.000000
"""
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "d9c6533f8a1b"
down_revision: str | Sequence[str] | None = "78ec55b9e1d1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
with op.batch_alter_table("category") as batch_op:
batch_op.drop_constraint("uq_category_name_parent", type_="unique")
batch_op.add_column(
sa.Column(
"valid_from",
sa.DateTime(timezone=True),
nullable=True,
info={
"description": "UTC start of validity; NULL means valid since the indefinite past."
},
)
)
batch_op.add_column(
sa.Column(
"valid_to",
sa.DateTime(timezone=True),
nullable=True,
info={
"description": "UTC end of validity; NULL means the version is currently valid."
},
)
)
batch_op.add_column(
sa.Column(
"last_action",
sa.String(),
nullable=False,
server_default="created",
info={
"description": "Most recent lifecycle action recorded for this category version."
},
)
)
# Existing rows predate temporal tracking, so their start is unbounded.
op.execute("UPDATE category SET last_action = 'migrated'")
# Separate root and non-root indexes so NULL parent IDs are unique too.
op.create_index(
"uq_category_active_name_parent",
"category",
["name", "parent_id"],
unique=True,
sqlite_where=sa.text("valid_to IS NULL AND parent_id IS NOT NULL"),
postgresql_where=sa.text("valid_to IS NULL AND parent_id IS NOT NULL"),
)
op.create_index(
"uq_category_active_root_name",
"category",
["name"],
unique=True,
sqlite_where=sa.text("valid_to IS NULL AND parent_id IS NULL"),
postgresql_where=sa.text("valid_to IS NULL AND parent_id IS NULL"),
)
def downgrade() -> None:
bind = op.get_bind()
has_temporal_history = bind.execute(
sa.text(
"SELECT 1 FROM category "
"WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL LIMIT 1"
)
).first()
if has_temporal_history:
raise RuntimeError(
"Cannot downgrade category validity while temporal history exists; "
"the downgrade would discard it."
)
op.drop_index("uq_category_active_root_name", table_name="category")
op.drop_index("uq_category_active_name_parent", table_name="category")
with op.batch_alter_table("category") as batch_op:
batch_op.drop_column("last_action")
batch_op.drop_column("valid_to")
batch_op.drop_column("valid_from")
batch_op.create_unique_constraint(
"uq_category_name_parent", ["name", "parent_id"]
)
+5 -1
View File
@@ -33,7 +33,11 @@ build-backend = "uv_build"
[tool.mypy]
explicit_package_bases = true
mypy_path = "."
exclude = ["^tests/"]
# Ignore test file type errors (inline class definitions, float literals)
# See AGENTS.md for type checking notes
[[tool.mypy.overrides]]
module = ["tests.test_db_utils_update_path", "tests.test_domain_utils_budget_overlap"]
ignore_errors = true
[tool.ruff]
target-version = "py314"
+1 -2
View File
@@ -29,8 +29,7 @@ def find_category_db(session: Session, category: Category | None) -> CategoryDb
return (
session.query(CategoryDb)
.filter_by(name=node.name, parent_id=parent_id)
.filter(CategoryDb.valid_to.is_(None))
.one_or_none()
.first()
)
return find_existing(category)
+1 -89
View File
@@ -1,4 +1,3 @@
from datetime import UTC, datetime
from pathlib import Path
from sqlalchemy.orm import Session
@@ -7,7 +6,7 @@ from src.db_utils import upsert
from src.ingest.common import _reader_raw
from src.ingest.transactions import consolidate_transactions, map_b4w_to_transaction
from src.models.b4w import B4wDb, B4wExport
from src.models.category import Category, CategoryDb, upsert_category
from src.models.category import upsert_category
def reader_b4w(path: Path) -> list[B4wExport]:
@@ -15,93 +14,6 @@ def reader_b4w(path: Path) -> list[B4wExport]:
return list(map(B4wExport.model_validate, raw))
def _utc_datetime(value: datetime | None) -> datetime:
if value is None:
return datetime.now(UTC)
if value.tzinfo is None or value.utcoffset() is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
def _ensure_after_valid_from(category: CategoryDb, effective_at: datetime) -> None:
if category.valid_from is None:
return
valid_from = _utc_datetime(category.valid_from)
if effective_at <= valid_from:
raise ValueError("The action time must be later than valid_from")
def change_category(
session: Session,
category_id: int,
replacement: Category,
effective_at: datetime | None = None,
do_commit: bool = True,
) -> CategoryDb:
"""Close an active category version and start its replacement version."""
current = (
session.query(CategoryDb).filter_by(id=category_id, valid_to=None).one_or_none()
)
if current is None:
raise ValueError(f"No active category found with id {category_id}")
action_time = _utc_datetime(effective_at)
_ensure_after_valid_from(current, action_time)
parent = upsert_category(session, replacement.parent, do_commit=False)
parent_id = parent.id if parent else None
if current.name == replacement.name and current.parent_id == parent_id:
return current
replacement_active = (
session.query(CategoryDb)
.filter_by(name=replacement.name, parent_id=parent_id, valid_to=None)
.one_or_none()
)
current.valid_to = action_time
current.last_action = "changed"
session.flush()
if replacement_active is None:
replacement_active = CategoryDb(
name=replacement.name,
parent=parent,
valid_from=action_time,
valid_to=None,
last_action="changed",
)
session.add(replacement_active)
session.flush()
if do_commit:
session.commit()
return replacement_active
def retire_category(
session: Session,
category_id: int,
effective_at: datetime | None = None,
do_commit: bool = True,
) -> CategoryDb:
"""Close an active category version without creating a replacement."""
category = (
session.query(CategoryDb).filter_by(id=category_id, valid_to=None).one_or_none()
)
if category is None:
raise ValueError(f"No active category found with id {category_id}")
action_time = _utc_datetime(effective_at)
_ensure_after_valid_from(category, action_time)
category.valid_to = action_time
category.last_action = "retired"
session.flush()
if do_commit:
session.commit()
return category
def upsert_b4w(
session: Session, data: B4wExport, do_update=False, do_commit=True
) -> B4wDb | None:
+2 -9
View File
@@ -2,16 +2,11 @@ from datetime import date
from decimal import Decimal
from pydantic import Field, computed_field, field_validator
from sqlalchemy import (
Date,
ForeignKey,
Integer,
String,
)
from sqlalchemy import Date, ForeignKey, Integer, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
from src.constants import IBAN_PATTERN, Currency
from src.models.category import Category, CategoryDb
from src.models.category import Category
from src.models.common import (
ExportBaseModel,
IngestdbBase,
@@ -20,8 +15,6 @@ from src.models.common import (
hashify,
)
__all__ = ["B4wDb", "B4wExport", "Category", "CategoryDb"]
class B4wExport(ExportBaseModel):
id: int = Field(alias="Id1", description="internal B4w ID")
+6 -114
View File
@@ -1,50 +1,15 @@
from __future__ import annotations
from datetime import UTC, date, datetime, timedelta
from datetime import date
from decimal import Decimal
from typing import Self
from pydantic import BaseModel, Field, computed_field, model_validator
from sqlalchemy import (
Date,
DateTime,
ForeignKey,
Index,
Numeric,
String,
TypeDecorator,
text,
)
from sqlalchemy import Date, ForeignKey, Numeric, String, UniqueConstraint
from sqlalchemy.orm import Mapped, Session, mapped_column, relationship
from src.db_utils import upsert
from src.models.common import BaseModelDb, ExportBaseModel, hashify
class UTCDateTime(TypeDecorator[datetime]):
"""Store UTC timestamps and return timezone-aware UTC values on every dialect."""
impl = DateTime
cache_ok = True
def load_dialect_impl(self, dialect):
return dialect.type_descriptor(DateTime(timezone=True))
def process_bind_param(self, value: datetime | None, dialect):
if value is None:
return None
if value.tzinfo is None or value.utcoffset() is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
def process_result_value(self, value: datetime | None, dialect):
if value is None:
return None
if value.tzinfo is None or value.utcoffset() is None:
return value.replace(tzinfo=UTC)
return value.astimezone(UTC)
class Category(ExportBaseModel):
name: str = Field(pattern=r"[^:\r\n]+")
parent: Category | None = None
@@ -79,21 +44,7 @@ class Category(ExportBaseModel):
class CategoryDb(BaseModelDb):
__tablename__ = "category"
__table_args__ = (
Index(
"uq_category_active_name_parent",
"name",
"parent_id",
unique=True,
sqlite_where=text("valid_to IS NULL AND parent_id IS NOT NULL"),
postgresql_where=text("valid_to IS NULL AND parent_id IS NOT NULL"),
),
Index(
"uq_category_active_root_name",
"name",
unique=True,
sqlite_where=text("valid_to IS NULL AND parent_id IS NULL"),
postgresql_where=text("valid_to IS NULL AND parent_id IS NULL"),
),
UniqueConstraint("name", "parent_id", name="uq_category_name_parent"),
)
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
@@ -101,29 +52,6 @@ class CategoryDb(BaseModelDb):
parent_id: Mapped[int | None] = mapped_column(
ForeignKey("category.id"), nullable=True
)
valid_from: Mapped[datetime | None] = mapped_column(
UTCDateTime(),
nullable=True,
info={
"description": "UTC start of validity; NULL means valid since the indefinite past."
},
)
valid_to: Mapped[datetime | None] = mapped_column(
UTCDateTime(),
nullable=True,
info={
"description": "UTC end of validity; NULL means the version is currently valid."
},
)
last_action: Mapped[str] = mapped_column(
String,
nullable=False,
default="created",
server_default="created",
info={
"description": "Most recent lifecycle action recorded for this category version."
},
)
# Relationships
parent = relationship(
@@ -141,45 +69,11 @@ def upsert_category(
parent = upsert_category(session, category.parent, do_commit=False)
parameters = {"name": category.name, "parent": parent}
matching_rows = session.query(CategoryDb).filter_by(
name=category.name, parent_id=parent.id if parent else None
)
active = matching_rows.filter(CategoryDb.valid_to.is_(None)).one_or_none()
if active is not None:
return active
historical = (
matching_rows.filter(CategoryDb.valid_to.is_not(None))
.order_by(CategoryDb.valid_to.desc())
.first()
)
valid_from = None
last_action = "created"
if historical is not None:
valid_from = datetime.now(UTC)
if historical.valid_to is not None:
previous_valid_to = historical.valid_to
if (
previous_valid_to.tzinfo is None
or previous_valid_to.utcoffset() is None
):
previous_valid_to = previous_valid_to.replace(tzinfo=UTC)
else:
previous_valid_to = previous_valid_to.astimezone(UTC)
valid_from = max(valid_from, previous_valid_to + timedelta(microseconds=1))
last_action = "changed"
parameters_new = {
**parameters,
"valid_from": valid_from,
"valid_to": None,
"last_action": last_action,
}
return upsert(
session,
CategoryDb,
{**parameters, "valid_to": None},
parameters_new,
parameters,
parameters,
do_update=False,
do_commit=do_commit,
)
@@ -219,9 +113,7 @@ class CategoryBudgetDb(BaseModelDb):
__tablename__ = "category_budget"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
category_id: Mapped[int] = mapped_column(
ForeignKey("category.id"), nullable=False, index=True
)
category_id: Mapped[int] = mapped_column(ForeignKey("category.id"), nullable=False)
amount_eur: Mapped[Decimal] = mapped_column(Numeric(12, 2), nullable=False)
valid_from: Mapped[date | None] = mapped_column(Date, nullable=True, default=None)
valid_to: Mapped[date | None] = mapped_column(Date, nullable=True, default=None)
+2 -29
View File
@@ -1,13 +1,13 @@
"""Comprehensive tests for set_category_budget overlap handling."""
from datetime import UTC, date, datetime
from datetime import date
import pytest
from sqlalchemy.orm import Session
from src.domain_utils import find_category_db, set_category_budget
from src.models import Category, CategoryBudget, CategoryBudgetDb
from src.models.category import CategoryDb, upsert_category
from src.models.category import upsert_category
from tests.factories import CategoryFactory
# NOTE: MYPY errors suppressed for test fixtures - see AGENTS.md for type checking notes.
@@ -615,33 +615,6 @@ class TestFindCategoryDb:
result = find_category_db(db_session, Category(name="NonExistent"))
assert result is None
def test_find_reintroduced_category_returns_active_version(
self, db_session: Session
):
category = Category(name="Reintroduced")
historical = CategoryDb(
name=category.name,
valid_to=datetime(2026, 1, 1, tzinfo=UTC),
last_action="retired",
)
db_session.add(historical)
db_session.commit()
active = upsert_category(db_session, category, do_commit=False)
assert active is not None
db_session.commit()
result = find_category_db(db_session, category)
assert result is not None
assert result.id == active.id
set_category_budget(
db_session,
CategoryBudget(category=category, amount_eur=100.00),
)
budget = db_session.query(CategoryBudgetDb).one()
assert budget.category_id == active.id
class TestSetCategoryBudgetErrorHandling:
"""Test error handling in set_category_budget."""
-132
View File
@@ -1,132 +0,0 @@
from datetime import UTC, datetime
import pytest
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from src.ingest.b4w import change_category, retire_category, upsert_category
from src.models import Category
from src.models.b4w import CategoryDb
def test_upsert_category_returns_orm_object(db_session: Session):
result = upsert_category(
db_session,
Category.from_str("dummy_parent_1:dummy_parent_2:dummy"),
do_commit=False,
)
assert result
assert isinstance(result, CategoryDb)
assert str(result.name) == "dummy"
assert str(result.parent.name) == "dummy_parent_2"
assert str(result.parent.parent.name) == "dummy_parent_1"
assert result.valid_from is None
assert result.valid_to is None
assert result.last_action == "created"
assert all(
CategoryDb.__table__.c[name].info.get("description")
for name in ("valid_from", "valid_to", "last_action")
)
def test_upsert_category_writes_new_to_db(db_session: Session):
upsert_category(
db_session,
Category.from_str("grand_father:father:new_category"),
do_commit=True,
)
result = db_session.execute(
select(CategoryDb).filter_by(name="new_category")
).scalar_one()
assert result
assert str(result.name) == "new_category"
assert str(result.parent.name) == "father"
assert str(result.parent.parent.name) == "grand_father"
def test_upsert_category_skips_existing(db_session: Session):
for _ in range(3):
upsert_category(
db_session,
Category.from_str("grand_father:father:try_again_and_again"),
do_commit=True,
)
result = db_session.execute(
select(CategoryDb).filter_by(name="try_again_and_again")
).all()
assert len(result) == 1
def test_change_category_closes_old_version_and_starts_new_one(
db_session: Session,
):
original = upsert_category(
db_session, Category.from_str("old_category"), do_commit=False
)
assert original is not None
effective_at = datetime(2026, 10, 16, 12, tzinfo=UTC)
replacement = change_category(
db_session,
original.id,
Category.from_str("new_category"),
effective_at=effective_at,
do_commit=False,
)
assert original.valid_from is None
assert original.valid_to == effective_at
assert original.last_action == "changed"
assert replacement.name == "new_category"
assert replacement.valid_from == effective_at
assert replacement.valid_to is None
assert replacement.last_action == "changed"
db_session.expire(original)
db_session.expire(replacement)
assert original.valid_to == effective_at
assert replacement.valid_from == effective_at
# A historical name can be introduced again as a new active version.
reintroduced = upsert_category(
db_session, Category.from_str("old_category"), do_commit=False
)
assert reintroduced is not None
assert reintroduced.id != original.id
assert reintroduced.valid_from is not None
assert reintroduced.valid_from > original.valid_to
assert reintroduced.last_action == "changed"
def test_retire_category_closes_active_version(db_session: Session):
category = upsert_category(
db_session, Category.from_str("category_to_retire"), do_commit=False
)
assert category is not None
effective_at = datetime(2026, 10, 16, 12, tzinfo=UTC)
retired = retire_category(
db_session, category.id, effective_at=effective_at, do_commit=False
)
assert retired.valid_to == effective_at
assert retired.last_action == "retired"
assert (
db_session.query(CategoryDb)
.filter_by(name="category_to_retire", valid_to=None)
.one_or_none()
is None
)
def test_active_root_categories_are_unique(db_session: Session):
db_session.add_all(
[
CategoryDb(name="duplicate_root"),
CategoryDb(name="duplicate_root"),
]
)
with pytest.raises(IntegrityError):
db_session.flush()