diff --git a/.env.template b/.env.template index 39cc3c7..4e2a77a 100644 --- a/.env.template +++ b/.env.template @@ -3,3 +3,8 @@ LLM_API_KEY = "not-relevant" LLM_DEFAULT_TEMPERATURE = 1.0 METADATA_PREFIX = "photools|" +LOG_LEVEL=INFO +LLM_REQUEST_TIMEOUT_SECONDS=10 +LLM_MAX_COMPLETION_TOKENS=1024 +LLM_MAX_RETRIES=5 +LLM_MAX_KEYWORDS=15 diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index eb71684..4242e21 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -35,5 +35,3 @@ jobs: run: uv run mypy . - name: Check formatting (black) run: uv run black --check . --extend-exclude .venv - - name: Check formatting (black) - run: uv run black --check . --extend-exclude .venv diff --git a/AGENTS.md b/AGENTS.md index eb30161..0c954a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,27 +1,42 @@ # Agent Instructions: PhotoTools3 ## Environment Setup + Requires a `.env` file with the following variables: + - `llm_api_key`: API key for the LLM provider. - `llm_api_base_url`: Base URL for the LLM API. - `llm_default_temperature`: Float value for temperature. ## System Dependencies + - `exiftool` binary: required by `pyexiftool` at runtime, not installable via pip/uv. - Arch: `perl-image-exiftool` - Debian/Ubuntu: `libimage-exiftool-perl` ## Development Toolchain + - **Dependency Management**: Use `uv`. - **Python Version**: Requires Python `>=3.14`. - **Testing**: Run `pytest` to execute the test suite in `test/`. - **Linting & Formatting**: Use `ruff` and `black`. ## CLI Usage + The primary entry point is the `photools3` command: + - `photools3 keywords `: Extracts keywords and captions from a single image. ## Architecture & Logic + - **`src/photools3/photo.py`**: Core image processing logic, including resizing, RGB conversion, and generating base64 JPEG encodings specifically for LLM vision compatibility. - **`src/photools3/llm_session.py`**: Manages the workflow of sending processed images to the LLM and parsing results. - **Configuration**: The list of keywords to extract is maintained in `config/keywords.txt`. + +## Code style + +- Avoid overly verbose docstrings or comments. Be precise! + +## Comunication + +- When done with code modifications, give a one-liner for the currently unstaged changes that can be used as a commit message. diff --git a/README.md b/README.md index 971625d..c1905f5 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,28 @@ Create a `.env` file in the project root with the LLM connection details and the prefix used to identify metadata written by PhotoTools3: ```dotenv -llm_api_base_url=https://your-openai-compatible-api/v1 +llm_api_base_url=http://localhost:1234/v1 llm_api_key=your-api-key llm_default_temperature=0.2 metadata_prefix=photools3: +LOG_LEVEL=DEBUG +LLM_REQUEST_TIMEOUT_SECONDS=10 +LLM_MAX_COMPLETION_TOKENS=1024 +LLM_MAX_RETRIES=5 +LLM_MAX_KEYWORDS=15 ``` Keyword suggestions are restricted to the vocabulary in `config/keywords.txt`. +`LOG_LEVEL=DEBUG` enables detailed per-request and per-photo timings. +`LLM_MAX_RETRIES` controls retries for malformed or structurally invalid +responses; `5` means up to 6 requests for a photo. A valid response with +unsupported keywords gets at most one targeted correction retry, then remaining +invalid keywords are discarded. +Each request is limited by `LLM_REQUEST_TIMEOUT_SECONDS`; a timeout reloads the +LM Studio model before retrying. `LLM_MAX_COMPLETION_TOKENS` caps the generated +response length. When all retries are exhausted, the model is reloaded before +the failed photo is reported. `LLM_MAX_KEYWORDS` sets the maximum number of +distinct keywords requested and stored per photo; excess keywords are discarded. ## Generate metadata diff --git a/pyproject.toml b/pyproject.toml index 80ade47..34537d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "pytest>=9.1.1", "ruff>=0.16.9", "typer>=0.12.0", - "utils>=0.1.2", + "utils>=0.1.3", ] [project.scripts] diff --git a/src/photools3/cli.py b/src/photools3/cli.py index 971b7c0..1d7a94f 100644 --- a/src/photools3/cli.py +++ b/src/photools3/cli.py @@ -13,9 +13,20 @@ from photools3.processing import ( process_batch, process_photo, ) +from photools3.settings import settings app = typer.Typer(help="PhotoTools3: A CLI for image processing.") -LoggingSettings() +LoggingSettings(LOG_LEVEL=settings.log_level) +for name in ( + "uvicorn", + "uvicorn.error", + "uvicorn.access", + "PIL", + "openai", + "httpx2", + "httpcore2", +): + logging.getLogger(name).setLevel(logging.WARNING) logger = logging.getLogger(__name__) diff --git a/src/photools3/llm_session.py b/src/photools3/llm_session.py index 14922af..a2ba265 100644 --- a/src/photools3/llm_session.py +++ b/src/photools3/llm_session.py @@ -1,11 +1,13 @@ import json import logging import re +import time import openai from openai import OpenAI from pydantic import BaseModel, ValidationError +from photools3.lm_studio import LMStudioModelManager from photools3.models import ( LlmMessageContent, LlmMessageRole, @@ -17,6 +19,7 @@ from photools3.models import ( LlmResponseGeneral, LlmResponseKeywords, LlmResponseQuality, + load_valid_keywords, ) from photools3.photo import Photo from photools3.settings import settings @@ -30,19 +33,23 @@ def _clean_llm_response(txt: str): return txt -def _correction_prompt( - original_request: LlmRequest, exception: Exception, origial_json -): - correction_prompt = ( - f"The previous response was invalid due to: {exception}. " - f"Please provide the corrected JSON based on this content: " - f"{origial_json}" +def _correction_prompt(instruction: str) -> LlmRequestMessage: + return LlmRequestMessage( + role=LlmMessageRole.user, + content=instruction, ) - original_request.messages.append( - LlmRequestMessage( - role=LlmMessageRole.user, - content=correction_prompt, - ) + + +def _request_with_correction(base_request: LlmRequest, instruction: str) -> LlmRequest: + return base_request.model_copy( + update={"messages": [*base_request.messages, _correction_prompt(instruction)]} + ) + + +def _validation_summary(exception: ValidationError) -> str: + return "; ".join( + f"{'.'.join(map(str, error['loc']))}: {error['msg']}" + for error in exception.errors(include_input=False) ) @@ -56,6 +63,7 @@ class LlmSession: self, system_prompt: str, temperature: float = settings.llm_default_temperature ): self._models: list = [] + self._model_id: str | None = None self.system_prompt = system_prompt self.temperature = temperature @@ -63,7 +71,9 @@ class LlmSession: self.client = OpenAI( base_url=self.base_url, api_key=self.api_key, + max_retries=0, ) + self._model_manager = LMStudioModelManager(self.base_url, self.api_key) def close(self): self.client.close() @@ -74,8 +84,17 @@ class LlmSession: def __exit__(self, exc_type, exc_value, traceback): self.close() + def flush_model(self): + """Unload and reload this session's model through LM Studio.""" + if self._model_id is None: + self._model_id = self.models[0].id + self._model_manager.reload_model(self._model_id) + def client_chat_completion_create(self, request: LlmRequest, choice_no: int = 0): - response = self.client.chat.completions.create(**request.model_dump()) + response = self.client.chat.completions.create( + **request.model_dump(), + timeout=settings.llm_request_timeout_seconds, + ) return response.choices[choice_no].message.content def is_api_reachable(self) -> bool: @@ -95,37 +114,139 @@ class LlmSession: self, photos: list[Photo], model_id=None, - max_retries: int = 2, + max_retries: int | None = None, prompt: str | None = None, ): - if not model_id: - model_id = self.models[0].id + retry_limit = settings.llm_max_retries if max_retries is None else max_retries + if retry_limit < 0: + raise ValueError("max_retries must be greater than or equal to zero") - request = self._build_request(prompt or self.prompt, photos, model_id) + if model_id is None: + if self._model_id is None: + self._model_id = self.models[0].id + model_id = self._model_id + else: + self._model_id = model_id - last_error = None - for attempt in range(max_retries + 1): + base_request = self._build_request(prompt or self.prompt, photos, model_id) + request = base_request + tag_correction_used = False + valid_keywords = load_valid_keywords() + max_keywords = settings.llm_max_keywords + + for attempt in range(retry_limit + 1): + request_started = time.perf_counter() try: result_raw = self.client_chat_completion_create(request) - json_str = _clean_llm_response(result_raw) - return self.response_model.model_validate_json(json_str) + except openai.APITimeoutError as exc: + logger.warning( + "LLM request timed out after %.2f seconds; reloading model", + time.perf_counter() - request_started, + ) + self.flush_model() + if attempt < retry_limit: + continue + raise ValueError( + f"LLM request timed out after {retry_limit + 1} attempts" + ) from exc + except Exception: + logger.debug( + "LLM request attempt %d/%d failed after %.2f seconds", + attempt + 1, + retry_limit + 1, + time.perf_counter() - request_started, + ) + raise - except (json.JSONDecodeError, ValidationError) as exc: - last_error = exc - if attempt < max_retries: - logger.info("Chatbot response is invalid; promting to correct ...") - _correction_prompt( - request, - exc, - json_str if "json_str" in locals() else "origial json", + logger.debug( + "LLM request attempt %d/%d completed in %.2f seconds", + attempt + 1, + retry_limit + 1, + time.perf_counter() - request_started, + ) + try: + response_data = json.loads(_clean_llm_response(result_raw)) + except (json.JSONDecodeError, TypeError) as exc: + if attempt < retry_limit: + logger.info("Chatbot response was not valid JSON; retrying") + request = _request_with_correction( + base_request, + "Return one complete JSON object matching the schema. " + f"The previous response was invalid: {exc}", ) continue - else: - raise ValueError( - f"Failed to parse or validate LLM response after {max_retries} retries: {exc}" + + logger.warning( + "LLM retry limit reached; reloading model before failing" + ) + self.flush_model() + raise ValueError( + f"Failed to parse LLM response after {retry_limit} retries: {exc}" + ) from exc + + if ( + isinstance(response_data, dict) + and "tags" in self.response_model.model_fields + and isinstance(response_data.get("tags"), list) + ): + tags = response_data["tags"] + considered_tags = tags[:max_keywords] + invalid_tags = [ + tag + for tag in considered_tags + if not isinstance(tag, str) or tag not in valid_keywords + ] + + if invalid_tags and not tag_correction_used and attempt < retry_limit: + tag_correction_used = True + logger.info( + "LLM returned unsupported keywords; requesting one correction" ) - except Exception: - raise + request = _request_with_correction( + base_request, + f"These keywords are not allowed: {invalid_tags!r}. " + "Replace them with exact allowed keywords when sensible; " + f"otherwise omit them. Return at most {max_keywords} " + "distinct keywords and do not list the vocabulary.", + ) + continue + + if len(tags) > max_keywords: + logger.info( + "Discarding keywords beyond configured limit of %d", + max_keywords, + ) + if invalid_tags: + logger.warning("Discarding unsupported keywords: %s", invalid_tags) + + response_data = dict(response_data) + valid_tags = [ + tag + for tag in tags + if isinstance(tag, str) and tag in valid_keywords + ] + response_data["tags"] = list(dict.fromkeys(valid_tags))[:max_keywords] + + try: + return self.response_model.model_validate(response_data) + except ValidationError as exc: + if attempt < retry_limit: + logger.info("LLM response did not match the schema; retrying") + request = _request_with_correction( + base_request, + "Return one complete JSON object matching the schema. " + f"Validation issues: {_validation_summary(exc)}", + ) + continue + + logger.warning( + "LLM retry limit reached; reloading model before failing" + ) + self.flush_model() + raise ValueError( + f"Failed to validate LLM response after {retry_limit} retries: " + f"{_validation_summary(exc)}" + ) from exc def _build_request( self, @@ -137,6 +258,7 @@ class LlmSession: return LlmRequest( model=model_id, temperature=self.temperature, + max_tokens=settings.llm_max_completion_tokens, messages=[ # System prompt LlmRequestMessage( @@ -168,13 +290,15 @@ class LlmSession: class LlmSessionCaptionsAndKeywords(LlmSession): response_model = LlmResponseCaptionsAndKeywords - prompt = "Give me a caption and keyword for the following photo(s)" + prompt = "Give me a caption and keywords for the following photo" def __init__(self, temperature: float = settings.llm_default_temperature): super().__init__( system_prompt=( "You are a helpful photographer's assistant that helps to write **captions** and **keywords** to photos:\n" "Answer in German.\n" + f"Choose at most {settings.llm_max_keywords} distinct, relevant keywords. " + "Do not repeat or enumerate the allowed-keyword list.\n" "Output must be a JSON of the following schema:\n" f"{self.response_model.model_json_schema()}" ), @@ -207,6 +331,8 @@ class LlmSessionKeywords(LlmSession): system_prompt=( "You are a helpful photographer's assistant that chooses keywords " "for photos. Answer in German.\n" + f"Choose at most {settings.llm_max_keywords} distinct, relevant keywords. " + "Do not repeat or enumerate the allowed-keyword list.\n" "Output must be a JSON of the following schema:\n" f"{self.response_model.model_json_schema()}" ), diff --git a/src/photools3/lm_studio.py b/src/photools3/lm_studio.py new file mode 100644 index 0000000..cb5c1e0 --- /dev/null +++ b/src/photools3/lm_studio.py @@ -0,0 +1,135 @@ +import json +import logging +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +logger = logging.getLogger(__name__) + +_LOAD_CONFIG_KEYS = ( + "context_length", + "eval_batch_size", + "flash_attention", + "num_experts", + "offload_kv_cache_to_gpu", +) + + +class LMStudioError(RuntimeError): + """Raised when LM Studio model management fails.""" + + +class LMStudioModelManager: + """Unload and reload a model through LM Studio's native REST API.""" + + def __init__(self, openai_base_url: str, api_key: str, timeout: float = 600): + parsed_url = urlsplit(openai_base_url) + path = parsed_url.path.rstrip("/") + path = path.removesuffix("/v1") + root_url = urlunsplit( + (parsed_url.scheme, parsed_url.netloc, path, "", "") + ).rstrip("/") + self.api_base_url = f"{root_url}/api/v1" + self.api_key = api_key + self.timeout = timeout + + def reload_model(self, model_id: str) -> None: + """Unload the matching loaded instance and reload it with its config.""" + started = time.perf_counter() + models_response = self._request("GET", "/models") + model, loaded_instance = self._find_model(models_response, model_id) + model_key = model["key"] + + if loaded_instance is not None: + self._request( + "POST", + "/models/unload", + {"instance_id": loaded_instance["id"]}, + ) + load_config = loaded_instance.get("config", {}) + else: + load_config = {} + + load_body = {"model": model_key} + load_body.update( + { + key: load_config[key] + for key in _LOAD_CONFIG_KEYS + if key in load_config and load_config[key] is not None + } + ) + self._request("POST", "/models/load", load_body) + logger.info( + "Reloaded LM Studio model %s in %.1f seconds", + model_key, + time.perf_counter() - started, + ) + + @staticmethod + def _find_model(models_response: dict, model_id: str) -> tuple[dict, dict | None]: + models = models_response.get("models") + if not isinstance(models, list): + raise LMStudioError("LM Studio model list response has no models array") + + for model in models: + loaded_instances = model.get("loaded_instances", []) + for instance in loaded_instances: + if instance.get("id") == model_id: + return model, instance + + matching_models = [model for model in models if model.get("key") == model_id] + if len(matching_models) != 1: + raise LMStudioError( + f"Could not uniquely identify LM Studio model {model_id!r}" + ) + + model = matching_models[0] + loaded_instances = model.get("loaded_instances", []) + if len(loaded_instances) > 1: + raise LMStudioError( + f"Multiple loaded instances found for LM Studio model {model_id!r}" + ) + loaded_instance = loaded_instances[0] if loaded_instances else None + return model, loaded_instance + + def _request(self, method: str, path: str, body: dict | None = None) -> dict: + data = None if body is None else json.dumps(body).encode("utf-8") + headers = {"Accept": "application/json"} + if data is not None: + headers["Content-Type"] = "application/json" + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + + request = Request( + f"{self.api_base_url}{path}", + data=data, + headers=headers, + method=method, + ) + try: + with urlopen(request, timeout=self.timeout) as response: + response_body = response.read() + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise LMStudioError( + f"LM Studio {method} {path} returned HTTP {exc.code}: {detail}" + ) from exc + except (URLError, TimeoutError) as exc: + raise LMStudioError( + f"Could not reach LM Studio at {self.api_base_url}: {exc}" + ) from exc + + if not response_body: + return {} + try: + result = json.loads(response_body) + except json.JSONDecodeError as exc: + raise LMStudioError( + f"LM Studio {method} {path} returned invalid JSON" + ) from exc + if not isinstance(result, dict): + raise LMStudioError( + f"LM Studio {method} {path} returned an unexpected response" + ) + return result diff --git a/src/photools3/models.py b/src/photools3/models.py index d545c00..97197c7 100644 --- a/src/photools3/models.py +++ b/src/photools3/models.py @@ -1,3 +1,4 @@ +import json from enum import StrEnum from functools import lru_cache from typing import Annotated, Any, Literal @@ -10,7 +11,7 @@ from pydantic import ( model_validator, ) -from .settings import KEYWORDS_LIST_PATH +from .settings import KEYWORDS_LIST_PATH, settings @lru_cache @@ -42,6 +43,7 @@ class LlmRequest(BaseModel): model: str = "gpt-4o" messages: list[LlmRequestMessage] temperature: float | None = None + max_tokens: int class LlmMessageRole(StrEnum): @@ -98,17 +100,22 @@ class LlmResponseCaption(BaseModel): ] -class LlmResponseKeywords(BaseModel): - tags: list[str] = Field( - description=( - "A list of keywords that properly tag the photo. " - "Allowed keywords: " - + str(load_valid_keywords()) - .replace("'", '"') - .replace("{", "[") - .replace("}", "]") - ) +def _keyword_tags_field(): + allowed_keywords = json.dumps( + sorted(load_valid_keywords()), ensure_ascii=False, separators=(",", ":") ) + return Field( + max_length=settings.llm_max_keywords, + description=( + f"Choose at most {settings.llm_max_keywords} distinct, relevant keywords. " + "Do not enumerate the vocabulary. Allowed keywords: " + f"{allowed_keywords}" + ), + ) + + +class _LlmResponseWithTags(BaseModel): + tags: list[str] = _keyword_tags_field() @field_validator("tags") @classmethod @@ -122,7 +129,11 @@ class LlmResponseKeywords(BaseModel): return v -class LlmResponseCaptionsAndKeywords(BaseModel): +class LlmResponseKeywords(_LlmResponseWithTags): + pass + + +class LlmResponseCaptionsAndKeywords(_LlmResponseWithTags): caption: Annotated[ str, Field( @@ -132,27 +143,6 @@ class LlmResponseCaptionsAndKeywords(BaseModel): ) ), ] - tags: list[str] = Field( - description=( - "A list of keywords that properly tag the photo. " - "Allowed keywords: " - + str(load_valid_keywords()) - .replace("'", '"') - .replace("{", "[") - .replace("}", "]") - ) - ) - - @field_validator("tags") - @classmethod - def validate_tags(cls, v: list[str]) -> list[str]: - valid_keywords = load_valid_keywords() - for tag in v: - if tag not in valid_keywords: - raise ValueError( - f"'{tag}' is not a valid keyword. Must be one from the configuration." - ) - return v class LlmResponseQuality(BaseModel): diff --git a/src/photools3/processing.py b/src/photools3/processing.py index 5e71ee1..fe7d14c 100644 --- a/src/photools3/processing.py +++ b/src/photools3/processing.py @@ -1,4 +1,5 @@ import logging +import time from collections.abc import Iterable from dataclasses import dataclass from enum import StrEnum @@ -65,6 +66,8 @@ def process_photo( owns_session = session is None active_session = session if session is not None else _create_session(mode) + logger.info(f"Processing photo: {path!s}") + try: with Photo(path) as photo: result = active_session.call([photo]) @@ -108,6 +111,7 @@ def process_batch( try: for item in paths: path = Path(item) + photo_started = time.perf_counter() try: result = process_photo(path, mode, session=session) except Exception as exc: @@ -122,6 +126,12 @@ def process_batch( ) else: outcomes.append(ProcessingOutcome(path=path, result=result)) + finally: + logger.debug( + "Photo processing for %s took %.2f seconds", + path, + time.perf_counter() - photo_started, + ) finally: session.close() diff --git a/src/photools3/settings.py b/src/photools3/settings.py index 2ceae41..8f7092a 100644 --- a/src/photools3/settings.py +++ b/src/photools3/settings.py @@ -1,5 +1,6 @@ from pathlib import Path +from pydantic import Field from pydantic_core import Url from pydantic_settings import BaseSettings, SettingsConfigDict @@ -13,11 +14,16 @@ KEYWORDS_LIST_PATH = CONFIG_DIR / "keywords.txt" class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env") + model_config = SettingsConfigDict(env_file=".env", extra="ignore") llm_api_base_url: Url llm_api_key: str llm_default_temperature: float + llm_request_timeout_seconds: float = Field(default=10, gt=0) + llm_max_completion_tokens: int = Field(default=1024, gt=0) + llm_max_retries: int = Field(default=5, ge=0) + llm_max_keywords: int = Field(default=15, ge=1) + log_level: str = "INFO" metadata_prefix: str diff --git a/test/test_llm_session_mocked.py b/test/test_llm_session_mocked.py index c6b9bb0..31160ee 100644 --- a/test/test_llm_session_mocked.py +++ b/test/test_llm_session_mocked.py @@ -1,6 +1,10 @@ from unittest.mock import MagicMock, patch -from photools3.llm_session import LlmSession +import openai +import pytest + +from photools3.llm_session import LlmSession, LlmSessionCaptionsAndKeywords +from photools3.models import LlmResponseCaptionsAndKeywords def test_llm_session_mocked(test_photo): @@ -84,3 +88,216 @@ def test_llm_session_captions_and_keywords_mocked(test_photo): # Verify that the mock was called assert mock_client.chat.completions.create.called + + +def test_llm_session_retries_from_original_request_without_accumulating_messages(): + with patch("photools3.llm_session.OpenAI") as mocked_openai: + mock_client = MagicMock() + mocked_openai.return_value = mock_client + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + + def response(content): + message = MagicMock() + message.content = content + choice = MagicMock() + choice.message = message + result = MagicMock() + result.choices = [choice] + return result + + mock_client.chat.completions.create.side_effect = [ + response('{"wrong": "first"}'), + response('{"wrong": "second"}'), + response('{"text": "correct"}'), + ] + + session = LlmSession(system_prompt="You are a helper.") + result = session.call(photos=[], prompt="Return JSON") + + assert result.text == "correct" + requests = mock_client.chat.completions.create.call_args_list + messages_per_attempt = [call.kwargs["messages"] for call in requests] + assert [len(messages) for messages in messages_per_attempt] == [2, 3, 3] + final_correction = messages_per_attempt[2][-1]["content"] + assert "Field required" in final_correction + assert '"wrong"' not in final_correction + + +def test_llm_session_uses_configured_retry_count(monkeypatch): + monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 1) + + with patch("photools3.llm_session.OpenAI") as mocked_openai: + mock_client = MagicMock() + mocked_openai.return_value = mock_client + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + + def response(content): + message = MagicMock() + message.content = content + choice = MagicMock() + choice.message = message + result = MagicMock() + result.choices = [choice] + return result + + mock_client.chat.completions.create.side_effect = [ + response('{"wrong": "first"}'), + response('{"wrong": "second"}'), + response('{"text": "would be a third call"}'), + ] + session = LlmSession(system_prompt="You are a helper.") + + with ( + patch.object(session, "flush_model") as flush_model, + pytest.raises(ValueError, match="after 1 retries"), + ): + session.call(photos=[]) + + assert mock_client.chat.completions.create.call_count == 2 + flush_model.assert_called_once() + + +def test_llm_session_reloads_after_timeout_and_retries(monkeypatch): + monkeypatch.setattr( + "photools3.llm_session.settings.llm_request_timeout_seconds", 10 + ) + monkeypatch.setattr("photools3.llm_session.settings.llm_max_completion_tokens", 768) + monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 1) + + with patch("photools3.llm_session.OpenAI") as mocked_openai: + mock_client = MagicMock() + mocked_openai.return_value = mock_client + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + + response_message = MagicMock(content='{"text": "recovered"}') + response_choice = MagicMock(message=response_message) + success_response = MagicMock(choices=[response_choice]) + mock_client.chat.completions.create.side_effect = [ + openai.APITimeoutError(request=MagicMock()), + success_response, + ] + + session = LlmSession(system_prompt="You are a helper.") + with patch.object(session, "flush_model") as flush_model: + result = session.call(photos=[]) + + assert result.text == "recovered" + flush_model.assert_called_once() + calls = mock_client.chat.completions.create.call_args_list + assert len(calls) == 2 + assert all(call.kwargs["timeout"] == 10 for call in calls) + assert all(call.kwargs["max_tokens"] == 768 for call in calls) + mocked_openai.assert_called_once_with( + base_url=session.base_url, + api_key=session.api_key, + max_retries=0, + ) + + +@pytest.mark.parametrize("failure_kind", ["invalid_response", "timeout"]) +def test_llm_session_reloads_when_retry_limit_is_exhausted(monkeypatch, failure_kind): + monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 0) + + with patch("photools3.llm_session.OpenAI") as mocked_openai: + mock_client = MagicMock() + mocked_openai.return_value = mock_client + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + + if failure_kind == "timeout": + mock_client.chat.completions.create.side_effect = openai.APITimeoutError( + request=MagicMock() + ) + else: + response_message = MagicMock(content='{"wrong": "shape"}') + response_choice = MagicMock(message=response_message) + mock_client.chat.completions.create.return_value = MagicMock( + choices=[response_choice] + ) + + session = LlmSession(system_prompt="You are a helper.") + with ( + patch.object(session, "flush_model") as flush_model, + pytest.raises(ValueError), + ): + session.call(photos=[]) + + flush_model.assert_called_once() + assert mock_client.chat.completions.create.call_count == 1 + + +def test_invalid_keywords_get_one_correction_then_invalid_tags_are_dropped(monkeypatch): + monkeypatch.setattr("photools3.llm_session.settings.llm_max_keywords", 3) + monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 5) + + with patch("photools3.llm_session.OpenAI") as mocked_openai: + mock_client = MagicMock() + mocked_openai.return_value = mock_client + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + + def response(content): + message = MagicMock(content=content) + choice = MagicMock(message=message) + return MagicMock(choices=[choice]) + + mock_client.chat.completions.create.side_effect = [ + response( + '{"caption": "A caption", "tags": ["not_allowed", "architektur"]}' + ), + response( + '{"caption": "A caption", ' + '"tags": ["still_invalid", "architektur", "personen"]}' + ), + ] + session = LlmSessionCaptionsAndKeywords() + with patch.object(session, "flush_model") as flush_model: + result = session.call(photos=[]) + + assert result.tags == ["architektur", "personen"] + assert mock_client.chat.completions.create.call_count == 2 + flush_model.assert_not_called() + correction = mock_client.chat.completions.create.call_args_list[1].kwargs[ + "messages" + ][-1]["content"] + assert "not_allowed" in correction + assert '"caption"' not in correction + + +def test_excess_keywords_are_truncated_to_configured_limit(monkeypatch): + monkeypatch.setattr("photools3.llm_session.settings.llm_max_keywords", 2) + + with patch("photools3.llm_session.OpenAI") as mocked_openai: + mock_client = MagicMock() + mocked_openai.return_value = mock_client + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + message = MagicMock( + content=( + '{"caption": "A caption", ' + '"tags": ["architektur", "personen", "landschaft"]}' + ) + ) + mock_client.chat.completions.create.return_value = MagicMock( + choices=[MagicMock(message=message)] + ) + + result = LlmSessionCaptionsAndKeywords().call(photos=[]) + + assert result.tags == ["architektur", "personen"] + assert mock_client.chat.completions.create.call_count == 1 + + +def test_keyword_response_schema_includes_configured_max_items(): + schema = LlmResponseCaptionsAndKeywords.model_json_schema() + + assert schema["properties"]["tags"]["maxItems"] == 15 diff --git a/test/test_lm_studio.py b/test/test_lm_studio.py new file mode 100644 index 0000000..254244b --- /dev/null +++ b/test/test_lm_studio.py @@ -0,0 +1,102 @@ +import json +from unittest.mock import MagicMock, patch + +import pytest + +from photools3.lm_studio import LMStudioError, LMStudioModelManager + + +def _http_response(payload: dict): + response = MagicMock() + response.__enter__.return_value = response + response.read.return_value = json.dumps(payload).encode("utf-8") + return response + + +def test_reload_model_unloads_and_reloads_preserving_context_config(): + model_response = { + "models": [ + { + "key": "qwen/qwen3-vl-4b", + "loaded_instances": [ + { + "id": "qwen/qwen3-vl-4b-instance", + "config": { + "context_length": 16384, + "eval_batch_size": 256, + "flash_attention": True, + "offload_kv_cache_to_gpu": False, + "parallel": 2, + }, + } + ], + } + ] + } + with patch( + "photools3.lm_studio.urlopen", + side_effect=[ + _http_response(model_response), + _http_response({"instance_id": "qwen/qwen3-vl-4b-instance"}), + _http_response({"instance_id": "qwen/qwen3-vl-4b-instance"}), + ], + ) as mocked_urlopen: + manager = LMStudioModelManager( + "http://localhost:1234/v1", "test-token", timeout=12 + ) + manager.reload_model("qwen/qwen3-vl-4b") + + requests = [call.args[0] for call in mocked_urlopen.call_args_list] + assert requests[0].full_url == "http://localhost:1234/api/v1/models" + assert requests[0].get_method() == "GET" + assert requests[0].get_header("Authorization") == "Bearer test-token" + + assert requests[1].full_url.endswith("/models/unload") + assert json.loads(requests[1].data) == {"instance_id": "qwen/qwen3-vl-4b-instance"} + + assert requests[2].full_url.endswith("/models/load") + assert json.loads(requests[2].data) == { + "model": "qwen/qwen3-vl-4b", + "context_length": 16384, + "eval_batch_size": 256, + "flash_attention": True, + "offload_kv_cache_to_gpu": False, + } + + +def test_reload_model_loads_by_key_if_no_instance_is_loaded(): + model_response = {"models": [{"key": "qwen/model", "loaded_instances": []}]} + with patch( + "photools3.lm_studio.urlopen", + side_effect=[ + _http_response(model_response), + _http_response({"status": "loaded"}), + ], + ) as mocked_urlopen: + manager = LMStudioModelManager("http://localhost:1234/v1", "token") + manager.reload_model("qwen/model") + + assert mocked_urlopen.call_count == 2 + load_request = mocked_urlopen.call_args.args[0] + assert load_request.full_url.endswith("/models/load") + assert json.loads(load_request.data) == {"model": "qwen/model"} + + +def test_reload_model_rejects_ambiguous_model_selection(): + model_response = { + "models": [ + { + "key": "qwen/model", + "loaded_instances": [ + {"id": "instance-1", "config": {}}, + {"id": "instance-2", "config": {}}, + ], + } + ] + } + with patch( + "photools3.lm_studio.urlopen", return_value=_http_response(model_response) + ): + manager = LMStudioModelManager("http://localhost:1234/v1", "token") + with pytest.raises(LMStudioError, match="Multiple loaded instances"): + manager.reload_model("qwen/model") diff --git a/uv.lock b/uv.lock index b88df17..56a3bc3 100644 --- a/uv.lock +++ b/uv.lock @@ -780,7 +780,7 @@ requires-dist = [ { name = "pytest", specifier = ">=9.1.1" }, { name = "ruff", specifier = ">=0.16.9" }, { name = "typer", specifier = ">=0.12.0" }, - { name = "utils", specifier = ">=0.1.2", index = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/simple" }, + { name = "utils", specifier = ">=0.1.3", index = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/simple" }, ] [[package]] @@ -1205,7 +1205,7 @@ wheels = [ [[package]] name = "utils" -version = "0.1.2" +version = "0.1.3" source = { registry = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/simple" } dependencies = [ { name = "black" }, @@ -1217,9 +1217,9 @@ dependencies = [ { name = "twine" }, { name = "uvicorn" }, ] -sdist = { url = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/files/utils/0.1.2/utils-0.1.2.tar.gz", hash = "sha256:c711755c3e2fabd9cd380e25e45bc19da9b901868a6eb67028968803c725b118" } +sdist = { url = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/files/utils/0.1.3/utils-0.1.3.tar.gz", hash = "sha256:8970a179573a63ab4c546cfd1da8ceca0d915bc4357b49ccffaba9b3d4a0195e" } wheels = [ - { url = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/files/utils/0.1.2/utils-0.1.2-py3-none-any.whl", hash = "sha256:f3335ba10bd8eb9f5c000287a6ee41987b188372f3dca2783d99e6e70513a5d7" }, + { url = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/files/utils/0.1.3/utils-0.1.3-py3-none-any.whl", hash = "sha256:c24078ba05f9f5b41fc3909598084b93c6df59f4ca60001e1e0b4fa55797e642" }, ] [[package]]