diff --git a/.env.template b/.env.template index 39cc3c7..43397fe 100644 --- a/.env.template +++ b/.env.template @@ -3,3 +3,6 @@ LLM_API_KEY = "not-relevant" LLM_DEFAULT_TEMPERATURE = 1.0 METADATA_PREFIX = "photools|" +LOG_LEVEL=INFO +LLM_FLUSH_EVERY_PHOTOS=50 +LLM_MAX_RETRIES=5 diff --git a/README.md b/README.md index 971625d..f8bd8b6 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,23 @@ Create a `.env` file in the project root with the LLM connection details and the prefix used to identify metadata written by PhotoTools3: ```dotenv -llm_api_base_url=https://your-openai-compatible-api/v1 +llm_api_base_url=http://localhost:1234/v1 llm_api_key=your-api-key llm_default_temperature=0.2 metadata_prefix=photools3: +LOG_LEVEL=DEBUG +LLM_FLUSH_EVERY_PHOTOS=50 +LLM_MAX_RETRIES=5 ``` Keyword suggestions are restricted to the vocabulary in `config/keywords.txt`. +`LOG_LEVEL=DEBUG` enables detailed per-request and per-photo timings. The flush +interval uses LM Studio's native model-management API to unload and reload the +model after every 50 attempted batch photos. This requires LM Studio's native +`/api/v1` model-management API. Set `LLM_FLUSH_EVERY_PHOTOS=0` to disable it; +use `0` for non-LM-Studio OpenAI-compatible providers. +`LLM_MAX_RETRIES` controls how many corrective retry requests are allowed after +the initial LLM response; `5` means up to 6 requests for a photo. ## Generate metadata diff --git a/pyproject.toml b/pyproject.toml index 80ade47..34537d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "pytest>=9.1.1", "ruff>=0.16.9", "typer>=0.12.0", - "utils>=0.1.2", + "utils>=0.1.3", ] [project.scripts] diff --git a/src/photools3/cli.py b/src/photools3/cli.py index 971b7c0..5adef79 100644 --- a/src/photools3/cli.py +++ b/src/photools3/cli.py @@ -13,9 +13,10 @@ from photools3.processing import ( process_batch, process_photo, ) +from photools3.settings import settings app = typer.Typer(help="PhotoTools3: A CLI for image processing.") -LoggingSettings() +LoggingSettings(LOG_LEVEL=settings.log_level) logger = logging.getLogger(__name__) diff --git a/src/photools3/llm_session.py b/src/photools3/llm_session.py index 14922af..68da969 100644 --- a/src/photools3/llm_session.py +++ b/src/photools3/llm_session.py @@ -1,11 +1,13 @@ import json import logging import re +import time import openai from openai import OpenAI from pydantic import BaseModel, ValidationError +from photools3.lm_studio import LMStudioModelManager from photools3.models import ( LlmMessageContent, LlmMessageRole, @@ -30,19 +32,15 @@ def _clean_llm_response(txt: str): return txt -def _correction_prompt( - original_request: LlmRequest, exception: Exception, origial_json -): +def _correction_prompt(exception: Exception, original_json: str) -> LlmRequestMessage: correction_prompt = ( f"The previous response was invalid due to: {exception}. " f"Please provide the corrected JSON based on this content: " - f"{origial_json}" + f"{original_json}" ) - original_request.messages.append( - LlmRequestMessage( - role=LlmMessageRole.user, - content=correction_prompt, - ) + return LlmRequestMessage( + role=LlmMessageRole.user, + content=correction_prompt, ) @@ -56,6 +54,7 @@ class LlmSession: self, system_prompt: str, temperature: float = settings.llm_default_temperature ): self._models: list = [] + self._model_id: str | None = None self.system_prompt = system_prompt self.temperature = temperature @@ -64,6 +63,7 @@ class LlmSession: base_url=self.base_url, api_key=self.api_key, ) + self._model_manager = LMStudioModelManager(self.base_url, self.api_key) def close(self): self.client.close() @@ -74,6 +74,12 @@ class LlmSession: def __exit__(self, exc_type, exc_value, traceback): self.close() + def flush_model(self): + """Unload and reload this session's model through LM Studio.""" + if self._model_id is None: + self._model_id = self.models[0].id + self._model_manager.reload_model(self._model_id) + def client_chat_completion_create(self, request: LlmRequest, choice_no: int = 0): response = self.client.chat.completions.create(**request.model_dump()) return response.choices[choice_no].message.content @@ -95,34 +101,62 @@ class LlmSession: self, photos: list[Photo], model_id=None, - max_retries: int = 2, + max_retries: int | None = None, prompt: str | None = None, ): - if not model_id: - model_id = self.models[0].id + retry_limit = settings.llm_max_retries if max_retries is None else max_retries + if retry_limit < 0: + raise ValueError("max_retries must be greater than or equal to zero") - request = self._build_request(prompt or self.prompt, photos, model_id) + if model_id is None: + if self._model_id is None: + self._model_id = self.models[0].id + model_id = self._model_id + else: + self._model_id = model_id - last_error = None - for attempt in range(max_retries + 1): + base_request = self._build_request(prompt or self.prompt, photos, model_id) + request = base_request + + for attempt in range(retry_limit + 1): + request_started = time.perf_counter() try: result_raw = self.client_chat_completion_create(request) + except Exception: + logger.debug( + "LLM request attempt %d/%d failed after %.2f seconds", + attempt + 1, + retry_limit + 1, + time.perf_counter() - request_started, + ) + raise + + logger.debug( + "LLM request attempt %d/%d completed in %.2f seconds", + attempt + 1, + retry_limit + 1, + time.perf_counter() - request_started, + ) + try: json_str = _clean_llm_response(result_raw) return self.response_model.model_validate_json(json_str) except (json.JSONDecodeError, ValidationError) as exc: - last_error = exc - if attempt < max_retries: - logger.info("Chatbot response is invalid; promting to correct ...") - _correction_prompt( - request, + if attempt < retry_limit: + logger.info("Chatbot response is invalid; prompting to correct ...") + correction = _correction_prompt( exc, - json_str if "json_str" in locals() else "origial json", + json_str if "json_str" in locals() else "unparseable response", + ) + # Start each retry from the initial prompt and only the latest + # correction; do not grow the conversation with all past retries. + request = base_request.model_copy( + update={"messages": [*base_request.messages, correction]} ) continue else: raise ValueError( - f"Failed to parse or validate LLM response after {max_retries} retries: {exc}" + f"Failed to parse or validate LLM response after {retry_limit} retries: {exc}" ) except Exception: raise diff --git a/src/photools3/lm_studio.py b/src/photools3/lm_studio.py new file mode 100644 index 0000000..cb5c1e0 --- /dev/null +++ b/src/photools3/lm_studio.py @@ -0,0 +1,135 @@ +import json +import logging +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request, urlopen + +logger = logging.getLogger(__name__) + +_LOAD_CONFIG_KEYS = ( + "context_length", + "eval_batch_size", + "flash_attention", + "num_experts", + "offload_kv_cache_to_gpu", +) + + +class LMStudioError(RuntimeError): + """Raised when LM Studio model management fails.""" + + +class LMStudioModelManager: + """Unload and reload a model through LM Studio's native REST API.""" + + def __init__(self, openai_base_url: str, api_key: str, timeout: float = 600): + parsed_url = urlsplit(openai_base_url) + path = parsed_url.path.rstrip("/") + path = path.removesuffix("/v1") + root_url = urlunsplit( + (parsed_url.scheme, parsed_url.netloc, path, "", "") + ).rstrip("/") + self.api_base_url = f"{root_url}/api/v1" + self.api_key = api_key + self.timeout = timeout + + def reload_model(self, model_id: str) -> None: + """Unload the matching loaded instance and reload it with its config.""" + started = time.perf_counter() + models_response = self._request("GET", "/models") + model, loaded_instance = self._find_model(models_response, model_id) + model_key = model["key"] + + if loaded_instance is not None: + self._request( + "POST", + "/models/unload", + {"instance_id": loaded_instance["id"]}, + ) + load_config = loaded_instance.get("config", {}) + else: + load_config = {} + + load_body = {"model": model_key} + load_body.update( + { + key: load_config[key] + for key in _LOAD_CONFIG_KEYS + if key in load_config and load_config[key] is not None + } + ) + self._request("POST", "/models/load", load_body) + logger.info( + "Reloaded LM Studio model %s in %.1f seconds", + model_key, + time.perf_counter() - started, + ) + + @staticmethod + def _find_model(models_response: dict, model_id: str) -> tuple[dict, dict | None]: + models = models_response.get("models") + if not isinstance(models, list): + raise LMStudioError("LM Studio model list response has no models array") + + for model in models: + loaded_instances = model.get("loaded_instances", []) + for instance in loaded_instances: + if instance.get("id") == model_id: + return model, instance + + matching_models = [model for model in models if model.get("key") == model_id] + if len(matching_models) != 1: + raise LMStudioError( + f"Could not uniquely identify LM Studio model {model_id!r}" + ) + + model = matching_models[0] + loaded_instances = model.get("loaded_instances", []) + if len(loaded_instances) > 1: + raise LMStudioError( + f"Multiple loaded instances found for LM Studio model {model_id!r}" + ) + loaded_instance = loaded_instances[0] if loaded_instances else None + return model, loaded_instance + + def _request(self, method: str, path: str, body: dict | None = None) -> dict: + data = None if body is None else json.dumps(body).encode("utf-8") + headers = {"Accept": "application/json"} + if data is not None: + headers["Content-Type"] = "application/json" + if self.api_key: + headers["Authorization"] = f"Bearer {self.api_key}" + + request = Request( + f"{self.api_base_url}{path}", + data=data, + headers=headers, + method=method, + ) + try: + with urlopen(request, timeout=self.timeout) as response: + response_body = response.read() + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise LMStudioError( + f"LM Studio {method} {path} returned HTTP {exc.code}: {detail}" + ) from exc + except (URLError, TimeoutError) as exc: + raise LMStudioError( + f"Could not reach LM Studio at {self.api_base_url}: {exc}" + ) from exc + + if not response_body: + return {} + try: + result = json.loads(response_body) + except json.JSONDecodeError as exc: + raise LMStudioError( + f"LM Studio {method} {path} returned invalid JSON" + ) from exc + if not isinstance(result, dict): + raise LMStudioError( + f"LM Studio {method} {path} returned an unexpected response" + ) + return result diff --git a/src/photools3/processing.py b/src/photools3/processing.py index 5e71ee1..747ae28 100644 --- a/src/photools3/processing.py +++ b/src/photools3/processing.py @@ -1,4 +1,5 @@ import logging +import time from collections.abc import Iterable from dataclasses import dataclass from enum import StrEnum @@ -17,6 +18,7 @@ from photools3.models import ( LlmResponseKeywords, ) from photools3.photo import Photo +from photools3.settings import settings logger = logging.getLogger(__name__) @@ -65,6 +67,8 @@ def process_photo( owns_session = session is None active_session = session if session is not None else _create_session(mode) + logger.info(f"Processing photo: {path!s}") + try: with Photo(path) as photo: result = active_session.call([photo]) @@ -104,10 +108,19 @@ def process_batch( mode = ProcessingMode(mode) session = _create_session(mode) outcomes: list[ProcessingOutcome] = [] + flush_every = settings.llm_flush_every_photos try: - for item in paths: + for attempted_photos, item in enumerate(paths): + if flush_every and attempted_photos and attempted_photos % flush_every == 0: + logger.info( + "Flushing LM Studio model after %d attempted photos", + attempted_photos, + ) + session.flush_model() + path = Path(item) + photo_started = time.perf_counter() try: result = process_photo(path, mode, session=session) except Exception as exc: @@ -122,6 +135,12 @@ def process_batch( ) else: outcomes.append(ProcessingOutcome(path=path, result=result)) + finally: + logger.debug( + "Photo processing for %s took %.2f seconds", + path, + time.perf_counter() - photo_started, + ) finally: session.close() diff --git a/src/photools3/settings.py b/src/photools3/settings.py index 2ceae41..7503a2b 100644 --- a/src/photools3/settings.py +++ b/src/photools3/settings.py @@ -1,5 +1,6 @@ from pathlib import Path +from pydantic import Field from pydantic_core import Url from pydantic_settings import BaseSettings, SettingsConfigDict @@ -13,11 +14,14 @@ KEYWORDS_LIST_PATH = CONFIG_DIR / "keywords.txt" class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env") + model_config = SettingsConfigDict(env_file=".env", extra="ignore") llm_api_base_url: Url llm_api_key: str llm_default_temperature: float + llm_flush_every_photos: int = Field(default=0, ge=0) + llm_max_retries: int = Field(default=5, ge=0) + log_level: str = "INFO" metadata_prefix: str diff --git a/test/test_llm_session_mocked.py b/test/test_llm_session_mocked.py index c6b9bb0..f7e1bc0 100644 --- a/test/test_llm_session_mocked.py +++ b/test/test_llm_session_mocked.py @@ -1,5 +1,7 @@ from unittest.mock import MagicMock, patch +import pytest + from photools3.llm_session import LlmSession @@ -84,3 +86,70 @@ def test_llm_session_captions_and_keywords_mocked(test_photo): # Verify that the mock was called assert mock_client.chat.completions.create.called + + +def test_llm_session_retries_from_original_request_without_accumulating_messages(): + with patch("photools3.llm_session.OpenAI") as mocked_openai: + mock_client = MagicMock() + mocked_openai.return_value = mock_client + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + + def response(content): + message = MagicMock() + message.content = content + choice = MagicMock() + choice.message = message + result = MagicMock() + result.choices = [choice] + return result + + mock_client.chat.completions.create.side_effect = [ + response('{"wrong": "first"}'), + response('{"wrong": "second"}'), + response('{"text": "correct"}'), + ] + + session = LlmSession(system_prompt="You are a helper.") + result = session.call(photos=[], prompt="Return JSON") + + assert result.text == "correct" + requests = mock_client.chat.completions.create.call_args_list + messages_per_attempt = [call.kwargs["messages"] for call in requests] + assert [len(messages) for messages in messages_per_attempt] == [2, 3, 3] + final_correction = messages_per_attempt[2][-1]["content"] + assert '"wrong": "second"' in final_correction + assert '"wrong": "first"' not in final_correction + + +def test_llm_session_uses_configured_retry_count(monkeypatch): + monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 1) + + with patch("photools3.llm_session.OpenAI") as mocked_openai: + mock_client = MagicMock() + mocked_openai.return_value = mock_client + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + + def response(content): + message = MagicMock() + message.content = content + choice = MagicMock() + choice.message = message + result = MagicMock() + result.choices = [choice] + return result + + mock_client.chat.completions.create.side_effect = [ + response('{"wrong": "first"}'), + response('{"wrong": "second"}'), + response('{"text": "would be a third call"}'), + ] + session = LlmSession(system_prompt="You are a helper.") + + with pytest.raises(ValueError, match="after 1 retries"): + session.call(photos=[]) + + assert mock_client.chat.completions.create.call_count == 2 diff --git a/test/test_lm_studio.py b/test/test_lm_studio.py new file mode 100644 index 0000000..254244b --- /dev/null +++ b/test/test_lm_studio.py @@ -0,0 +1,102 @@ +import json +from unittest.mock import MagicMock, patch + +import pytest + +from photools3.lm_studio import LMStudioError, LMStudioModelManager + + +def _http_response(payload: dict): + response = MagicMock() + response.__enter__.return_value = response + response.read.return_value = json.dumps(payload).encode("utf-8") + return response + + +def test_reload_model_unloads_and_reloads_preserving_context_config(): + model_response = { + "models": [ + { + "key": "qwen/qwen3-vl-4b", + "loaded_instances": [ + { + "id": "qwen/qwen3-vl-4b-instance", + "config": { + "context_length": 16384, + "eval_batch_size": 256, + "flash_attention": True, + "offload_kv_cache_to_gpu": False, + "parallel": 2, + }, + } + ], + } + ] + } + with patch( + "photools3.lm_studio.urlopen", + side_effect=[ + _http_response(model_response), + _http_response({"instance_id": "qwen/qwen3-vl-4b-instance"}), + _http_response({"instance_id": "qwen/qwen3-vl-4b-instance"}), + ], + ) as mocked_urlopen: + manager = LMStudioModelManager( + "http://localhost:1234/v1", "test-token", timeout=12 + ) + manager.reload_model("qwen/qwen3-vl-4b") + + requests = [call.args[0] for call in mocked_urlopen.call_args_list] + assert requests[0].full_url == "http://localhost:1234/api/v1/models" + assert requests[0].get_method() == "GET" + assert requests[0].get_header("Authorization") == "Bearer test-token" + + assert requests[1].full_url.endswith("/models/unload") + assert json.loads(requests[1].data) == {"instance_id": "qwen/qwen3-vl-4b-instance"} + + assert requests[2].full_url.endswith("/models/load") + assert json.loads(requests[2].data) == { + "model": "qwen/qwen3-vl-4b", + "context_length": 16384, + "eval_batch_size": 256, + "flash_attention": True, + "offload_kv_cache_to_gpu": False, + } + + +def test_reload_model_loads_by_key_if_no_instance_is_loaded(): + model_response = {"models": [{"key": "qwen/model", "loaded_instances": []}]} + with patch( + "photools3.lm_studio.urlopen", + side_effect=[ + _http_response(model_response), + _http_response({"status": "loaded"}), + ], + ) as mocked_urlopen: + manager = LMStudioModelManager("http://localhost:1234/v1", "token") + manager.reload_model("qwen/model") + + assert mocked_urlopen.call_count == 2 + load_request = mocked_urlopen.call_args.args[0] + assert load_request.full_url.endswith("/models/load") + assert json.loads(load_request.data) == {"model": "qwen/model"} + + +def test_reload_model_rejects_ambiguous_model_selection(): + model_response = { + "models": [ + { + "key": "qwen/model", + "loaded_instances": [ + {"id": "instance-1", "config": {}}, + {"id": "instance-2", "config": {}}, + ], + } + ] + } + with patch( + "photools3.lm_studio.urlopen", return_value=_http_response(model_response) + ): + manager = LMStudioModelManager("http://localhost:1234/v1", "token") + with pytest.raises(LMStudioError, match="Multiple loaded instances"): + manager.reload_model("qwen/model") diff --git a/test/test_processing.py b/test/test_processing.py index c1a27d3..a0069da 100644 --- a/test/test_processing.py +++ b/test/test_processing.py @@ -127,3 +127,39 @@ def test_process_batch_is_sequential_reuses_session_and_continues_after_error(): assert all(len(call.args[0]) == 1 for call in session.call.call_args_list) assert writer_class.return_value.write.call_count == 2 session.close.assert_called_once() + + +def test_process_batch_flushes_after_configured_attempt_count(monkeypatch): + monkeypatch.setattr("photools3.processing.settings.llm_flush_every_photos", 2) + + class FakePhoto: + def __init__(self, path): + self.path = path + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return None + + response = LlmResponseCaptionsAndKeywords(caption="Ein Haus", tags=["architektur"]) + session = MagicMock() + session.call.side_effect = [ + response, + RuntimeError("temporary error"), + response, + response, + response, + ] + paths = [Path(f"{i}.jpg") for i in range(5)] + + with ( + patch("photools3.processing._create_session", return_value=session), + patch("photools3.processing.Photo", FakePhoto), + patch("photools3.processing.MetadataWriter"), + ): + outcomes = process_batch(paths, ProcessingMode.BOTH) + + assert len(outcomes) == len(paths) + assert session.flush_model.call_count == 2 + session.close.assert_called_once() diff --git a/uv.lock b/uv.lock index b88df17..56a3bc3 100644 --- a/uv.lock +++ b/uv.lock @@ -780,7 +780,7 @@ requires-dist = [ { name = "pytest", specifier = ">=9.1.1" }, { name = "ruff", specifier = ">=0.16.9" }, { name = "typer", specifier = ">=0.12.0" }, - { name = "utils", specifier = ">=0.1.2", index = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/simple" }, + { name = "utils", specifier = ">=0.1.3", index = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/simple" }, ] [[package]] @@ -1205,7 +1205,7 @@ wheels = [ [[package]] name = "utils" -version = "0.1.2" +version = "0.1.3" source = { registry = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/simple" } dependencies = [ { name = "black" }, @@ -1217,9 +1217,9 @@ dependencies = [ { name = "twine" }, { name = "uvicorn" }, ] -sdist = { url = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/files/utils/0.1.2/utils-0.1.2.tar.gz", hash = "sha256:c711755c3e2fabd9cd380e25e45bc19da9b901868a6eb67028968803c725b118" } +sdist = { url = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/files/utils/0.1.3/utils-0.1.3.tar.gz", hash = "sha256:8970a179573a63ab4c546cfd1da8ceca0d915bc4357b49ccffaba9b3d4a0195e" } wheels = [ - { url = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/files/utils/0.1.2/utils-0.1.2-py3-none-any.whl", hash = "sha256:f3335ba10bd8eb9f5c000287a6ee41987b188372f3dca2783d99e6e70513a5d7" }, + { url = "https://gitea.schneiderserver.site/api/packages/mathias2784/pypi/files/utils/0.1.3/utils-0.1.3-py3-none-any.whl", hash = "sha256:c24078ba05f9f5b41fc3909598084b93c6df59f4ca60001e1e0b4fa55797e642" }, ] [[package]]