E2e workflow (#21)

Reviewed-on: #21
Co-authored-by: Mathias Scheider <mathias2784@web.de>

test/test_llm_session_mocked.py

@@ -1,6 +1,10 @@
 from unittest.mock import MagicMock, patch
 
-from photools3.llm_session import LlmSession
+import openai
+import pytest
+
+from photools3.llm_session import LlmSession, LlmSessionCaptionsAndKeywords
+from photools3.models import LlmResponseCaptionsAndKeywords
 
 
 def test_llm_session_mocked(test_photo):
@@ -84,3 +88,216 @@ def test_llm_session_captions_and_keywords_mocked(test_photo):
 
         # Verify that the mock was called
         assert mock_client.chat.completions.create.called
+
+
+def test_llm_session_retries_from_original_request_without_accumulating_messages():
+    with patch("photools3.llm_session.OpenAI") as mocked_openai:
+        mock_client = MagicMock()
+        mocked_openai.return_value = mock_client
+        mock_model = MagicMock()
+        mock_model.id = "gpt-4o"
+        mock_client.models.list.return_value.data = [mock_model]
+
+        def response(content):
+            message = MagicMock()
+            message.content = content
+            choice = MagicMock()
+            choice.message = message
+            result = MagicMock()
+            result.choices = [choice]
+            return result
+
+        mock_client.chat.completions.create.side_effect = [
+            response('{"wrong": "first"}'),
+            response('{"wrong": "second"}'),
+            response('{"text": "correct"}'),
+        ]
+
+        session = LlmSession(system_prompt="You are a helper.")
+        result = session.call(photos=[], prompt="Return JSON")
+
+    assert result.text == "correct"
+    requests = mock_client.chat.completions.create.call_args_list
+    messages_per_attempt = [call.kwargs["messages"] for call in requests]
+    assert [len(messages) for messages in messages_per_attempt] == [2, 3, 3]
+    final_correction = messages_per_attempt[2][-1]["content"]
+    assert "Field required" in final_correction
+    assert '"wrong"' not in final_correction
+
+
+def test_llm_session_uses_configured_retry_count(monkeypatch):
+    monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 1)
+
+    with patch("photools3.llm_session.OpenAI") as mocked_openai:
+        mock_client = MagicMock()
+        mocked_openai.return_value = mock_client
+        mock_model = MagicMock()
+        mock_model.id = "gpt-4o"
+        mock_client.models.list.return_value.data = [mock_model]
+
+        def response(content):
+            message = MagicMock()
+            message.content = content
+            choice = MagicMock()
+            choice.message = message
+            result = MagicMock()
+            result.choices = [choice]
+            return result
+
+        mock_client.chat.completions.create.side_effect = [
+            response('{"wrong": "first"}'),
+            response('{"wrong": "second"}'),
+            response('{"text": "would be a third call"}'),
+        ]
+        session = LlmSession(system_prompt="You are a helper.")
+
+        with (
+            patch.object(session, "flush_model") as flush_model,
+            pytest.raises(ValueError, match="after 1 retries"),
+        ):
+            session.call(photos=[])
+
+    assert mock_client.chat.completions.create.call_count == 2
+    flush_model.assert_called_once()
+
+
+def test_llm_session_reloads_after_timeout_and_retries(monkeypatch):
+    monkeypatch.setattr(
+        "photools3.llm_session.settings.llm_request_timeout_seconds", 10
+    )
+    monkeypatch.setattr("photools3.llm_session.settings.llm_max_completion_tokens", 768)
+    monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 1)
+
+    with patch("photools3.llm_session.OpenAI") as mocked_openai:
+        mock_client = MagicMock()
+        mocked_openai.return_value = mock_client
+        mock_model = MagicMock()
+        mock_model.id = "gpt-4o"
+        mock_client.models.list.return_value.data = [mock_model]
+
+        response_message = MagicMock(content='{"text": "recovered"}')
+        response_choice = MagicMock(message=response_message)
+        success_response = MagicMock(choices=[response_choice])
+        mock_client.chat.completions.create.side_effect = [
+            openai.APITimeoutError(request=MagicMock()),
+            success_response,
+        ]
+
+        session = LlmSession(system_prompt="You are a helper.")
+        with patch.object(session, "flush_model") as flush_model:
+            result = session.call(photos=[])
+
+    assert result.text == "recovered"
+    flush_model.assert_called_once()
+    calls = mock_client.chat.completions.create.call_args_list
+    assert len(calls) == 2
+    assert all(call.kwargs["timeout"] == 10 for call in calls)
+    assert all(call.kwargs["max_tokens"] == 768 for call in calls)
+    mocked_openai.assert_called_once_with(
+        base_url=session.base_url,
+        api_key=session.api_key,
+        max_retries=0,
+    )
+
+
+@pytest.mark.parametrize("failure_kind", ["invalid_response", "timeout"])
+def test_llm_session_reloads_when_retry_limit_is_exhausted(monkeypatch, failure_kind):
+    monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 0)
+
+    with patch("photools3.llm_session.OpenAI") as mocked_openai:
+        mock_client = MagicMock()
+        mocked_openai.return_value = mock_client
+        mock_model = MagicMock()
+        mock_model.id = "gpt-4o"
+        mock_client.models.list.return_value.data = [mock_model]
+
+        if failure_kind == "timeout":
+            mock_client.chat.completions.create.side_effect = openai.APITimeoutError(
+                request=MagicMock()
+            )
+        else:
+            response_message = MagicMock(content='{"wrong": "shape"}')
+            response_choice = MagicMock(message=response_message)
+            mock_client.chat.completions.create.return_value = MagicMock(
+                choices=[response_choice]
+            )
+
+        session = LlmSession(system_prompt="You are a helper.")
+        with (
+            patch.object(session, "flush_model") as flush_model,
+            pytest.raises(ValueError),
+        ):
+            session.call(photos=[])
+
+    flush_model.assert_called_once()
+    assert mock_client.chat.completions.create.call_count == 1
+
+
+def test_invalid_keywords_get_one_correction_then_invalid_tags_are_dropped(monkeypatch):
+    monkeypatch.setattr("photools3.llm_session.settings.llm_max_keywords", 3)
+    monkeypatch.setattr("photools3.llm_session.settings.llm_max_retries", 5)
+
+    with patch("photools3.llm_session.OpenAI") as mocked_openai:
+        mock_client = MagicMock()
+        mocked_openai.return_value = mock_client
+        mock_model = MagicMock()
+        mock_model.id = "gpt-4o"
+        mock_client.models.list.return_value.data = [mock_model]
+
+        def response(content):
+            message = MagicMock(content=content)
+            choice = MagicMock(message=message)
+            return MagicMock(choices=[choice])
+
+        mock_client.chat.completions.create.side_effect = [
+            response(
+                '{"caption": "A caption", "tags": ["not_allowed", "architektur"]}'
+            ),
+            response(
+                '{"caption": "A caption", '
+                '"tags": ["still_invalid", "architektur", "personen"]}'
+            ),
+        ]
+        session = LlmSessionCaptionsAndKeywords()
+        with patch.object(session, "flush_model") as flush_model:
+            result = session.call(photos=[])
+
+    assert result.tags == ["architektur", "personen"]
+    assert mock_client.chat.completions.create.call_count == 2
+    flush_model.assert_not_called()
+    correction = mock_client.chat.completions.create.call_args_list[1].kwargs[
+        "messages"
+    ][-1]["content"]
+    assert "not_allowed" in correction
+    assert '"caption"' not in correction
+
+
+def test_excess_keywords_are_truncated_to_configured_limit(monkeypatch):
+    monkeypatch.setattr("photools3.llm_session.settings.llm_max_keywords", 2)
+
+    with patch("photools3.llm_session.OpenAI") as mocked_openai:
+        mock_client = MagicMock()
+        mocked_openai.return_value = mock_client
+        mock_model = MagicMock()
+        mock_model.id = "gpt-4o"
+        mock_client.models.list.return_value.data = [mock_model]
+        message = MagicMock(
+            content=(
+                '{"caption": "A caption", '
+                '"tags": ["architektur", "personen", "landschaft"]}'
+            )
+        )
+        mock_client.chat.completions.create.return_value = MagicMock(
+            choices=[MagicMock(message=message)]
+        )
+
+        result = LlmSessionCaptionsAndKeywords().call(photos=[])
+
+    assert result.tags == ["architektur", "personen"]
+    assert mock_client.chat.completions.create.call_count == 1
+
+
+def test_keyword_response_schema_includes_configured_max_items():
+    schema = LlmResponseCaptionsAndKeywords.model_json_schema()
+
+    assert schema["properties"]["tags"]["maxItems"] == 15

test/test_lm_studio.py

@@ -0,0 +1,102 @@
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from photools3.lm_studio import LMStudioError, LMStudioModelManager
+
+
+def _http_response(payload: dict):
+    response = MagicMock()
+    response.__enter__.return_value = response
+    response.read.return_value = json.dumps(payload).encode("utf-8")
+    return response
+
+
+def test_reload_model_unloads_and_reloads_preserving_context_config():
+    model_response = {
+        "models": [
+            {
+                "key": "qwen/qwen3-vl-4b",
+                "loaded_instances": [
+                    {
+                        "id": "qwen/qwen3-vl-4b-instance",
+                        "config": {
+                            "context_length": 16384,
+                            "eval_batch_size": 256,
+                            "flash_attention": True,
+                            "offload_kv_cache_to_gpu": False,
+                            "parallel": 2,
+                        },
+                    }
+                ],
+            }
+        ]
+    }
+    with patch(
+        "photools3.lm_studio.urlopen",
+        side_effect=[
+            _http_response(model_response),
+            _http_response({"instance_id": "qwen/qwen3-vl-4b-instance"}),
+            _http_response({"instance_id": "qwen/qwen3-vl-4b-instance"}),
+        ],
+    ) as mocked_urlopen:
+        manager = LMStudioModelManager(
+            "http://localhost:1234/v1", "test-token", timeout=12
+        )
+        manager.reload_model("qwen/qwen3-vl-4b")
+
+    requests = [call.args[0] for call in mocked_urlopen.call_args_list]
+    assert requests[0].full_url == "http://localhost:1234/api/v1/models"
+    assert requests[0].get_method() == "GET"
+    assert requests[0].get_header("Authorization") == "Bearer test-token"
+
+    assert requests[1].full_url.endswith("/models/unload")
+    assert json.loads(requests[1].data) == {"instance_id": "qwen/qwen3-vl-4b-instance"}
+
+    assert requests[2].full_url.endswith("/models/load")
+    assert json.loads(requests[2].data) == {
+        "model": "qwen/qwen3-vl-4b",
+        "context_length": 16384,
+        "eval_batch_size": 256,
+        "flash_attention": True,
+        "offload_kv_cache_to_gpu": False,
+    }
+
+
+def test_reload_model_loads_by_key_if_no_instance_is_loaded():
+    model_response = {"models": [{"key": "qwen/model", "loaded_instances": []}]}
+    with patch(
+        "photools3.lm_studio.urlopen",
+        side_effect=[
+            _http_response(model_response),
+            _http_response({"status": "loaded"}),
+        ],
+    ) as mocked_urlopen:
+        manager = LMStudioModelManager("http://localhost:1234/v1", "token")
+        manager.reload_model("qwen/model")
+
+    assert mocked_urlopen.call_count == 2
+    load_request = mocked_urlopen.call_args.args[0]
+    assert load_request.full_url.endswith("/models/load")
+    assert json.loads(load_request.data) == {"model": "qwen/model"}
+
+
+def test_reload_model_rejects_ambiguous_model_selection():
+    model_response = {
+        "models": [
+            {
+                "key": "qwen/model",
+                "loaded_instances": [
+                    {"id": "instance-1", "config": {}},
+                    {"id": "instance-2", "config": {}},
+                ],
+            }
+        ]
+    }
+    with patch(
+        "photools3.lm_studio.urlopen", return_value=_http_response(model_response)
+    ):
+        manager = LMStudioModelManager("http://localhost:1234/v1", "token")
+        with pytest.raises(LMStudioError, match="Multiple loaded instances"):
+            manager.reload_model("qwen/model")