From ae7e1684e49ad1b75a8c9d94796c5658d58f8812 Mon Sep 17 00:00:00 2001 From: Mathias Scheider Date: Fri, 2 Oct 2026 10:13:03 +0200 Subject: [PATCH] refactor: move llm tests to separate integration and mocked files and centralize fixtures in conftest.py --- src/photools3/llm_session.py | 56 ++++++++++++++++++---------- test/conftest.py | 17 +++++++++ test/test_llm_session.py | 14 ------- test/test_llm_session_integration.py | 10 +++++ test/test_llm_session_mocked.py | 46 +++++++++++++++++++++++ 5 files changed, 110 insertions(+), 33 deletions(-) create mode 100644 test/conftest.py delete mode 100644 test/test_llm_session.py create mode 100644 test/test_llm_session_integration.py create mode 100644 test/test_llm_session_mocked.py diff --git a/src/photools3/llm_session.py b/src/photools3/llm_session.py index bb839c8..2310135 100644 --- a/src/photools3/llm_session.py +++ b/src/photools3/llm_session.py @@ -2,6 +2,7 @@ import json import re from openai import OpenAI +import openai from openai.types.chat import ChatCompletion from photools3.models import ( @@ -14,7 +15,7 @@ from photools3.models import ( from photools3.photo import Photo from photools3.settings import Settings -settings = Settings() +settings = Settings() # type: ignore[reportCallIssue] class LlmSession: @@ -32,6 +33,17 @@ class LlmSession: api_key=self.api_key, ) + def client_chat_completion_create(self, request: LlmRequest, choice_no: int = 0): + response = self.client.chat.completions.create(**request.model_dump()) + return response.choices[choice_no].message.content + + def is_api_reachable(self) -> bool: + try: + _ = self.client.models.list().data + return True + except openai.APIConnectionError: + return False + @property def models(self): if not self._models: @@ -43,11 +55,33 @@ class LlmSession: prompt: str, photos: list[Photo], model_id=None, - ) -> ChatCompletion: + ): if not model_id: model_id = self.models[0].id - request = LlmRequest( + request = self._build_request(prompt, photos, model_id) + result_raw = self.client_chat_completion_create(request) + + try: + result = json.loads(result_raw) + except json.JSONDecodeError as e: + try: + json_str = re.sub( + r"^```json(.*)```$", r"\1", result_raw, flags=re.DOTALL + ).strip() + result = json.loads(json_str) + except json.JSONDecodeError: + raise e + return result + + def _build_request( + self, + prompt: str, + photos: list[Photo], + model_id: str, + ) -> LlmRequest: + """Build a valid request that complies with openai API""" + return LlmRequest( model=model_id, temperature=self.temperature, messages=[ @@ -77,19 +111,3 @@ class LlmSession: ), ], ) - response = self.client.chat.completions.create(**request.model_dump()) - result_raw = response.choices[0].message.content - - # TODO: Add guard for non-JSON output - - try: - result = json.loads(result_raw) - except json.JSONDecodeError as e: - try: - json_str = re.sub( - r"^```json(.*)```$", r"\1", result_raw, flags=re.DOTALL - ).strip() - result = json.loads(json_str) - except json.JSONDecodeError: - raise e - return result diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..2c16a29 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,17 @@ +import pytest +from photools3.llm_session import LlmSession +from photools3.photo import Photo +from test.test_photo import PATHS_DNG_EXAMPLES + +@pytest.fixture +def test_photo(): + return Photo(PATHS_DNG_EXAMPLES[0]) + +@pytest.fixture +def llm_session(): + return LlmSession( + system_prompt=( + "You are a helpful assistent to photographer. " + 'Answer in JSON format: `{"caption": "...", "tags": ["tag1", "tag2", ...]}`' + ) + ) diff --git a/test/test_llm_session.py b/test/test_llm_session.py deleted file mode 100644 index eb1e4a0..0000000 --- a/test/test_llm_session.py +++ /dev/null @@ -1,14 +0,0 @@ -from photools3.llm_session import LlmSession -from photools3.photo import Photo -from test.test_photo import PATHS_DNG_EXAMPLES - -llm_session = LlmSession(system_prompt="You are a helpful assistent to photographer") - -test_photo = Photo(PATHS_DNG_EXAMPLES[0]) - - -def test_llm_session(): - result = llm_session.call( - prompt="keywords for the attaches picture", photos=[test_photo] - ) - assert result diff --git a/test/test_llm_session_integration.py b/test/test_llm_session_integration.py new file mode 100644 index 0000000..270c4c7 --- /dev/null +++ b/test/test_llm_session_integration.py @@ -0,0 +1,10 @@ +import pytest +from photools3.llm_session import LlmSession +from photools3.photo import Photo + +@pytest.mark.skipif(not LlmSession(system_prompt="").is_api_reachable(), reason="API not available") +def test_llm_session(llm_session, test_photo): + result = llm_session.call( + prompt="keywords for the attached picture", photos=[test_photo] + ) + assert result diff --git a/test/test_llm_session_mocked.py b/test/test_llm_session_mocked.py new file mode 100644 index 0000000..69c3560 --- /dev/null +++ b/test/test_llm_session_mocked.py @@ -0,0 +1,46 @@ +import pytest +from unittest.mock import MagicMock, patch +from photools3.llm_session import LlmSession +from photools3.photo import Photo +from test.test_photo import PATHS_DNG_EXAMPLES + +def test_llm_session_mocked(test_photo): + # We want to mock OpenAI client's chat.completions.create + with patch("photools3.llm_session.OpenAI") as mocked_openai: + # Setup the mock response structure + mock_client = MagicMock() + mocked_openai.return_value = mock_client + + # Mock the models list to return a proper object with an ID string + # This prevents Pydantic ValidationError when LlmSession tries to use model_id + mock_model = MagicMock() + mock_model.id = "gpt-4o" + mock_client.models.list.return_value.data = [mock_model] + + # Create a response object: response.choices[0].message.content + mock_response = MagicMock() + mock_choice = MagicMock() + mock_message = MagicMock() + + mock_message.content = '{"caption": "A beautiful sunset", "tags": ["sunset", "nature"]}' + mock_choice.message = mock_message + mock_response.choices = [mock_choice] + + # Configure the client to return our mock response + mock_client.chat.completions.create.return_value = mock_response + + # Create a new LlmSession instance that will use the mocked OpenAI client + mocked_session = LlmSession(system_prompt="You are a helper.") + + # Run the test + result = mocked_session.call( + prompt="keywords for the attached picture", photos=[test_photo] + ) + + # Verify the results + assert result["caption"] == "A beautiful sunset" + assert "sunset" in result["tags"] + assert "nature" in result["tags"] + + # Verify that the mock was called + assert mock_client.chat.completions.create.called