diff --git a/docs/04_using_models.md b/docs/04_using_models.md index 97ca694b..97d56cd9 100644 --- a/docs/04_using_models.md +++ b/docs/04_using_models.md @@ -20,7 +20,7 @@ with ComputerAgent() as agent: If you want to use another model, you select one of the available ones and set as an environment variable (**currently only supported for vlm_provider!**): ``` -VLM_PROVIDER_MODEL_ID=claude-opus-4-6 +VLM_PROVIDER_MODEL_ID=claude-opus-5 ``` Alternatively, you can also set it through overriding the model_id in the provider: @@ -30,7 +30,7 @@ from askui import AgentSettings, ComputerAgent from askui.model_providers import AskUIVlmProvider, AskUIImageQAProvider with ComputerAgent(settings=AgentSettings( - vlm_provider=AskUIVlmProvider(model_id="claude-opus-4-6"), + vlm_provider=AskUIVlmProvider(model_id="claude-opus-5"), image_qa_provider=AskUIImageQAProvider(model_id="gemini-2.5-pro"), )) as agent: agent.act("Complete the checkout process") @@ -38,12 +38,11 @@ with ComputerAgent(settings=AgentSettings( The following models are available with your AskUI credentials through the AskUI API: -**VLM Provider** (for `act()`): Claude models via AskUI's Anthropic proxy +**VLM Provider** (for `act()`): - `claude-haiku-4-5-20251001` (most cost efficient) -- `claude-sonnet-4-5-20250929` -- `claude-opus-4-5-20251101` -- `claude-sonnet-4-6`(default) -- `claude-opus-4-6` (most capable) +- - `gemini-3.5-flash` (fastest) +- `claude-sonnet-5`(default) +- `claude-opus-5` (most capable) **Image Q&A Provider** (for `get()`): Gemini models via AskUI's Gemini proxy diff --git a/src/askui/__init__.py b/src/askui/__init__.py index 79a13d20..a44495a8 100644 --- a/src/askui/__init__.py +++ b/src/askui/__init__.py @@ -1,6 +1,6 @@ """AskUI Python SDK""" -__version__ = "0.42.0" +__version__ = "0.44.0" import logging import os diff --git a/src/askui/android_agent.py b/src/askui/android_agent.py index 956655e7..916b78ec 100644 --- a/src/askui/android_agent.py +++ b/src/askui/android_agent.py @@ -12,6 +12,7 @@ from askui.models.models import Point from askui.models.shared.secrets import Secret from askui.models.shared.settings import ActSettings, MessageSettings +from askui.models.shared.thinking import make_non_thinking_settings from askui.models.shared.tools import Tool from askui.models.shared.truncation_strategies import TruncationStrategy from askui.prompts.act_prompts import create_android_agent_prompt @@ -116,12 +117,13 @@ def __init__( image_scaler=self._vlm_provider.image_scaler, ) self.act_tool_collection.add_agent_os(self.act_agent_os_facade) - # Override default act settings with Android-specific settings + # Override default act settings with Android-specific settings: + # thinking disabled, temperature 0 — where the model still accepts + # those (newer generations reject one or both). self.act_settings = ActSettings( messages=MessageSettings( system=create_android_agent_prompt(), - thinking={"type": "disabled"}, - temperature=0.0, + **make_non_thinking_settings(self._vlm_provider.model_id), ), ) diff --git a/src/askui/models/openai/messages_api.py b/src/askui/models/openai/messages_api.py index 83d8c4d4..ae023d90 100644 --- a/src/askui/models/openai/messages_api.py +++ b/src/askui/models/openai/messages_api.py @@ -291,8 +291,15 @@ def _from_openai_response(response: ChatCompletion) -> MessageParam: cached_tokens: int | None = None if response.usage.prompt_tokens_details is not None: cached_tokens = response.usage.prompt_tokens_details.cached_tokens + # OpenAI-style usage counts cached tokens INSIDE prompt_tokens, while + # `UsageParam` consumers (statistics callback, reporting) expect + # Anthropic-style disjoint fields — subtract so cached tokens are + # never counted or billed twice. + input_tokens = response.usage.prompt_tokens + if isinstance(cached_tokens, int) and cached_tokens > 0: + input_tokens = max(0, input_tokens - cached_tokens) usage = UsageParam( - input_tokens=response.usage.prompt_tokens, + input_tokens=input_tokens, output_tokens=response.usage.completion_tokens, cache_read_input_tokens=cached_tokens, ) diff --git a/src/askui/models/shared/agent_message_param.py b/src/askui/models/shared/agent_message_param.py index 07e118e5..200c0624 100644 --- a/src/askui/models/shared/agent_message_param.py +++ b/src/askui/models/shared/agent_message_param.py @@ -136,8 +136,9 @@ class BetaRedactedThinkingBlock(BaseModel): # adaptive thinking (`thinking={"type": "adaptive"}`). It replaces the integer # `budget_tokens` used with `thinking={"type": "enabled", ...}` on older models. # Anthropic maps this to `output_config.effort`; providers that do not support -# it (e.g. OpenAI chat) ignore it. -EffortLevel = Literal["low", "medium", "high", "max"] +# it (e.g. OpenAI chat) ignore it. "xhigh" (between "high" and "max") is +# supported from the Opus 4.7 generation onward; the 4.6 generation rejects it. +EffortLevel = Literal["low", "medium", "high", "xhigh", "max"] class UsageParam(BaseModel): diff --git a/src/askui/models/shared/thinking.py b/src/askui/models/shared/thinking.py index a3ff58c4..5fe1f94b 100644 --- a/src/askui/models/shared/thinking.py +++ b/src/askui/models/shared/thinking.py @@ -15,6 +15,14 @@ rejected, and ``effort`` is a separate parameter sent via ``output_config.effort`` (not part of ``thinking``). +The budget-token generation is a closed set, so classification is inverted: +every Claude model *not* in the frozen legacy list is treated as adaptive, +which makes unknown future models work by default. Model IDs are matched on +their ``claude-...`` core, so gateway-prefixed identifiers (Bedrock +``anthropic.claude-opus-4-8`` or ``us.anthropic.claude-...-v1:0``, LiteLLM +``anthropic/claude-...``, Vertex ``claude-...@20260401``) resolve like the +bare model ID. + `make_thinking_settings()` returns the `MessageSettings` keyword arguments that enable thinking for a given ``model_id``, so agents can turn thinking on by default without knowing which generation they run on. Callers can still override @@ -27,33 +35,115 @@ _DEFAULT_BUDGET_TOKENS = 2048 -# Model-ID prefixes for Anthropic models that use adaptive thinking. These are -# the models where the integer `budget_tokens` is removed or deprecated in favour -# of adaptive thinking (`{"type": "adaptive"}`) plus the `effort` setting. -# `str.startswith` matches dated snapshots too (e.g. "claude-sonnet-4-6-20260401"). -# Note: "claude-sonnet-5" does not match the older "claude-sonnet-4-5". -_ADAPTIVE_THINKING_MODEL_PREFIXES = ( +# Model-ID prefixes (after normalization) of the Anthropic model families that +# take the fixed integer `budget_tokens`. This set is FROZEN: budget thinking +# was replaced by adaptive thinking with the 4.6 generation, so no future model +# will ever be added here. +_LEGACY_BUDGET_THINKING_MODEL_PREFIXES = ( + "claude-2", + "claude-instant", + "claude-3-", + "claude-haiku-4-5", + "claude-sonnet-4-0", + "claude-sonnet-4-1", + "claude-sonnet-4-2", # dated snapshots, e.g. "claude-sonnet-4-20250514" + "claude-sonnet-4-5", + "claude-opus-4-0", + "claude-opus-4-1", + "claude-opus-4-2", # dated snapshots, e.g. "claude-opus-4-20250514" + "claude-opus-4-5", +) + +# The one adaptive-thinking generation that still accepts sampling parameters +# (temperature/top_p/top_k). From Opus 4.7 / Sonnet 5 / Fable 5 onward the API +# rejects them with a 400. +_SAMPLING_CAPABLE_ADAPTIVE_MODEL_PREFIXES = ( "claude-sonnet-4-6", - "claude-sonnet-5", "claude-opus-4-6", - "claude-opus-4-7", - "claude-opus-4-8", - "claude-opus-5", +) + +# Models where thinking is always on: an explicit {"type": "disabled"} is +# rejected with a 400, so the thinking field must be omitted entirely. +_ALWAYS_ON_THINKING_MODEL_PREFIXES = ( "claude-fable-5", + "claude-mythos", ) +def _normalize(model_id: str) -> str | None: + """Extract the ``claude-...`` core of a model ID. + + Gateway wrappers then match like bare IDs (e.g. + ``"us.anthropic.claude-opus-4-8-v1:0"`` and ``"anthropic/claude-opus-4-8"`` + both normalize to ``"claude-opus-4-8..."``). + + Args: + model_id (str): The (possibly gateway-prefixed) model identifier. + + Returns: + str | None: The model ID from its ``claude-`` core onward, or ``None`` + if the ID does not reference a Claude model. + """ + index = model_id.find("claude-") + return None if index < 0 else model_id[index:] + + def uses_adaptive_thinking(model_id: str) -> bool: """Whether ``model_id`` uses adaptive thinking instead of a token budget. + True for every Claude model outside the frozen legacy budget families (so + unknown future models default to adaptive); False for non-Claude model IDs. + Args: - model_id (str): The Anthropic model identifier. + model_id (str): The model identifier (bare or gateway-prefixed). Returns: bool: ``True`` if the model expects ``{"type": "adaptive"}`` and the `effort` setting, ``False`` if it expects a fixed ``budget_tokens``. """ - return model_id.startswith(_ADAPTIVE_THINKING_MODEL_PREFIXES) + normalized = _normalize(model_id) + return normalized is not None and not normalized.startswith( + _LEGACY_BUDGET_THINKING_MODEL_PREFIXES + ) + + +def accepts_sampling_params(model_id: str) -> bool: + """Whether the model accepts sampling parameters such as ``temperature``. + + False for adaptive-thinking Claude models newer than the 4.6 generation + (Opus 4.7/4.8, Sonnet 5, Fable 5, and future models), which reject them + with a 400. True for older Claude models and non-Claude model IDs (other + providers manage their own sampling parameters). + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + bool: ``True`` if sampling parameters may be sent to the model. + """ + normalized = _normalize(model_id) + return normalized is None or normalized.startswith( + _LEGACY_BUDGET_THINKING_MODEL_PREFIXES + + _SAMPLING_CAPABLE_ADAPTIVE_MODEL_PREFIXES + ) + + +def supports_disabled_thinking(model_id: str) -> bool: + """Whether the model accepts an explicit ``{"type": "disabled"}`` thinking config. + + False for always-on-thinking models (Fable 5, Mythos 5), which reject it + with a 400 — omit the thinking field there. + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + bool: ``True`` if ``{"type": "disabled"}`` may be sent to the model. + """ + normalized = _normalize(model_id) + return normalized is None or not normalized.startswith( + _ALWAYS_ON_THINKING_MODEL_PREFIXES + ) def make_thinking_settings( @@ -69,25 +159,58 @@ def make_thinking_settings( **make_thinking_settings(self._vlm_provider.model_id), ) - Models that support adaptive thinking get ``thinking={"type": "adaptive"}`` - (with ``effort`` sent via ``provider_options["output_config"]`` when given); - older models get a fixed token budget of - ``thinking={"type": "enabled", "budget_tokens": 2048}`` and ignore ``effort``. + Models that support adaptive thinking get + ``thinking={"type": "adaptive", "display": "summarized"}`` (with ``effort`` + sent via ``provider_options["output_config"]`` when given); older models + get a fixed token budget of + ``thinking={"type": "enabled", "budget_tokens": 2048}`` and ignore + ``effort``. ``display`` is set explicitly because the newest adaptive + models (Sonnet 5 generation onward) default it to ``"omitted"``, which + returns thinking blocks whose text is EMPTY while the full thinking + tokens are still billed — reasoning silently disappears from reports and + logs. ``"summarized"`` restores the visible text at no extra cost (billing + is identical for both display modes). Args: - model_id (str): The Anthropic model identifier. - effort (EffortLevel | None, optional): How much the model should think and - act (``"low"``, ``"medium"``, ``"high"`` or ``"max"``). Only applied - for models that support adaptive thinking. Default: None (the model - uses its own default). + model_id (str): The model identifier (bare or gateway-prefixed). + effort (EffortLevel | None, optional): How much the model should think + and act (``"low"``, ``"medium"``, ``"high"``, ``"xhigh"`` or + ``"max"``). Only applied for models that support adaptive thinking. + Default: None (the model uses its own default). Returns: dict[str, Any]: `MessageSettings` keyword arguments (``thinking`` and, when applicable, ``provider_options``). """ if uses_adaptive_thinking(model_id): - settings: dict[str, Any] = {"thinking": {"type": "adaptive"}} + settings: dict[str, Any] = { + "thinking": {"type": "adaptive", "display": "summarized"} + } if effort is not None: settings["provider_options"] = {"output_config": {"effort": effort}} return settings return {"thinking": {"type": "enabled", "budget_tokens": _DEFAULT_BUDGET_TOKENS}} + + +def make_non_thinking_settings(model_id: str) -> dict[str, Any]: + """Return `MessageSettings` keyword arguments for thinking-off, deterministic runs. + + Used by device agents (Android) that historically pinned + ``thinking={"type": "disabled"}`` and ``temperature=0.0``. Each field is + included only where the model still accepts it: models from the Opus 4.7 + generation onward reject sampling parameters, and always-on-thinking models + (Fable 5) reject an explicit ``"disabled"``. + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + dict[str, Any]: `MessageSettings` keyword arguments (``thinking`` + and/or ``temperature``, possibly empty). + """ + settings: dict[str, Any] = {} + if supports_disabled_thinking(model_id): + settings["thinking"] = {"type": "disabled"} + if accepts_sampling_params(model_id): + settings["temperature"] = 0.0 + return settings diff --git a/src/askui/reporting.py b/src/askui/reporting.py index 4090ea9b..99c4d43f 100644 --- a/src/askui/reporting.py +++ b/src/askui/reporting.py @@ -34,14 +34,30 @@ def normalize_to_pil_images( image: Image.Image | list[Image.Image] | AnnotatedImage | None, ) -> list[Image.Image]: - """Normalize various image input types to a list of PIL images.""" + """Normalize various image input types to a list of PIL images. + + Zero-size images (width or height of 0) are filtered out because PIL + cannot encode them to PNG, which would crash any reporter that tries. + """ if image is None: return [] if isinstance(image, AnnotatedImage): - return image.get_images() - if isinstance(image, list): - return image - return [image] + images: list[Image.Image] = image.get_images() + elif isinstance(image, list): + images = image + else: + images = [image] + valid = [] + for img in images: + if img.width == 0 or img.height == 0: + logger.warning( + "Skipping zero-size image (%dx%d) — cannot encode an empty image.", + img.width, + img.height, + ) + else: + valid.append(img) + return valid def _format_duration(seconds: float) -> str: diff --git a/src/askui/tools/askui/askui_controller.py b/src/askui/tools/askui/askui_controller.py index fc892858..e0e0f1ad 100644 --- a/src/askui/tools/askui/askui_controller.py +++ b/src/askui/tools/askui/askui_controller.py @@ -28,6 +28,7 @@ ) from askui.tools.askui.askui_ui_controller_grpc.desktop_agent_os_error import ( DesktopAgentOsError, + DesktopAgentOsException, ) from askui.tools.askui.askui_ui_controller_grpc.generated import ( Controller_V1_pb2 as controller_v1_pbs, @@ -378,6 +379,28 @@ def __exit__( """ self.disconnect() + @staticmethod + def _check_bitmap_dimensions(width: int, height: int) -> None: + """Raise `AskUiControllerError` when the captured bitmap has zero dimensions. + + A zero-size bitmap is returned when the display is unavailable, minimized, + or otherwise unable to produce a frame. PIL cannot encode such an image, + so we reject it here before it reaches the reporter or the model. + + Args: + width (int): Bitmap width returned by `CaptureScreen`. + height (int): Bitmap height returned by `CaptureScreen`. + + Raises: + AskUiControllerError: If either `width` or `height` is zero. + """ + if width == 0 or height == 0: + error_msg = ( + f"Screen capture returned an empty bitmap ({width}×{height}). " + "The display may be unavailable, minimized, or zero-sized." + ) + raise AskUiControllerError(error_msg) + @telemetry.record_call() @override def screenshot(self, report: bool = True, unscaled: bool = False) -> Image.Image: @@ -403,9 +426,12 @@ def screenshot(self, report: bool = True, unscaled: bool = False) -> Image.Image ), ) ) + width = screenResponse.bitmap.width + height = screenResponse.bitmap.height + self._check_bitmap_dimensions(width, height) r, g, b, _ = Image.frombytes( "RGBA", - (screenResponse.bitmap.width, screenResponse.bitmap.height), + (width, height), screenResponse.bitmap.data, ).split() image = Image.merge("RGB", (b, g, r)) @@ -1417,7 +1443,7 @@ def get_file_names(self, absolute_directory_path: str) -> list[str]: message = f"unexpected response type: {res}" raise DesktopAgentOsError(message) if res.error is not None: - raise DesktopAgentOsError(res.error) + raise DesktopAgentOsException(res.error) if res.response is None: message = f"{type(res).__name__} is missing both error and response" raise DesktopAgentOsError(message) @@ -1447,7 +1473,10 @@ def get_file(self, path: str) -> Image.Image | PdfSource | str: Image.Image | PdfSource | str: The decoded file contents. Raises: - DesktopAgentOsError: If the file cannot be read or the response is invalid. + DesktopAgentOsException: If the file cannot be read (e.g. the path + does not exist or its contents cannot be decoded). + DesktopAgentOsError: If the controller response violates the expected + protocol. """ self._reporter.add_message(self._REPORTER_SOURCE, f"get_file({path})") command = GetFileCommand(parameters=[path]) @@ -1456,7 +1485,7 @@ def get_file(self, path: str) -> Image.Image | PdfSource | str: message = f"unexpected response type: {res}" raise DesktopAgentOsError(message) if res.error is not None: - raise DesktopAgentOsError(res.error) + raise DesktopAgentOsException(res.error) if res.response is None: message = f"{type(res).__name__} is missing both error and response" raise DesktopAgentOsError(message) @@ -1510,7 +1539,7 @@ def _decode_file_payload(base64_data: str) -> Image.Image | PdfSource | str: except UnicodeDecodeError: pass message = "File contents are neither a supported image, PDF, nor UTF-8 text" - raise DesktopAgentOsError(message) + raise DesktopAgentOsException(message) AskUiControllerClient = MultiComputerTargetAgentOS diff --git a/src/askui/tools/askui/askui_ui_controller_grpc/desktop_agent_os_error.py b/src/askui/tools/askui/askui_ui_controller_grpc/desktop_agent_os_error.py index 14a66aba..2ac37161 100644 --- a/src/askui/tools/askui/askui_ui_controller_grpc/desktop_agent_os_error.py +++ b/src/askui/tools/askui/askui_ui_controller_grpc/desktop_agent_os_error.py @@ -1,5 +1,30 @@ -class DesktopAgentOsError(BaseException): - """Base class for Desktop Agent OS errors. +from askui.models.exceptions import AutomationError - This error is raised when an error occurs in the Desktop Agent OS. + +class DesktopAgentOsError(AutomationError): + """Unfixable error raised by the Desktop Agent OS. + + Raised when the Desktop Agent OS returns a response that violates the + expected protocol (e.g. an unexpected response type or a response missing + both an error and a payload). These indicate a broken controller or + connection rather than something the agent can recover from, so - like + other `AutomationError`s - they are re-raised by the tool-calling loop and + terminate the run. + + For failures the agent can react to and work around (e.g. a path that does + not exist), raise `DesktopAgentOsException` instead. + """ + + +class DesktopAgentOsException(Exception): # noqa: N818 + """Recoverable error raised by the Desktop Agent OS. + + Raised when an operation on the Desktop Agent OS fails in a way the agent + can react to and work around - for example, reading a file or directory + that does not exist, or a file whose contents cannot be decoded. Because it + derives from `Exception` (and not `AutomationError`), the tool-calling loop + catches it and surfaces it to the agent as a tool error result instead of + terminating the run. + + For unfixable protocol violations, raise `DesktopAgentOsError` instead. """ diff --git a/src/askui/tools/store/computer/experimental/get_file_names.py b/src/askui/tools/store/computer/experimental/get_file_names.py index 643820fb..dd313f58 100644 --- a/src/askui/tools/store/computer/experimental/get_file_names.py +++ b/src/askui/tools/store/computer/experimental/get_file_names.py @@ -28,9 +28,11 @@ def __init__(self, agent_os: ComputerAgentOS | None = None) -> None: name="get_file_names_tool", description=( "Lists the names of regular files in an absolute directory on the " - "computer under automation. Subdirectories are not included—only " - "files are returned. Use absolute paths as on the target machine. " - "Returns names only; use get_file_tool to read a file's contents." + "computer under automation. IMPORTANT: only files are returned - " + "folders and subdirectories are NOT included in the result and this " + "tool cannot be used to discover or navigate them. Use absolute " + "paths as on the target machine. Returns names only; use " + "get_file_tool to read a file's contents." ), input_schema={ "type": "object", diff --git a/tests/unit/models/openai/test_messages_api.py b/tests/unit/models/openai/test_messages_api.py index b3ebfeb2..fe8b74c0 100644 --- a/tests/unit/models/openai/test_messages_api.py +++ b/tests/unit/models/openai/test_messages_api.py @@ -41,8 +41,14 @@ def _make_completion( finish_reason: str = "stop", prompt_tokens: int = 10, completion_tokens: int = 20, + cached_tokens: int | None = None, ) -> ChatCompletion: """Create a mock ChatCompletion response.""" + prompt_tokens_details = None + if cached_tokens is not None: + from openai.types.completion_usage import PromptTokensDetails + + prompt_tokens_details = PromptTokensDetails(cached_tokens=cached_tokens) return ChatCompletion( id="chatcmpl-test", choices=[ @@ -63,6 +69,7 @@ def _make_completion( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=prompt_tokens_details, ), ) @@ -456,6 +463,31 @@ def test_usage_captured(self) -> None: assert result.usage.input_tokens == 50 assert result.usage.output_tokens == 100 + def test_cached_tokens_are_subtracted_from_input(self) -> None: + """OpenAI reports cached tokens as a SUBSET of prompt_tokens; + `UsageParam` consumers expect disjoint fields (Anthropic style). + Without the subtraction, cached tokens are counted and billed twice + (statistics callback, reporting).""" + completion = _make_completion( + content="ok", + prompt_tokens=1000, + completion_tokens=50, + cached_tokens=400, + ) + result = _from_openai_response(completion) + assert result.usage is not None + assert result.usage.input_tokens == 600 # 1000 - 400 cached + assert result.usage.cache_read_input_tokens == 400 + assert result.usage.output_tokens == 50 + + def test_cached_tokens_never_drive_input_negative(self) -> None: + completion = _make_completion( + content="ok", prompt_tokens=100, completion_tokens=5, cached_tokens=150 + ) + result = _from_openai_response(completion) + assert result.usage is not None + assert result.usage.input_tokens == 0 + class TestOpenAIMessagesApi: def test_create_message_delegates_to_client(self) -> None: diff --git a/tests/unit/models/shared/test_desktop_agent_os_error_handling.py b/tests/unit/models/shared/test_desktop_agent_os_error_handling.py new file mode 100644 index 00000000..6454cecb --- /dev/null +++ b/tests/unit/models/shared/test_desktop_agent_os_error_handling.py @@ -0,0 +1,77 @@ +"""Tests that Desktop Agent OS errors are routed by recoverability. + +The Desktop Agent OS raises two error types: + +- `DesktopAgentOsException` for failures the agent can react to (e.g. reading a + path that does not exist). The tool-calling loop catches it and surfaces it to + the agent as a tool error result so the run can continue. +- `DesktopAgentOsError` for unfixable protocol violations. It derives from + `AutomationError` and is re-raised by the tool-calling loop, terminating the + run instead of being fed back to the agent. +""" + +import pytest + +from askui.models.exceptions import AutomationError +from askui.models.shared.agent_message_param import ( + ToolResultBlockParam, + ToolUseBlockParam, +) +from askui.models.shared.tools import Tool, ToolCollection +from askui.tools.askui.askui_ui_controller_grpc.desktop_agent_os_error import ( + DesktopAgentOsError, + DesktopAgentOsException, +) + +_RECOVERABLE_MESSAGE = ( + "directory_iterator::directory_iterator: The system cannot find the " + 'path specified.: "FrontEnd\\Traces"' +) +_FATAL_MESSAGE = "unexpected response type: " + + +class _RaisingTool(Tool): + """A tool whose `__call__` raises the exception it was constructed with.""" + + _error: BaseException + + def __init__(self, error: BaseException) -> None: + super().__init__( + name="raising_tool", + description="Raises a preconfigured Desktop Agent OS error.", + ) + self._error = error + + def __call__(self) -> str: + raise self._error + + +def _run(tool: Tool) -> list: + collection = ToolCollection(tools=[tool]) + tool_use = ToolUseBlockParam(id="tool_use_1", input={}, name=tool.name) + return collection.run([tool_use]) + + +class TestDesktopAgentOsErrorHierarchy: + def test_error_is_an_automation_error(self) -> None: + assert issubclass(DesktopAgentOsError, AutomationError) + + def test_exception_is_a_plain_exception_not_automation_error(self) -> None: + assert issubclass(DesktopAgentOsException, Exception) + assert not issubclass(DesktopAgentOsException, AutomationError) + + +class TestDesktopAgentOsErrorHandling: + def test_recoverable_exception_returns_error_result(self) -> None: + results = _run(_RaisingTool(DesktopAgentOsException(_RECOVERABLE_MESSAGE))) + + assert len(results) == 1 + result = results[0] + assert isinstance(result, ToolResultBlockParam) + assert result.is_error is True + assert result.tool_use_id == "tool_use_1" + assert "FrontEnd\\Traces" in str(result.content) + + def test_fatal_error_propagates_and_terminates(self) -> None: + with pytest.raises(DesktopAgentOsError): + _run(_RaisingTool(DesktopAgentOsError(_FATAL_MESSAGE))) diff --git a/tests/unit/models/test_thinking.py b/tests/unit/models/test_thinking.py index 689eedc9..7249d82d 100644 --- a/tests/unit/models/test_thinking.py +++ b/tests/unit/models/test_thinking.py @@ -3,7 +3,10 @@ import pytest from askui.models.shared.thinking import ( + accepts_sampling_params, + make_non_thinking_settings, make_thinking_settings, + supports_disabled_thinking, uses_adaptive_thinking, ) @@ -17,6 +20,15 @@ "claude-opus-4-8", "claude-opus-5", "claude-fable-5", + "claude-haiku-5", # unknown future model -> adaptive by default + # Gateway-prefixed IDs (Bedrock, LiteLLM, Vertex) classify like bare IDs. + "anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8-v1:0", + "eu.anthropic.claude-sonnet-5-v1:0", + "anthropic/claude-opus-4-8", + "anthropic/claude-fable-5", + "bedrock/us.anthropic.claude-opus-4-7-v1:0", + "vertex_ai/claude-sonnet-4-6", ] _BUDGET_MODELS = [ @@ -25,6 +37,11 @@ "claude-opus-4-5-20251101", "claude-opus-4-1-20250805", "claude-haiku-4-5-20251001", + "claude-3-5-sonnet-20241022", + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic/claude-haiku-4-5", + "claude-opus-4-5@20251101", # Vertex version separator "gpt-5.4", "some-unknown-model", ] @@ -33,7 +50,9 @@ @pytest.mark.parametrize("model_id", _ADAPTIVE_MODELS) def test_adaptive_models_use_adaptive_thinking(model_id: str) -> None: assert uses_adaptive_thinking(model_id) is True - assert make_thinking_settings(model_id) == {"thinking": {"type": "adaptive"}} + assert make_thinking_settings(model_id) == { + "thinking": {"type": "adaptive", "display": "summarized"} + } @pytest.mark.parametrize("model_id", _BUDGET_MODELS) @@ -46,7 +65,7 @@ def test_other_models_use_budget_tokens(model_id: str) -> None: def test_effort_is_added_via_provider_options_for_adaptive_models() -> None: assert make_thinking_settings("claude-sonnet-5", effort="high") == { - "thinking": {"type": "adaptive"}, + "thinking": {"type": "adaptive", "display": "summarized"}, "provider_options": {"output_config": {"effort": "high"}}, } @@ -62,3 +81,66 @@ def test_sonnet_5_is_not_confused_with_sonnet_4_5() -> None: # "claude-sonnet-5" must not match the older "claude-sonnet-4-5" prefix. assert uses_adaptive_thinking("claude-sonnet-5") is True assert uses_adaptive_thinking("claude-sonnet-4-5") is False + + +@pytest.mark.parametrize( + ("model_id", "expected"), + [ + ("claude-sonnet-4-5-20250929", True), # legacy: sampling params fine + ("claude-haiku-4-5", True), + ("claude-sonnet-4-6", True), # 4.6 generation still accepts them + ("claude-opus-4-6", True), + ("claude-opus-4-7", False), # removed from Opus 4.7 onward + ("claude-opus-4-8", False), + ("claude-sonnet-5", False), + ("claude-fable-5", False), + ("anthropic.claude-opus-4-8", False), # gateway IDs classified too + ("anthropic/claude-sonnet-5", False), + ("gpt-5.4", True), # non-Claude: other providers manage their own + ], +) +def test_accepts_sampling_params(model_id: str, expected: bool) -> None: + assert accepts_sampling_params(model_id) is expected + + +@pytest.mark.parametrize( + ("model_id", "expected"), + [ + ("claude-sonnet-4-6", True), + ("claude-opus-4-8", True), + ("claude-sonnet-5", True), + ("claude-fable-5", False), # always-on thinking rejects "disabled" + ("claude-mythos-5", False), + ("anthropic/claude-fable-5", False), + ("gpt-5.4", True), + ], +) +def test_supports_disabled_thinking(model_id: str, expected: bool) -> None: + assert supports_disabled_thinking(model_id) is expected + + +def test_non_thinking_settings_keep_parity_on_older_models() -> None: + assert make_non_thinking_settings("claude-sonnet-4-6") == { + "thinking": {"type": "disabled"}, + "temperature": 0.0, + } + + +def test_non_thinking_settings_drop_temperature_from_opus_4_7_on() -> None: + assert make_non_thinking_settings("claude-opus-4-8") == { + "thinking": {"type": "disabled"}, + } + assert make_non_thinking_settings("anthropic/claude-sonnet-5") == { + "thinking": {"type": "disabled"}, + } + + +def test_non_thinking_settings_omit_thinking_on_always_on_models() -> None: + assert make_non_thinking_settings("claude-fable-5") == {} + + +def test_effort_supports_xhigh() -> None: + assert make_thinking_settings("claude-opus-4-8", effort="xhigh") == { + "thinking": {"type": "adaptive", "display": "summarized"}, + "provider_options": {"output_config": {"effort": "xhigh"}}, + } diff --git a/tests/unit/test_reporting.py b/tests/unit/test_reporting.py index 1e87779c..da5d7837 100644 --- a/tests/unit/test_reporting.py +++ b/tests/unit/test_reporting.py @@ -7,7 +7,9 @@ from typing import Any -from askui.reporting import truncate_base64_media +from PIL import Image + +from askui.reporting import normalize_to_pil_images, truncate_base64_media def _base64_source(media_type: str) -> dict[str, Any]: @@ -58,3 +60,40 @@ def test_leaves_plain_content_untouched(self) -> None: "type": "text", "text": "hello", } + + +class TestNormalizeToPilImages: + def test_none_returns_empty_list(self) -> None: + assert normalize_to_pil_images(None) == [] + + def test_single_valid_image_is_wrapped_in_list(self) -> None: + img = Image.new("RGB", (10, 10)) + result = normalize_to_pil_images(img) + assert result == [img] + + def test_list_of_valid_images_is_returned_as_is(self) -> None: + images = [Image.new("RGB", (10, 10)), Image.new("RGB", (20, 20))] + result = normalize_to_pil_images(images) + assert result == images + + def test_zero_width_image_is_filtered_out(self) -> None: + empty = Image.new("RGB", (0, 5)) + valid = Image.new("RGB", (10, 10)) + result = normalize_to_pil_images([empty, valid]) + assert result == [valid] + + def test_zero_height_image_is_filtered_out(self) -> None: + empty = Image.new("RGB", (5, 0)) + valid = Image.new("RGB", (10, 10)) + result = normalize_to_pil_images([empty, valid]) + assert result == [valid] + + def test_zero_by_zero_image_is_filtered_out(self) -> None: + result = normalize_to_pil_images(Image.new("RGB", (0, 0))) + assert result == [] + + def test_list_of_only_empty_images_returns_empty_list(self) -> None: + result = normalize_to_pil_images( + [Image.new("RGB", (0, 0)), Image.new("RGB", (0, 5))] + ) + assert result == [] diff --git a/tests/unit/tools/askui/test_askui_controller_client.py b/tests/unit/tools/askui/test_askui_controller_client.py index 4c007f5a..70a5a27b 100644 --- a/tests/unit/tools/askui/test_askui_controller_client.py +++ b/tests/unit/tools/askui/test_askui_controller_client.py @@ -214,3 +214,22 @@ def test_underlying_manager_is_an_agent_os_target_computer_manager(self) -> None agent_os_target_computers=[_make_local(computer_id="l")] ) assert isinstance(client.agent_os_target_computer_manager, ComputerTargetPool) + + +class TestScreenshotValidation: + """_check_bitmap_dimensions() raises AskUiControllerError for zero-size bitmaps.""" + + def test_zero_width_raises(self) -> None: + with pytest.raises(AskUiControllerError, match="empty bitmap"): + MultiComputerTargetAgentOS._check_bitmap_dimensions(0, 100) + + def test_zero_height_raises(self) -> None: + with pytest.raises(AskUiControllerError, match="empty bitmap"): + MultiComputerTargetAgentOS._check_bitmap_dimensions(100, 0) + + def test_zero_by_zero_raises(self) -> None: + with pytest.raises(AskUiControllerError, match="empty bitmap"): + MultiComputerTargetAgentOS._check_bitmap_dimensions(0, 0) + + def test_valid_dimensions_do_not_raise(self) -> None: + MultiComputerTargetAgentOS._check_bitmap_dimensions(1920, 1080) diff --git a/tests/unit/tools/askui/test_decode_file_payload.py b/tests/unit/tools/askui/test_decode_file_payload.py index 6d898e83..53206245 100644 --- a/tests/unit/tools/askui/test_decode_file_payload.py +++ b/tests/unit/tools/askui/test_decode_file_payload.py @@ -14,7 +14,7 @@ from askui.tools.askui.askui_controller import ( AskUiControllerClient, - DesktopAgentOsError, + DesktopAgentOsException, ) from askui.utils.pdf_utils import PdfSource @@ -47,5 +47,5 @@ def test_decodes_utf8_text(self) -> None: assert result == "hello world" def test_rejects_unsupported_binary(self) -> None: - with pytest.raises(DesktopAgentOsError): + with pytest.raises(DesktopAgentOsException): AskUiControllerClient._decode_file_payload(_b64(b"\x00\x01\x02\x03"))