From cb40764ef2f20238ff29a4fdee36f957518dd1a0 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Thu, 30 Jul 2026 07:55:31 +0200 Subject: [PATCH 01/10] chore: update model docs to new available models --- docs/04_using_models.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/04_using_models.md b/docs/04_using_models.md index 97ca694b..97d56cd9 100644 --- a/docs/04_using_models.md +++ b/docs/04_using_models.md @@ -20,7 +20,7 @@ with ComputerAgent() as agent: If you want to use another model, you select one of the available ones and set as an environment variable (**currently only supported for vlm_provider!**): ``` -VLM_PROVIDER_MODEL_ID=claude-opus-4-6 +VLM_PROVIDER_MODEL_ID=claude-opus-5 ``` Alternatively, you can also set it through overriding the model_id in the provider: @@ -30,7 +30,7 @@ from askui import AgentSettings, ComputerAgent from askui.model_providers import AskUIVlmProvider, AskUIImageQAProvider with ComputerAgent(settings=AgentSettings( - vlm_provider=AskUIVlmProvider(model_id="claude-opus-4-6"), + vlm_provider=AskUIVlmProvider(model_id="claude-opus-5"), image_qa_provider=AskUIImageQAProvider(model_id="gemini-2.5-pro"), )) as agent: agent.act("Complete the checkout process") @@ -38,12 +38,11 @@ with ComputerAgent(settings=AgentSettings( The following models are available with your AskUI credentials through the AskUI API: -**VLM Provider** (for `act()`): Claude models via AskUI's Anthropic proxy +**VLM Provider** (for `act()`): - `claude-haiku-4-5-20251001` (most cost efficient) -- `claude-sonnet-4-5-20250929` -- `claude-opus-4-5-20251101` -- `claude-sonnet-4-6`(default) -- `claude-opus-4-6` (most capable) +- - `gemini-3.5-flash` (fastest) +- `claude-sonnet-5`(default) +- `claude-opus-5` (most capable) **Image Q&A Provider** (for `get()`): Gemini models via AskUI's Gemini proxy From 7a1ae43f00ee99c5b6a25deae8d8c0471908df7e Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Fri, 31 Jul 2026 11:31:29 +0200 Subject: [PATCH 02/10] fix(models): correct thinking config for gateway model IDs and new model generations - thinking.py now normalizes model IDs to their claude- core, so Bedrock (anthropic.claude-..., us.anthropic....-v1:0), LiteLLM (anthropic/claude-...), and Vertex (claude-...@date) IDs classify like bare IDs instead of falling into the budget_tokens path (400 on >=4.7 generation models). - Inverted classification: adaptive thinking is the default; only the frozen legacy budget_tokens families (2.x/3.x, Haiku 4.5, Sonnet 4/4.5, Opus 4/4.1/4.5) keep the integer budget, so unknown future Claude models work by default. - New accepts_sampling_params() / supports_disabled_thinking() helpers; AndroidAgent act defaults now use make_non_thinking_settings(), which drops temperature on models that reject sampling params (Opus 4.7+, Sonnet 5, Fable 5) and omits thinking "disabled" on always-on models (Fable 5 / Mythos 5). - EffortLevel gains "xhigh" (supported from the Opus 4.7 generation on). Co-Authored-By: Claude Fable 5 --- src/askui/android_agent.py | 8 +- .../models/shared/agent_message_param.py | 5 +- src/askui/models/shared/thinking.py | 148 ++++++++++++++++-- tests/unit/models/test_thinking.py | 80 ++++++++++ 4 files changed, 219 insertions(+), 22 deletions(-) diff --git a/src/askui/android_agent.py b/src/askui/android_agent.py index 956655e7..916b78ec 100644 --- a/src/askui/android_agent.py +++ b/src/askui/android_agent.py @@ -12,6 +12,7 @@ from askui.models.models import Point from askui.models.shared.secrets import Secret from askui.models.shared.settings import ActSettings, MessageSettings +from askui.models.shared.thinking import make_non_thinking_settings from askui.models.shared.tools import Tool from askui.models.shared.truncation_strategies import TruncationStrategy from askui.prompts.act_prompts import create_android_agent_prompt @@ -116,12 +117,13 @@ def __init__( image_scaler=self._vlm_provider.image_scaler, ) self.act_tool_collection.add_agent_os(self.act_agent_os_facade) - # Override default act settings with Android-specific settings + # Override default act settings with Android-specific settings: + # thinking disabled, temperature 0 — where the model still accepts + # those (newer generations reject one or both). self.act_settings = ActSettings( messages=MessageSettings( system=create_android_agent_prompt(), - thinking={"type": "disabled"}, - temperature=0.0, + **make_non_thinking_settings(self._vlm_provider.model_id), ), ) diff --git a/src/askui/models/shared/agent_message_param.py b/src/askui/models/shared/agent_message_param.py index 07e118e5..200c0624 100644 --- a/src/askui/models/shared/agent_message_param.py +++ b/src/askui/models/shared/agent_message_param.py @@ -136,8 +136,9 @@ class BetaRedactedThinkingBlock(BaseModel): # adaptive thinking (`thinking={"type": "adaptive"}`). It replaces the integer # `budget_tokens` used with `thinking={"type": "enabled", ...}` on older models. # Anthropic maps this to `output_config.effort`; providers that do not support -# it (e.g. OpenAI chat) ignore it. -EffortLevel = Literal["low", "medium", "high", "max"] +# it (e.g. OpenAI chat) ignore it. "xhigh" (between "high" and "max") is +# supported from the Opus 4.7 generation onward; the 4.6 generation rejects it. +EffortLevel = Literal["low", "medium", "high", "xhigh", "max"] class UsageParam(BaseModel): diff --git a/src/askui/models/shared/thinking.py b/src/askui/models/shared/thinking.py index a3ff58c4..b668a12b 100644 --- a/src/askui/models/shared/thinking.py +++ b/src/askui/models/shared/thinking.py @@ -15,6 +15,14 @@ rejected, and ``effort`` is a separate parameter sent via ``output_config.effort`` (not part of ``thinking``). +The budget-token generation is a closed set, so classification is inverted: +every Claude model *not* in the frozen legacy list is treated as adaptive, +which makes unknown future models work by default. Model IDs are matched on +their ``claude-...`` core, so gateway-prefixed identifiers (Bedrock +``anthropic.claude-opus-4-8`` or ``us.anthropic.claude-...-v1:0``, LiteLLM +``anthropic/claude-...``, Vertex ``claude-...@20260401``) resolve like the +bare model ID. + `make_thinking_settings()` returns the `MessageSettings` keyword arguments that enable thinking for a given ``model_id``, so agents can turn thinking on by default without knowing which generation they run on. Callers can still override @@ -27,33 +35,115 @@ _DEFAULT_BUDGET_TOKENS = 2048 -# Model-ID prefixes for Anthropic models that use adaptive thinking. These are -# the models where the integer `budget_tokens` is removed or deprecated in favour -# of adaptive thinking (`{"type": "adaptive"}`) plus the `effort` setting. -# `str.startswith` matches dated snapshots too (e.g. "claude-sonnet-4-6-20260401"). -# Note: "claude-sonnet-5" does not match the older "claude-sonnet-4-5". -_ADAPTIVE_THINKING_MODEL_PREFIXES = ( +# Model-ID prefixes (after normalization) of the Anthropic model families that +# take the fixed integer `budget_tokens`. This set is FROZEN: budget thinking +# was replaced by adaptive thinking with the 4.6 generation, so no future model +# will ever be added here. +_LEGACY_BUDGET_THINKING_MODEL_PREFIXES = ( + "claude-2", + "claude-instant", + "claude-3-", + "claude-haiku-4-5", + "claude-sonnet-4-0", + "claude-sonnet-4-1", + "claude-sonnet-4-2", # dated snapshots, e.g. "claude-sonnet-4-20250514" + "claude-sonnet-4-5", + "claude-opus-4-0", + "claude-opus-4-1", + "claude-opus-4-2", # dated snapshots, e.g. "claude-opus-4-20250514" + "claude-opus-4-5", +) + +# The one adaptive-thinking generation that still accepts sampling parameters +# (temperature/top_p/top_k). From Opus 4.7 / Sonnet 5 / Fable 5 onward the API +# rejects them with a 400. +_SAMPLING_CAPABLE_ADAPTIVE_MODEL_PREFIXES = ( "claude-sonnet-4-6", - "claude-sonnet-5", "claude-opus-4-6", - "claude-opus-4-7", - "claude-opus-4-8", - "claude-opus-5", +) + +# Models where thinking is always on: an explicit {"type": "disabled"} is +# rejected with a 400, so the thinking field must be omitted entirely. +_ALWAYS_ON_THINKING_MODEL_PREFIXES = ( "claude-fable-5", + "claude-mythos", ) +def _normalize(model_id: str) -> str | None: + """Extract the ``claude-...`` core of a model ID. + + Gateway wrappers then match like bare IDs (e.g. + ``"us.anthropic.claude-opus-4-8-v1:0"`` and ``"anthropic/claude-opus-4-8"`` + both normalize to ``"claude-opus-4-8..."``). + + Args: + model_id (str): The (possibly gateway-prefixed) model identifier. + + Returns: + str | None: The model ID from its ``claude-`` core onward, or ``None`` + if the ID does not reference a Claude model. + """ + index = model_id.find("claude-") + return None if index < 0 else model_id[index:] + + def uses_adaptive_thinking(model_id: str) -> bool: """Whether ``model_id`` uses adaptive thinking instead of a token budget. + True for every Claude model outside the frozen legacy budget families (so + unknown future models default to adaptive); False for non-Claude model IDs. + Args: - model_id (str): The Anthropic model identifier. + model_id (str): The model identifier (bare or gateway-prefixed). Returns: bool: ``True`` if the model expects ``{"type": "adaptive"}`` and the `effort` setting, ``False`` if it expects a fixed ``budget_tokens``. """ - return model_id.startswith(_ADAPTIVE_THINKING_MODEL_PREFIXES) + normalized = _normalize(model_id) + return normalized is not None and not normalized.startswith( + _LEGACY_BUDGET_THINKING_MODEL_PREFIXES + ) + + +def accepts_sampling_params(model_id: str) -> bool: + """Whether the model accepts sampling parameters such as ``temperature``. + + False for adaptive-thinking Claude models newer than the 4.6 generation + (Opus 4.7/4.8, Sonnet 5, Fable 5, and future models), which reject them + with a 400. True for older Claude models and non-Claude model IDs (other + providers manage their own sampling parameters). + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + bool: ``True`` if sampling parameters may be sent to the model. + """ + normalized = _normalize(model_id) + return normalized is None or normalized.startswith( + _LEGACY_BUDGET_THINKING_MODEL_PREFIXES + + _SAMPLING_CAPABLE_ADAPTIVE_MODEL_PREFIXES + ) + + +def supports_disabled_thinking(model_id: str) -> bool: + """Whether the model accepts an explicit ``{"type": "disabled"}`` thinking config. + + False for always-on-thinking models (Fable 5, Mythos 5), which reject it + with a 400 — omit the thinking field there. + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + bool: ``True`` if ``{"type": "disabled"}`` may be sent to the model. + """ + normalized = _normalize(model_id) + return normalized is None or not normalized.startswith( + _ALWAYS_ON_THINKING_MODEL_PREFIXES + ) def make_thinking_settings( @@ -75,11 +165,11 @@ def make_thinking_settings( ``thinking={"type": "enabled", "budget_tokens": 2048}`` and ignore ``effort``. Args: - model_id (str): The Anthropic model identifier. - effort (EffortLevel | None, optional): How much the model should think and - act (``"low"``, ``"medium"``, ``"high"`` or ``"max"``). Only applied - for models that support adaptive thinking. Default: None (the model - uses its own default). + model_id (str): The model identifier (bare or gateway-prefixed). + effort (EffortLevel | None, optional): How much the model should think + and act (``"low"``, ``"medium"``, ``"high"``, ``"xhigh"`` or + ``"max"``). Only applied for models that support adaptive thinking. + Default: None (the model uses its own default). Returns: dict[str, Any]: `MessageSettings` keyword arguments (``thinking`` and, @@ -91,3 +181,27 @@ def make_thinking_settings( settings["provider_options"] = {"output_config": {"effort": effort}} return settings return {"thinking": {"type": "enabled", "budget_tokens": _DEFAULT_BUDGET_TOKENS}} + + +def make_non_thinking_settings(model_id: str) -> dict[str, Any]: + """Return `MessageSettings` keyword arguments for thinking-off, deterministic runs. + + Used by device agents (Android) that historically pinned + ``thinking={"type": "disabled"}`` and ``temperature=0.0``. Each field is + included only where the model still accepts it: models from the Opus 4.7 + generation onward reject sampling parameters, and always-on-thinking models + (Fable 5) reject an explicit ``"disabled"``. + + Args: + model_id (str): The model identifier (bare or gateway-prefixed). + + Returns: + dict[str, Any]: `MessageSettings` keyword arguments (``thinking`` + and/or ``temperature``, possibly empty). + """ + settings: dict[str, Any] = {} + if supports_disabled_thinking(model_id): + settings["thinking"] = {"type": "disabled"} + if accepts_sampling_params(model_id): + settings["temperature"] = 0.0 + return settings diff --git a/tests/unit/models/test_thinking.py b/tests/unit/models/test_thinking.py index 689eedc9..58682494 100644 --- a/tests/unit/models/test_thinking.py +++ b/tests/unit/models/test_thinking.py @@ -3,7 +3,10 @@ import pytest from askui.models.shared.thinking import ( + accepts_sampling_params, + make_non_thinking_settings, make_thinking_settings, + supports_disabled_thinking, uses_adaptive_thinking, ) @@ -17,6 +20,15 @@ "claude-opus-4-8", "claude-opus-5", "claude-fable-5", + "claude-haiku-5", # unknown future model -> adaptive by default + # Gateway-prefixed IDs (Bedrock, LiteLLM, Vertex) classify like bare IDs. + "anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8-v1:0", + "eu.anthropic.claude-sonnet-5-v1:0", + "anthropic/claude-opus-4-8", + "anthropic/claude-fable-5", + "bedrock/us.anthropic.claude-opus-4-7-v1:0", + "vertex_ai/claude-sonnet-4-6", ] _BUDGET_MODELS = [ @@ -25,6 +37,11 @@ "claude-opus-4-5-20251101", "claude-opus-4-1-20250805", "claude-haiku-4-5-20251001", + "claude-3-5-sonnet-20241022", + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "anthropic/claude-haiku-4-5", + "claude-opus-4-5@20251101", # Vertex version separator "gpt-5.4", "some-unknown-model", ] @@ -62,3 +79,66 @@ def test_sonnet_5_is_not_confused_with_sonnet_4_5() -> None: # "claude-sonnet-5" must not match the older "claude-sonnet-4-5" prefix. assert uses_adaptive_thinking("claude-sonnet-5") is True assert uses_adaptive_thinking("claude-sonnet-4-5") is False + + +@pytest.mark.parametrize( + ("model_id", "expected"), + [ + ("claude-sonnet-4-5-20250929", True), # legacy: sampling params fine + ("claude-haiku-4-5", True), + ("claude-sonnet-4-6", True), # 4.6 generation still accepts them + ("claude-opus-4-6", True), + ("claude-opus-4-7", False), # removed from Opus 4.7 onward + ("claude-opus-4-8", False), + ("claude-sonnet-5", False), + ("claude-fable-5", False), + ("anthropic.claude-opus-4-8", False), # gateway IDs classified too + ("anthropic/claude-sonnet-5", False), + ("gpt-5.4", True), # non-Claude: other providers manage their own + ], +) +def test_accepts_sampling_params(model_id: str, expected: bool) -> None: + assert accepts_sampling_params(model_id) is expected + + +@pytest.mark.parametrize( + ("model_id", "expected"), + [ + ("claude-sonnet-4-6", True), + ("claude-opus-4-8", True), + ("claude-sonnet-5", True), + ("claude-fable-5", False), # always-on thinking rejects "disabled" + ("claude-mythos-5", False), + ("anthropic/claude-fable-5", False), + ("gpt-5.4", True), + ], +) +def test_supports_disabled_thinking(model_id: str, expected: bool) -> None: + assert supports_disabled_thinking(model_id) is expected + + +def test_non_thinking_settings_keep_parity_on_older_models() -> None: + assert make_non_thinking_settings("claude-sonnet-4-6") == { + "thinking": {"type": "disabled"}, + "temperature": 0.0, + } + + +def test_non_thinking_settings_drop_temperature_from_opus_4_7_on() -> None: + assert make_non_thinking_settings("claude-opus-4-8") == { + "thinking": {"type": "disabled"}, + } + assert make_non_thinking_settings("anthropic/claude-sonnet-5") == { + "thinking": {"type": "disabled"}, + } + + +def test_non_thinking_settings_omit_thinking_on_always_on_models() -> None: + assert make_non_thinking_settings("claude-fable-5") == {} + + +def test_effort_supports_xhigh() -> None: + assert make_thinking_settings("claude-opus-4-8", effort="xhigh") == { + "thinking": {"type": "adaptive"}, + "provider_options": {"output_config": {"effort": "xhigh"}}, + } From 630c0000b9e14e399475e832f88ff72694f7540e Mon Sep 17 00:00:00 2001 From: Samir Mlika Date: Fri, 31 Jul 2026 13:05:34 +0200 Subject: [PATCH 03/10] fix(openai): report cached tokens disjoint from input_tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI-style usage counts prompt_tokens_details.cached_tokens INSIDE prompt_tokens, while UsageParam consumers (conversation statistics callback, HTML reporting) treat the fields as disjoint, Anthropic-style — so cached tokens were counted and billed twice for OpenAI-route models (observed ~5x cost overstatement on cache-heavy Gemini agentic runs). Subtract the cached subset at the response boundary so UsageParam means the same thing on every route. Guarded so absent/malformed details pass through unchanged. --- src/askui/models/openai/messages_api.py | 9 +++++- tests/unit/models/openai/test_messages_api.py | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/askui/models/openai/messages_api.py b/src/askui/models/openai/messages_api.py index 83d8c4d4..ae023d90 100644 --- a/src/askui/models/openai/messages_api.py +++ b/src/askui/models/openai/messages_api.py @@ -291,8 +291,15 @@ def _from_openai_response(response: ChatCompletion) -> MessageParam: cached_tokens: int | None = None if response.usage.prompt_tokens_details is not None: cached_tokens = response.usage.prompt_tokens_details.cached_tokens + # OpenAI-style usage counts cached tokens INSIDE prompt_tokens, while + # `UsageParam` consumers (statistics callback, reporting) expect + # Anthropic-style disjoint fields — subtract so cached tokens are + # never counted or billed twice. + input_tokens = response.usage.prompt_tokens + if isinstance(cached_tokens, int) and cached_tokens > 0: + input_tokens = max(0, input_tokens - cached_tokens) usage = UsageParam( - input_tokens=response.usage.prompt_tokens, + input_tokens=input_tokens, output_tokens=response.usage.completion_tokens, cache_read_input_tokens=cached_tokens, ) diff --git a/tests/unit/models/openai/test_messages_api.py b/tests/unit/models/openai/test_messages_api.py index b3ebfeb2..fe8b74c0 100644 --- a/tests/unit/models/openai/test_messages_api.py +++ b/tests/unit/models/openai/test_messages_api.py @@ -41,8 +41,14 @@ def _make_completion( finish_reason: str = "stop", prompt_tokens: int = 10, completion_tokens: int = 20, + cached_tokens: int | None = None, ) -> ChatCompletion: """Create a mock ChatCompletion response.""" + prompt_tokens_details = None + if cached_tokens is not None: + from openai.types.completion_usage import PromptTokensDetails + + prompt_tokens_details = PromptTokensDetails(cached_tokens=cached_tokens) return ChatCompletion( id="chatcmpl-test", choices=[ @@ -63,6 +69,7 @@ def _make_completion( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=prompt_tokens_details, ), ) @@ -456,6 +463,31 @@ def test_usage_captured(self) -> None: assert result.usage.input_tokens == 50 assert result.usage.output_tokens == 100 + def test_cached_tokens_are_subtracted_from_input(self) -> None: + """OpenAI reports cached tokens as a SUBSET of prompt_tokens; + `UsageParam` consumers expect disjoint fields (Anthropic style). + Without the subtraction, cached tokens are counted and billed twice + (statistics callback, reporting).""" + completion = _make_completion( + content="ok", + prompt_tokens=1000, + completion_tokens=50, + cached_tokens=400, + ) + result = _from_openai_response(completion) + assert result.usage is not None + assert result.usage.input_tokens == 600 # 1000 - 400 cached + assert result.usage.cache_read_input_tokens == 400 + assert result.usage.output_tokens == 50 + + def test_cached_tokens_never_drive_input_negative(self) -> None: + completion = _make_completion( + content="ok", prompt_tokens=100, completion_tokens=5, cached_tokens=150 + ) + result = _from_openai_response(completion) + assert result.usage is not None + assert result.usage.input_tokens == 0 + class TestOpenAIMessagesApi: def test_create_message_delegates_to_client(self) -> None: From 3279410a47aa82e3886b6dbb64dd8955e1dc3817 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Fri, 7 Aug 2026 10:23:59 -0700 Subject: [PATCH 04/10] fix: reject zero-size screen captures before they reach the reporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the AskUI controller returns a 0×0 bitmap (e.g. because the display is minimised or unavailable), PIL cannot encode the resulting image to PNG. This caused the SimpleHtmlReporter to crash and be permanently disabled by ReporterErrorHandler, losing the entire HTML report for the run. Two-layer fix: - askui_controller.py: `_check_bitmap_dimensions()` raises `AskUiControllerError` immediately after a zero-size bitmap is received, before any PIL image is constructed or passed downstream. - reporting.py: `normalize_to_pil_images()` now filters out any zero-size PIL images as defence-in-depth, so no reporter can crash on an empty image regardless of its origin. Co-Authored-By: Claude Sonnet 4.6 --- src/askui/reporting.py | 16 +++++--- src/askui/tools/askui/askui_controller.py | 27 +++++++++++- tests/unit/test_reporting.py | 41 ++++++++++++++++++- .../askui/test_askui_controller_client.py | 19 +++++++++ 4 files changed, 96 insertions(+), 7 deletions(-) diff --git a/src/askui/reporting.py b/src/askui/reporting.py index 4090ea9b..f1c49648 100644 --- a/src/askui/reporting.py +++ b/src/askui/reporting.py @@ -34,14 +34,20 @@ def normalize_to_pil_images( image: Image.Image | list[Image.Image] | AnnotatedImage | None, ) -> list[Image.Image]: - """Normalize various image input types to a list of PIL images.""" + """Normalize various image input types to a list of PIL images. + + Zero-size images (width or height of 0) are filtered out because PIL + cannot encode them to PNG, which would crash any reporter that tries. + """ if image is None: return [] if isinstance(image, AnnotatedImage): - return image.get_images() - if isinstance(image, list): - return image - return [image] + images: list[Image.Image] = image.get_images() + elif isinstance(image, list): + images = image + else: + images = [image] + return [img for img in images if img.width > 0 and img.height > 0] def _format_duration(seconds: float) -> str: diff --git a/src/askui/tools/askui/askui_controller.py b/src/askui/tools/askui/askui_controller.py index fc892858..6bb879e2 100644 --- a/src/askui/tools/askui/askui_controller.py +++ b/src/askui/tools/askui/askui_controller.py @@ -378,6 +378,28 @@ def __exit__( """ self.disconnect() + @staticmethod + def _check_bitmap_dimensions(width: int, height: int) -> None: + """Raise `AskUiControllerError` when the captured bitmap has zero dimensions. + + A zero-size bitmap is returned when the display is unavailable, minimized, + or otherwise unable to produce a frame. PIL cannot encode such an image, + so we reject it here before it reaches the reporter or the model. + + Args: + width (int): Bitmap width returned by `CaptureScreen`. + height (int): Bitmap height returned by `CaptureScreen`. + + Raises: + AskUiControllerError: If either `width` or `height` is zero. + """ + if width == 0 or height == 0: + error_msg = ( + f"Screen capture returned an empty bitmap ({width}×{height}). " + "The display may be unavailable, minimized, or zero-sized." + ) + raise AskUiControllerError(error_msg) + @telemetry.record_call() @override def screenshot(self, report: bool = True, unscaled: bool = False) -> Image.Image: @@ -403,9 +425,12 @@ def screenshot(self, report: bool = True, unscaled: bool = False) -> Image.Image ), ) ) + width = screenResponse.bitmap.width + height = screenResponse.bitmap.height + self._check_bitmap_dimensions(width, height) r, g, b, _ = Image.frombytes( "RGBA", - (screenResponse.bitmap.width, screenResponse.bitmap.height), + (width, height), screenResponse.bitmap.data, ).split() image = Image.merge("RGB", (b, g, r)) diff --git a/tests/unit/test_reporting.py b/tests/unit/test_reporting.py index 1e87779c..da5d7837 100644 --- a/tests/unit/test_reporting.py +++ b/tests/unit/test_reporting.py @@ -7,7 +7,9 @@ from typing import Any -from askui.reporting import truncate_base64_media +from PIL import Image + +from askui.reporting import normalize_to_pil_images, truncate_base64_media def _base64_source(media_type: str) -> dict[str, Any]: @@ -58,3 +60,40 @@ def test_leaves_plain_content_untouched(self) -> None: "type": "text", "text": "hello", } + + +class TestNormalizeToPilImages: + def test_none_returns_empty_list(self) -> None: + assert normalize_to_pil_images(None) == [] + + def test_single_valid_image_is_wrapped_in_list(self) -> None: + img = Image.new("RGB", (10, 10)) + result = normalize_to_pil_images(img) + assert result == [img] + + def test_list_of_valid_images_is_returned_as_is(self) -> None: + images = [Image.new("RGB", (10, 10)), Image.new("RGB", (20, 20))] + result = normalize_to_pil_images(images) + assert result == images + + def test_zero_width_image_is_filtered_out(self) -> None: + empty = Image.new("RGB", (0, 5)) + valid = Image.new("RGB", (10, 10)) + result = normalize_to_pil_images([empty, valid]) + assert result == [valid] + + def test_zero_height_image_is_filtered_out(self) -> None: + empty = Image.new("RGB", (5, 0)) + valid = Image.new("RGB", (10, 10)) + result = normalize_to_pil_images([empty, valid]) + assert result == [valid] + + def test_zero_by_zero_image_is_filtered_out(self) -> None: + result = normalize_to_pil_images(Image.new("RGB", (0, 0))) + assert result == [] + + def test_list_of_only_empty_images_returns_empty_list(self) -> None: + result = normalize_to_pil_images( + [Image.new("RGB", (0, 0)), Image.new("RGB", (0, 5))] + ) + assert result == [] diff --git a/tests/unit/tools/askui/test_askui_controller_client.py b/tests/unit/tools/askui/test_askui_controller_client.py index 4c007f5a..70a5a27b 100644 --- a/tests/unit/tools/askui/test_askui_controller_client.py +++ b/tests/unit/tools/askui/test_askui_controller_client.py @@ -214,3 +214,22 @@ def test_underlying_manager_is_an_agent_os_target_computer_manager(self) -> None agent_os_target_computers=[_make_local(computer_id="l")] ) assert isinstance(client.agent_os_target_computer_manager, ComputerTargetPool) + + +class TestScreenshotValidation: + """_check_bitmap_dimensions() raises AskUiControllerError for zero-size bitmaps.""" + + def test_zero_width_raises(self) -> None: + with pytest.raises(AskUiControllerError, match="empty bitmap"): + MultiComputerTargetAgentOS._check_bitmap_dimensions(0, 100) + + def test_zero_height_raises(self) -> None: + with pytest.raises(AskUiControllerError, match="empty bitmap"): + MultiComputerTargetAgentOS._check_bitmap_dimensions(100, 0) + + def test_zero_by_zero_raises(self) -> None: + with pytest.raises(AskUiControllerError, match="empty bitmap"): + MultiComputerTargetAgentOS._check_bitmap_dimensions(0, 0) + + def test_valid_dimensions_do_not_raise(self) -> None: + MultiComputerTargetAgentOS._check_bitmap_dimensions(1920, 1080) From f5030b9651c926f429f1ff6c14aa7178c02a5623 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Fri, 7 Aug 2026 11:20:12 -0700 Subject: [PATCH 05/10] fix: warn when a zero-size image is dropped by normalize_to_pil_images Co-Authored-By: Claude Sonnet 4.6 --- src/askui/reporting.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/askui/reporting.py b/src/askui/reporting.py index f1c49648..99c4d43f 100644 --- a/src/askui/reporting.py +++ b/src/askui/reporting.py @@ -47,7 +47,17 @@ def normalize_to_pil_images( images = image else: images = [image] - return [img for img in images if img.width > 0 and img.height > 0] + valid = [] + for img in images: + if img.width == 0 or img.height == 0: + logger.warning( + "Skipping zero-size image (%dx%d) — cannot encode an empty image.", + img.width, + img.height, + ) + else: + valid.append(img) + return valid def _format_duration(seconds: float) -> str: From dc333cf46e6047ede0498c251064189e90f07328 Mon Sep 17 00:00:00 2001 From: Samir Mlika Date: Wed, 12 Aug 2026 16:06:28 +0200 Subject: [PATCH 06/10] bump version to 0.43.0 --- src/askui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/askui/__init__.py b/src/askui/__init__.py index 79a13d20..7d52380b 100644 --- a/src/askui/__init__.py +++ b/src/askui/__init__.py @@ -1,6 +1,6 @@ """AskUI Python SDK""" -__version__ = "0.42.0" +__version__ = "0.43.0" import logging import os From 163af4203b903e458570e0f6c43a7fe1822270c0 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Wed, 12 Aug 2026 13:40:28 -0700 Subject: [PATCH 07/10] fix: split Desktop Agent OS errors into recoverable vs unfixable Reading a remote file or directory that does not exist raised a DesktopAgentOsError that inherited from BaseException, so the tool-calling loop's `except Exception` handler never caught it. The error propagated all the way up and crashed the run instead of being surfaced to the agent as a tool error result, as recoverable tool failures are everywhere else. Introduce two error types that map onto the existing fatal/recoverable handling in the tool-calling loop: - DesktopAgentOsException (recoverable): a plain Exception for failures the agent can work around (path not found, undecodable file contents). The loop's generic `except Exception` catches it and returns it to the agent. - DesktopAgentOsError (unfixable): now inherits from AutomationError for protocol violations (unexpected response type, missing error and response). The loop's existing `except (AgentError, AutomationError): raise` re-raises it, terminating the run cleanly instead of crashing on an uncaught BaseException. Controller operation failures (res.error) and undecodable payloads now raise the recoverable type; contract violations keep the fatal type. No changes to the generic tool-calling loop are required. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/askui/tools/askui/askui_controller.py | 12 ++- .../desktop_agent_os_error.py | 31 +++++++- .../test_desktop_agent_os_error_handling.py | 77 +++++++++++++++++++ .../tools/askui/test_decode_file_payload.py | 4 +- 4 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 tests/unit/models/shared/test_desktop_agent_os_error_handling.py diff --git a/src/askui/tools/askui/askui_controller.py b/src/askui/tools/askui/askui_controller.py index 6bb879e2..e0e0f1ad 100644 --- a/src/askui/tools/askui/askui_controller.py +++ b/src/askui/tools/askui/askui_controller.py @@ -28,6 +28,7 @@ ) from askui.tools.askui.askui_ui_controller_grpc.desktop_agent_os_error import ( DesktopAgentOsError, + DesktopAgentOsException, ) from askui.tools.askui.askui_ui_controller_grpc.generated import ( Controller_V1_pb2 as controller_v1_pbs, @@ -1442,7 +1443,7 @@ def get_file_names(self, absolute_directory_path: str) -> list[str]: message = f"unexpected response type: {res}" raise DesktopAgentOsError(message) if res.error is not None: - raise DesktopAgentOsError(res.error) + raise DesktopAgentOsException(res.error) if res.response is None: message = f"{type(res).__name__} is missing both error and response" raise DesktopAgentOsError(message) @@ -1472,7 +1473,10 @@ def get_file(self, path: str) -> Image.Image | PdfSource | str: Image.Image | PdfSource | str: The decoded file contents. Raises: - DesktopAgentOsError: If the file cannot be read or the response is invalid. + DesktopAgentOsException: If the file cannot be read (e.g. the path + does not exist or its contents cannot be decoded). + DesktopAgentOsError: If the controller response violates the expected + protocol. """ self._reporter.add_message(self._REPORTER_SOURCE, f"get_file({path})") command = GetFileCommand(parameters=[path]) @@ -1481,7 +1485,7 @@ def get_file(self, path: str) -> Image.Image | PdfSource | str: message = f"unexpected response type: {res}" raise DesktopAgentOsError(message) if res.error is not None: - raise DesktopAgentOsError(res.error) + raise DesktopAgentOsException(res.error) if res.response is None: message = f"{type(res).__name__} is missing both error and response" raise DesktopAgentOsError(message) @@ -1535,7 +1539,7 @@ def _decode_file_payload(base64_data: str) -> Image.Image | PdfSource | str: except UnicodeDecodeError: pass message = "File contents are neither a supported image, PDF, nor UTF-8 text" - raise DesktopAgentOsError(message) + raise DesktopAgentOsException(message) AskUiControllerClient = MultiComputerTargetAgentOS diff --git a/src/askui/tools/askui/askui_ui_controller_grpc/desktop_agent_os_error.py b/src/askui/tools/askui/askui_ui_controller_grpc/desktop_agent_os_error.py index 14a66aba..2ac37161 100644 --- a/src/askui/tools/askui/askui_ui_controller_grpc/desktop_agent_os_error.py +++ b/src/askui/tools/askui/askui_ui_controller_grpc/desktop_agent_os_error.py @@ -1,5 +1,30 @@ -class DesktopAgentOsError(BaseException): - """Base class for Desktop Agent OS errors. +from askui.models.exceptions import AutomationError - This error is raised when an error occurs in the Desktop Agent OS. + +class DesktopAgentOsError(AutomationError): + """Unfixable error raised by the Desktop Agent OS. + + Raised when the Desktop Agent OS returns a response that violates the + expected protocol (e.g. an unexpected response type or a response missing + both an error and a payload). These indicate a broken controller or + connection rather than something the agent can recover from, so - like + other `AutomationError`s - they are re-raised by the tool-calling loop and + terminate the run. + + For failures the agent can react to and work around (e.g. a path that does + not exist), raise `DesktopAgentOsException` instead. + """ + + +class DesktopAgentOsException(Exception): # noqa: N818 + """Recoverable error raised by the Desktop Agent OS. + + Raised when an operation on the Desktop Agent OS fails in a way the agent + can react to and work around - for example, reading a file or directory + that does not exist, or a file whose contents cannot be decoded. Because it + derives from `Exception` (and not `AutomationError`), the tool-calling loop + catches it and surfaces it to the agent as a tool error result instead of + terminating the run. + + For unfixable protocol violations, raise `DesktopAgentOsError` instead. """ diff --git a/tests/unit/models/shared/test_desktop_agent_os_error_handling.py b/tests/unit/models/shared/test_desktop_agent_os_error_handling.py new file mode 100644 index 00000000..6454cecb --- /dev/null +++ b/tests/unit/models/shared/test_desktop_agent_os_error_handling.py @@ -0,0 +1,77 @@ +"""Tests that Desktop Agent OS errors are routed by recoverability. + +The Desktop Agent OS raises two error types: + +- `DesktopAgentOsException` for failures the agent can react to (e.g. reading a + path that does not exist). The tool-calling loop catches it and surfaces it to + the agent as a tool error result so the run can continue. +- `DesktopAgentOsError` for unfixable protocol violations. It derives from + `AutomationError` and is re-raised by the tool-calling loop, terminating the + run instead of being fed back to the agent. +""" + +import pytest + +from askui.models.exceptions import AutomationError +from askui.models.shared.agent_message_param import ( + ToolResultBlockParam, + ToolUseBlockParam, +) +from askui.models.shared.tools import Tool, ToolCollection +from askui.tools.askui.askui_ui_controller_grpc.desktop_agent_os_error import ( + DesktopAgentOsError, + DesktopAgentOsException, +) + +_RECOVERABLE_MESSAGE = ( + "directory_iterator::directory_iterator: The system cannot find the " + 'path specified.: "FrontEnd\\Traces"' +) +_FATAL_MESSAGE = "unexpected response type: " + + +class _RaisingTool(Tool): + """A tool whose `__call__` raises the exception it was constructed with.""" + + _error: BaseException + + def __init__(self, error: BaseException) -> None: + super().__init__( + name="raising_tool", + description="Raises a preconfigured Desktop Agent OS error.", + ) + self._error = error + + def __call__(self) -> str: + raise self._error + + +def _run(tool: Tool) -> list: + collection = ToolCollection(tools=[tool]) + tool_use = ToolUseBlockParam(id="tool_use_1", input={}, name=tool.name) + return collection.run([tool_use]) + + +class TestDesktopAgentOsErrorHierarchy: + def test_error_is_an_automation_error(self) -> None: + assert issubclass(DesktopAgentOsError, AutomationError) + + def test_exception_is_a_plain_exception_not_automation_error(self) -> None: + assert issubclass(DesktopAgentOsException, Exception) + assert not issubclass(DesktopAgentOsException, AutomationError) + + +class TestDesktopAgentOsErrorHandling: + def test_recoverable_exception_returns_error_result(self) -> None: + results = _run(_RaisingTool(DesktopAgentOsException(_RECOVERABLE_MESSAGE))) + + assert len(results) == 1 + result = results[0] + assert isinstance(result, ToolResultBlockParam) + assert result.is_error is True + assert result.tool_use_id == "tool_use_1" + assert "FrontEnd\\Traces" in str(result.content) + + def test_fatal_error_propagates_and_terminates(self) -> None: + with pytest.raises(DesktopAgentOsError): + _run(_RaisingTool(DesktopAgentOsError(_FATAL_MESSAGE))) diff --git a/tests/unit/tools/askui/test_decode_file_payload.py b/tests/unit/tools/askui/test_decode_file_payload.py index 6d898e83..53206245 100644 --- a/tests/unit/tools/askui/test_decode_file_payload.py +++ b/tests/unit/tools/askui/test_decode_file_payload.py @@ -14,7 +14,7 @@ from askui.tools.askui.askui_controller import ( AskUiControllerClient, - DesktopAgentOsError, + DesktopAgentOsException, ) from askui.utils.pdf_utils import PdfSource @@ -47,5 +47,5 @@ def test_decodes_utf8_text(self) -> None: assert result == "hello world" def test_rejects_unsupported_binary(self) -> None: - with pytest.raises(DesktopAgentOsError): + with pytest.raises(DesktopAgentOsException): AskUiControllerClient._decode_file_payload(_b64(b"\x00\x01\x02\x03")) From d4d63c4a867719cba2d771cef7a25f4fb2a2f87e Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Wed, 12 Aug 2026 14:08:58 -0700 Subject: [PATCH 08/10] fix(models): request visible thinking summaries on adaptive-thinking models Models of the Sonnet 5 generation onward default the thinking `display` setting to "omitted": the API returns thinking blocks whose text is empty while the full thinking tokens are still billed. Any consumer relying on make_thinking_settings() therefore silently loses all visible reasoning in reports and logs on the newest models. Set display: "summarized" explicitly in the adaptive branch. Billing is identical for both display modes, so this only restores visibility. Verified end-to-end against Vertex claude-sonnet-5 (empty before, 3k chars of summarized thinking after). Co-Authored-By: Claude Fable 5 --- src/askui/models/shared/thinking.py | 19 ++++++++++++++----- tests/unit/models/test_thinking.py | 8 +++++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/askui/models/shared/thinking.py b/src/askui/models/shared/thinking.py index b668a12b..5fe1f94b 100644 --- a/src/askui/models/shared/thinking.py +++ b/src/askui/models/shared/thinking.py @@ -159,10 +159,17 @@ def make_thinking_settings( **make_thinking_settings(self._vlm_provider.model_id), ) - Models that support adaptive thinking get ``thinking={"type": "adaptive"}`` - (with ``effort`` sent via ``provider_options["output_config"]`` when given); - older models get a fixed token budget of - ``thinking={"type": "enabled", "budget_tokens": 2048}`` and ignore ``effort``. + Models that support adaptive thinking get + ``thinking={"type": "adaptive", "display": "summarized"}`` (with ``effort`` + sent via ``provider_options["output_config"]`` when given); older models + get a fixed token budget of + ``thinking={"type": "enabled", "budget_tokens": 2048}`` and ignore + ``effort``. ``display`` is set explicitly because the newest adaptive + models (Sonnet 5 generation onward) default it to ``"omitted"``, which + returns thinking blocks whose text is EMPTY while the full thinking + tokens are still billed — reasoning silently disappears from reports and + logs. ``"summarized"`` restores the visible text at no extra cost (billing + is identical for both display modes). Args: model_id (str): The model identifier (bare or gateway-prefixed). @@ -176,7 +183,9 @@ def make_thinking_settings( when applicable, ``provider_options``). """ if uses_adaptive_thinking(model_id): - settings: dict[str, Any] = {"thinking": {"type": "adaptive"}} + settings: dict[str, Any] = { + "thinking": {"type": "adaptive", "display": "summarized"} + } if effort is not None: settings["provider_options"] = {"output_config": {"effort": effort}} return settings diff --git a/tests/unit/models/test_thinking.py b/tests/unit/models/test_thinking.py index 58682494..7249d82d 100644 --- a/tests/unit/models/test_thinking.py +++ b/tests/unit/models/test_thinking.py @@ -50,7 +50,9 @@ @pytest.mark.parametrize("model_id", _ADAPTIVE_MODELS) def test_adaptive_models_use_adaptive_thinking(model_id: str) -> None: assert uses_adaptive_thinking(model_id) is True - assert make_thinking_settings(model_id) == {"thinking": {"type": "adaptive"}} + assert make_thinking_settings(model_id) == { + "thinking": {"type": "adaptive", "display": "summarized"} + } @pytest.mark.parametrize("model_id", _BUDGET_MODELS) @@ -63,7 +65,7 @@ def test_other_models_use_budget_tokens(model_id: str) -> None: def test_effort_is_added_via_provider_options_for_adaptive_models() -> None: assert make_thinking_settings("claude-sonnet-5", effort="high") == { - "thinking": {"type": "adaptive"}, + "thinking": {"type": "adaptive", "display": "summarized"}, "provider_options": {"output_config": {"effort": "high"}}, } @@ -139,6 +141,6 @@ def test_non_thinking_settings_omit_thinking_on_always_on_models() -> None: def test_effort_supports_xhigh() -> None: assert make_thinking_settings("claude-opus-4-8", effort="xhigh") == { - "thinking": {"type": "adaptive"}, + "thinking": {"type": "adaptive", "display": "summarized"}, "provider_options": {"output_config": {"effort": "xhigh"}}, } From 47ba7950f65b5bcd2f02233ba5586e79ea7aa366 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Wed, 12 Aug 2026 14:42:47 -0700 Subject: [PATCH 09/10] docs: clarify get_file_names_tool returns files only, not folders Make the tool description explicit that the tool lists only regular files and never includes folders/subdirectories, so the agent does not try to use it to discover or navigate directories. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tools/store/computer/experimental/get_file_names.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/askui/tools/store/computer/experimental/get_file_names.py b/src/askui/tools/store/computer/experimental/get_file_names.py index 643820fb..dd313f58 100644 --- a/src/askui/tools/store/computer/experimental/get_file_names.py +++ b/src/askui/tools/store/computer/experimental/get_file_names.py @@ -28,9 +28,11 @@ def __init__(self, agent_os: ComputerAgentOS | None = None) -> None: name="get_file_names_tool", description=( "Lists the names of regular files in an absolute directory on the " - "computer under automation. Subdirectories are not included—only " - "files are returned. Use absolute paths as on the target machine. " - "Returns names only; use get_file_tool to read a file's contents." + "computer under automation. IMPORTANT: only files are returned - " + "folders and subdirectories are NOT included in the result and this " + "tool cannot be used to discover or navigate them. Use absolute " + "paths as on the target machine. Returns names only; use " + "get_file_tool to read a file's contents." ), input_schema={ "type": "object", From f0b99ba400cb551bfcbce032296f63e7f5269c78 Mon Sep 17 00:00:00 2001 From: philipph-askui Date: Thu, 13 Aug 2026 10:22:43 -0700 Subject: [PATCH 10/10] chore: bump version to v0.44.0 --- src/askui/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/askui/__init__.py b/src/askui/__init__.py index 7d52380b..a44495a8 100644 --- a/src/askui/__init__.py +++ b/src/askui/__init__.py @@ -1,6 +1,6 @@ """AskUI Python SDK""" -__version__ = "0.43.0" +__version__ = "0.44.0" import logging import os