"""Diffbot LLM RAG API: stream a chat completion."""
import json
import re
from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, List, Optional
from .errors import ValidationError
if TYPE_CHECKING:
from .client import Diffbot, DiffbotAsync
MODEL = "diffbot-small-xl"
#: `response_format` types accepted by the Diffbot LLM endpoint.
RESPONSE_FORMAT_TYPES = ("text", "json_object", "json_schema")
# The RAG loop may prefix its final answer with a think block, and the model
# occasionally wraps JSON in a markdown fence despite being told not to.
_THINK_BLOCK = re.compile(r".*?", re.DOTALL)
_JSON_FENCE = re.compile(r"^```(?:json)?\s*|\s*```$", re.MULTILINE)
def json_schema_format(schema: Dict[str, Any], *, name: str = "response") -> Dict[str, Any]:
"""Build a ``response_format`` value that constrains output to ``schema``.
The endpoint requires the schema nested under ``json_schema.schema``; passing
it anywhere else is ignored server-side without an error, so prefer this
helper over hand-building the dict.
Example:
>>> json_schema_format({"type": "object", "properties": {"city": {"type": "string"}}})
{'type': 'json_schema', 'json_schema': {'name': 'response', 'schema': {...}}}
"""
if not isinstance(schema, dict):
raise ValidationError("schema must be a JSON Schema dict")
return {"type": "json_schema", "json_schema": {"name": name, "schema": schema}}
def _validate_response_format(response_format: Optional[Dict[str, Any]]) -> None:
"""Reject shapes the endpoint would accept but silently not enforce."""
if response_format is None:
return
if not isinstance(response_format, dict):
raise ValidationError("response_format must be a dict")
fmt_type = response_format.get("type", "text")
if fmt_type not in RESPONSE_FORMAT_TYPES:
raise ValidationError(
f"response_format type must be one of {', '.join(RESPONSE_FORMAT_TYPES)}; got {fmt_type!r}"
)
if fmt_type == "json_schema":
json_schema = response_format.get("json_schema")
if not isinstance(json_schema, dict) or "schema" not in json_schema:
raise ValidationError(
'response_format {"type": "json_schema"} requires the schema nested as '
'{"json_schema": {"schema": {...}}}. Without it the server returns 200 and '
"ignores the constraint. Use diffbot.json_schema_format(schema) to build it."
)
def _build_payload(
client: Any,
messages: List[Dict[str, str]],
*,
response_format: Optional[Dict[str, Any]] = None,
) -> tuple:
_validate_response_format(response_format)
headers = {"Authorization": f"Bearer {client.token}"}
payload: Dict[str, Any] = {"model": MODEL, "messages": messages, "stream": True}
if response_format is not None:
payload["response_format"] = response_format
return headers, payload
def _parse_chunk(line: str):
try:
chunk = json.loads(line.replace("data: ", ""))
except json.JSONDecodeError:
return None
choices = chunk.get("choices")
if choices and choices[0].get("delta", {}).get("content"):
return choices[0]["delta"]["content"]
return None
def _extract_json(text: str) -> Any:
"""Parse the model's final answer as JSON, tolerating think blocks and fences."""
cleaned = _THINK_BLOCK.sub("", text).strip()
cleaned = _JSON_FENCE.sub("", cleaned).strip()
try:
return json.loads(cleaned)
except json.JSONDecodeError:
pass
# Fall back to the outermost object or array span in the response.
spans = []
for opener, closer in (("{", "}"), ("[", "]")):
start, end = cleaned.find(opener), cleaned.rfind(closer)
if start != -1 and end > start:
spans.append((start, cleaned[start : end + 1]))
for _, span in sorted(spans):
try:
return json.loads(span)
except json.JSONDecodeError:
continue
raise ValidationError(f"could not parse JSON from the model response: {text[:200]!r}")
#: Schema used when the caller wants JSON but has no shape in mind. This goes
#: through the json_schema path rather than {"type": "json_object"} on purpose:
#: json_object applies no server-side grammar, so the RAG loop's internal
#: JSON satisfies it and gets returned as the final answer. That
#: is reproducible whenever the request carries a system message.
ANY_OBJECT_SCHEMA = {"type": "object"}
def _resolve_format(
schema: Optional[Dict[str, Any]],
response_format: Optional[Dict[str, Any]],
) -> Dict[str, Any]:
if schema is not None and response_format is not None:
raise ValidationError("pass either schema or response_format, not both")
if response_format is not None:
return response_format
return json_schema_format(schema if schema is not None else ANY_OBJECT_SCHEMA)
def _check_tool_call_leak(parsed: Any) -> Any:
"""Catch the internal tool call surfacing as the answer (see ANY_OBJECT_SCHEMA)."""
if isinstance(parsed, dict) and parsed.get("name") == "functioncall" and "arguments" in parsed:
raise ValidationError(
"the model returned its internal tool call instead of an answer; this happens with "
'response_format {"type": "json_object"} because the server applies no grammar to it. '
"Pass a schema instead."
)
return parsed
def ask(
client: "Diffbot",
messages: List[Dict[str, str]],
*,
response_format: Optional[Dict[str, Any]] = None,
) -> Iterator[str]:
headers, payload = _build_payload(client, messages, response_format=response_format)
with client._http.stream("POST", client.llm_url, headers=headers, json=payload) as response:
client._raise_for_status(response)
for line in response.iter_lines():
if line:
content = _parse_chunk(line)
if content:
yield content
async def ask_async(
client: "DiffbotAsync",
messages: List[Dict[str, str]],
*,
response_format: Optional[Dict[str, Any]] = None,
) -> AsyncIterator[str]:
headers, payload = _build_payload(client, messages, response_format=response_format)
async with client._http.stream("POST", client.llm_url, headers=headers, json=payload) as response:
client._raise_for_status(response)
async for line in response.aiter_lines():
if line:
content = _parse_chunk(line)
if content:
yield content
def ask_json(
client: "Diffbot",
messages: List[Dict[str, str]],
schema: Optional[Dict[str, Any]] = None,
*,
response_format: Optional[Dict[str, Any]] = None,
) -> Any:
fmt = _resolve_format(schema, response_format)
text = "".join(ask(client, messages, response_format=fmt))
return _check_tool_call_leak(_extract_json(text))
async def ask_json_async(
client: "DiffbotAsync",
messages: List[Dict[str, str]],
schema: Optional[Dict[str, Any]] = None,
*,
response_format: Optional[Dict[str, Any]] = None,
) -> Any:
fmt = _resolve_format(schema, response_format)
chunks = [chunk async for chunk in ask_async(client, messages, response_format=fmt)]
return _check_tool_call_leak(_extract_json("".join(chunks)))