From 0e19b5e9d52ace58c4a82e7954b4185e72f86a69 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 00:35:22 +0800 Subject: [PATCH 001/304] Update Submodule vendor/llama.cpp 58190cc..69c28f1 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 58190cc84d..69c28f1547 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 58190cc84d846d8575ba26e8486bc29d9fd8ad55 +Subproject commit 69c28f1547c169902f62ca48bee75fb876c4d8e6 From c3d6fdecb80e1a8389fa30acaa809496253c2ec9 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 00:37:06 +0800 Subject: [PATCH 002/304] Workflow (metal): Try using gh to replace the unmaintained softprops/action-gh-release. --- .github/workflows/build-wheels-metal.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index abb8969247..b583d37db3 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -85,10 +85,9 @@ jobs: # Store the date in environment variable for the release step echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV - - name: Publish Release - uses: softprops/action-gh-release@v2.2.2 - with: - files: dist2/* - tag_name: v${{ needs.build_wheels.outputs.version }}-Metal-macos-${{ env.BUILD_DATE }} + - name: Publish Release via GitHub CLI env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: + TAG_NAME="v${{ needs.build_wheels.outputs.version }}-Metal-macos-$BUILD_DATE" + gh release create "$TAG_NAME" dist2/* --title "$TAG_NAME" --generate-notes From ac38f388cd8c52a56ad6f498193a5c0c9a0f5369 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 01:44:35 +0800 Subject: [PATCH 003/304] ci: restrict cudaarch to Volta-Hopper to fix GitHub Actions timeout Using the `all` option for `cudaarch` on CUDA 12.4-12.6 causes the compilation process to exceed the 6-hour maximum execution limit on GitHub Actions, leading to cancelled jobs. To resolve this and reduce build times, the target architectures are now restricted to explicitly support compute capabilities 7.0 through 9.0 (`70-real` to `90-real`). This maintains support for all modern NVIDIA GPUs equipped with Tensor Cores (from Volta up to Hopper architectures) while keeping the build time safely within CI constraints. Signed-off-by: JamePeng --- .github/workflows/build-wheels-cu124-win.yml | 2 +- .github/workflows/build-wheels-cu126-win.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-cu124-win.yml b/.github/workflows/build-wheels-cu124-win.yml index c6800e246a..f020cd6708 100644 --- a/.github/workflows/build-wheels-cu124-win.yml +++ b/.github/workflows/build-wheels-cu124-win.yml @@ -16,7 +16,7 @@ jobs: pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] cuda: ["12.4.1"] releasetag: ["Basic"] - cudaarch: ["all"] + cudaarch: ["70-real;75-real;80-real;86-real;87-real;89-real;90-real"] defaults: run: shell: pwsh diff --git a/.github/workflows/build-wheels-cu126-win.yml b/.github/workflows/build-wheels-cu126-win.yml index eec32f6f0d..08115a2ef5 100644 --- a/.github/workflows/build-wheels-cu126-win.yml +++ b/.github/workflows/build-wheels-cu126-win.yml @@ -16,7 +16,7 @@ jobs: pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] cuda: ["12.6.3"] releasetag: ["Basic"] - cudaarch: ["all"] + cudaarch: ["70-real;75-real;80-real;86-real;87-real;89-real;90-real"] defaults: run: shell: pwsh From 4f2a132e35e1a5eb6491e1046fb36efd9660cff5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 02:05:31 +0800 Subject: [PATCH 004/304] Update CI Action runner version microsoft/setup-msbuild@v2 -> v3 actions/checkout@v5 -> v6 actions/upload-artifact@v4 -> v6 actions/download-artifact@v4 -> v6 Signed-off-by: JamePeng --- .github/workflows/build-wheels-cu124-linux.yml | 2 +- .github/workflows/build-wheels-cu124-win.yml | 4 ++-- .github/workflows/build-wheels-cu126-linux.yml | 2 +- .github/workflows/build-wheels-cu126-win.yml | 4 ++-- .github/workflows/build-wheels-cu128-linux.yml | 2 +- .github/workflows/build-wheels-cu128-win.yml | 4 ++-- .github/workflows/build-wheels-cu130-linux.yml | 2 +- .github/workflows/build-wheels-cu130-win.yml | 4 ++-- .github/workflows/build-wheels-metal.yaml | 4 ++-- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-wheels-cu124-linux.yml b/.github/workflows/build-wheels-cu124-linux.yml index 9a55248124..f14684289d 100644 --- a/.github/workflows/build-wheels-cu124-linux.yml +++ b/.github/workflows/build-wheels-cu124-linux.yml @@ -34,7 +34,7 @@ jobs: apt update apt install -y build-essential ccache cmake curl git libgomp1 libjpeg-dev libssl-dev - - uses: actions/checkout@v4 # Checkout code + - uses: actions/checkout@v6 # Checkout code with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-cu124-win.yml b/.github/workflows/build-wheels-cu124-win.yml index f020cd6708..0989afc8dd 100644 --- a/.github/workflows/build-wheels-cu124-win.yml +++ b/.github/workflows/build-wheels-cu124-win.yml @@ -32,11 +32,11 @@ jobs: steps: - name: Add MSBuild to PATH if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v2 + uses: microsoft/setup-msbuild@v3 with: msbuild-architecture: x64 - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-cu126-linux.yml b/.github/workflows/build-wheels-cu126-linux.yml index bca09d2f66..1eda5d10f2 100644 --- a/.github/workflows/build-wheels-cu126-linux.yml +++ b/.github/workflows/build-wheels-cu126-linux.yml @@ -34,7 +34,7 @@ jobs: apt update apt install -y build-essential ccache cmake curl git libgomp1 libjpeg-dev libssl-dev - - uses: actions/checkout@v4 # Checkout code + - uses: actions/checkout@v6 # Checkout code with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-cu126-win.yml b/.github/workflows/build-wheels-cu126-win.yml index 08115a2ef5..19474b530d 100644 --- a/.github/workflows/build-wheels-cu126-win.yml +++ b/.github/workflows/build-wheels-cu126-win.yml @@ -32,11 +32,11 @@ jobs: steps: - name: Add MSBuild to PATH if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v2 + uses: microsoft/setup-msbuild@v3 with: msbuild-architecture: x64 - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-cu128-linux.yml b/.github/workflows/build-wheels-cu128-linux.yml index ad13b30706..a4ab9e8eb2 100644 --- a/.github/workflows/build-wheels-cu128-linux.yml +++ b/.github/workflows/build-wheels-cu128-linux.yml @@ -34,7 +34,7 @@ jobs: apt update apt install -y build-essential ccache cmake curl git libgomp1 libjpeg-dev libssl-dev - - uses: actions/checkout@v4 # Checkout code + - uses: actions/checkout@v6 # Checkout code with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-cu128-win.yml b/.github/workflows/build-wheels-cu128-win.yml index e9d36602bd..0d87e09a45 100644 --- a/.github/workflows/build-wheels-cu128-win.yml +++ b/.github/workflows/build-wheels-cu128-win.yml @@ -32,11 +32,11 @@ jobs: steps: - name: Add MSBuild to PATH if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v2 + uses: microsoft/setup-msbuild@v3 with: msbuild-architecture: x64 - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-cu130-linux.yml b/.github/workflows/build-wheels-cu130-linux.yml index 574690cdf2..dbc710e18a 100644 --- a/.github/workflows/build-wheels-cu130-linux.yml +++ b/.github/workflows/build-wheels-cu130-linux.yml @@ -34,7 +34,7 @@ jobs: apt update apt install -y build-essential ccache cmake curl git libgomp1 libjpeg-dev libssl-dev - - uses: actions/checkout@v5 # Checkout code + - uses: actions/checkout@v6 # Checkout code with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-cu130-win.yml b/.github/workflows/build-wheels-cu130-win.yml index d055db43af..5b8b9c1992 100644 --- a/.github/workflows/build-wheels-cu130-win.yml +++ b/.github/workflows/build-wheels-cu130-win.yml @@ -32,11 +32,11 @@ jobs: steps: - name: Add MSBuild to PATH if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v2 + uses: microsoft/setup-msbuild@v3 with: msbuild-architecture: x64 - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: "recursive" diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index b583d37db3..094b735594 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -60,7 +60,7 @@ jobs: output-dir: wheelhouse2 - name: Upload artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: name: wheels-metal_${{ matrix.os }} path: ./wheelhouse2/*.whl @@ -72,7 +72,7 @@ jobs: steps: - name: Download artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: merge-multiple: true path: dist2 From e9c3013d6deb8ffcc08e43284371e9fb363ec32c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 02:10:15 +0800 Subject: [PATCH 005/304] fix multi-line script in metal workflow --- .github/workflows/build-wheels-metal.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 094b735594..09ae8470a6 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -88,6 +88,9 @@ jobs: - name: Publish Release via GitHub CLI env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: + run: | TAG_NAME="v${{ needs.build_wheels.outputs.version }}-Metal-macos-$BUILD_DATE" + + echo "Ready to create release with tag: $TAG_NAME" + gh release create "$TAG_NAME" dist2/* --title "$TAG_NAME" --generate-notes From 58c5fe1f5a90654d1cf5e4f31f6be398801471ec Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 02:37:47 +0800 Subject: [PATCH 006/304] Update Submodule vendor/llama.cpp 69c28f1..4eb1951 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 69c28f1547..4eb19514dd 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 69c28f1547c169902f62ca48bee75fb876c4d8e6 +Subproject commit 4eb19514dd2984662f13aacbb052c559c8fde3b1 From fbbf2ddb3d05b06bbf8de2ef774d4f8be1f52fd9 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 02:38:40 +0800 Subject: [PATCH 007/304] Adding the `actions/checkout` step to the release job to provide the `.git` context required by the CLI's `--generate-notes` flag. --- .github/workflows/build-wheels-metal.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 09ae8470a6..40675b4c26 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -71,6 +71,9 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Download artifacts uses: actions/download-artifact@v6 with: From 645c8ed91c53c00400cfc90b4d2604e94f718a84 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 04:06:28 +0800 Subject: [PATCH 008/304] feat(types): align with latest OpenAI API spec and fix type issues - Expand `CompletionUsage` with `PromptTokensDetails` and `CompletionTokensDetails` for granular token tracking. - Add `usage` to `CreateChatCompletionStreamResponse` to support usage reporting in streaming mode. - Fix duplicate `object` field in `CreateCompletionResponse`. - Update `ChatCompletionRequestAssistantMessage` to accept `None` for `content` and introduce the new `refusal` field. - Clean up `ChatCompletionRequestMessage` Union by removing the duplicate user message type. - Broaden `ChatCompletionToolChoiceOption` to fully support `allowed_tools` and `custom` tool choice behaviors. Signed-off-by: JamePeng --- llama_cpp/llama_types.py | 44 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 336cddab0f..60202ae8f0 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -49,10 +49,24 @@ class CompletionChoice(TypedDict): finish_reason: Optional[Literal["stop", "length", "content_filter"]] +class PromptTokensDetails(TypedDict): + cached_tokens: NotRequired[int] + audio_tokens: NotRequired[int] + + +class CompletionTokensDetails(TypedDict): + reasoning_tokens: NotRequired[int] + audio_tokens: NotRequired[int] + accepted_prediction_tokens: NotRequired[int] + rejected_prediction_tokens: NotRequired[int] + + class CompletionUsage(TypedDict): prompt_tokens: int completion_tokens: int total_tokens: int + prompt_tokens_details: NotRequired[PromptTokensDetails] + completion_tokens_details: NotRequired[CompletionTokensDetails] class CreateCompletionResponse(TypedDict): @@ -61,7 +75,6 @@ class CreateCompletionResponse(TypedDict): created: int model: str choices: List[CompletionChoice] - object: Optional[Literal["text_completion"]] usage: NotRequired[CompletionUsage] @@ -198,6 +211,7 @@ class CreateChatCompletionStreamResponse(TypedDict): object: Literal["chat.completion.chunk"] created: int choices: List[ChatCompletionStreamResponseChoice] + usage: NotRequired[CompletionUsage] class ChatCompletionFunctions(TypedDict): @@ -307,7 +321,8 @@ class ChatCompletionRequestAssistantMessageFunctionCall(TypedDict): class ChatCompletionRequestAssistantMessage(TypedDict): role: Literal["assistant"] - content: NotRequired[str] + content: NotRequired[Optional[str]] + refusal: NotRequired[Optional[str]] tool_calls: NotRequired[ChatCompletionMessageToolCalls] function_call: NotRequired[ ChatCompletionRequestAssistantMessageFunctionCall @@ -331,7 +346,6 @@ class ChatCompletionRequestFunctionMessage(TypedDict): ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage, ChatCompletionRequestAssistantMessage, - ChatCompletionRequestUserMessage, ChatCompletionRequestToolMessage, ChatCompletionRequestFunctionMessage, ] @@ -359,6 +373,16 @@ class ChatCompletionTool(TypedDict): function: ChatCompletionToolFunction +class ChatCompletionAllowedTools(TypedDict): + mode: Literal["auto", "required"] + tools: List[Dict[str, Any]] + + +class ChatCompletionAllowedToolsChoice(TypedDict): + type: Literal["allowed_tools"] + allowed_tools: ChatCompletionAllowedTools + + class ChatCompletionNamedToolChoiceFunction(TypedDict): name: str @@ -368,8 +392,20 @@ class ChatCompletionNamedToolChoice(TypedDict): function: ChatCompletionNamedToolChoiceFunction +class ChatCompletionNamedToolChoiceCustomObject(TypedDict): + name: str + + +class ChatCompletionNamedToolChoiceCustom(TypedDict): + type: Literal["custom"] + custom: ChatCompletionNamedToolChoiceCustomObject + + ChatCompletionToolChoiceOption = Union[ - Literal["none", "auto", "required"], ChatCompletionNamedToolChoice + Literal["none", "auto", "required"], + ChatCompletionAllowedToolsChoice, + ChatCompletionNamedToolChoice, + ChatCompletionNamedToolChoiceCustom ] From 6ebab28a06c79cd02d3b766c61fa75b73c85203f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Apr 2026 19:32:02 +0800 Subject: [PATCH 009/304] Update Submodule vendor/llama.cpp 4eb1951..d12cc3d Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4eb19514dd..d12cc3d1ca 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4eb19514dd2984662f13aacbb052c559c8fde3b1 +Subproject commit d12cc3d1ca6bba741cd77887ac9c9ee18c8415c7 From 9241b0fafad5ba1ed3730893ff9e60e9b2b6c91f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 9 Apr 2026 03:09:49 +0800 Subject: [PATCH 010/304] Sync ggml: add Q1_0 1-bit quantization support (CPU) (#21273) --- llama_cpp/_ggml.py | 8 ++++++-- llama_cpp/llama_cpp.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index f22c9eb94d..733a3520c1 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -121,7 +121,8 @@ class GGMLStatus(enum.IntEnum): # // GGML_TYPE_IQ4_NL_8_8 = 38, # GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) # GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) -# GGML_TYPE_COUNT = 41, +# GGML_TYPE_Q1_0 = 41, +# GGML_TYPE_COUNT = 42, # }; class GGMLType(enum.IntEnum): GGML_TYPE_F32 = 0 @@ -157,7 +158,8 @@ class GGMLType(enum.IntEnum): GGML_TYPE_TQ2_0 = 35 GGML_TYPE_MXFP4 = 39 GGML_TYPE_NVFP4 = 40 - GGML_TYPE_COUNT = 41 + GGML_TYPE_Q1_0 = 41 + GGML_TYPE_COUNT = 42 # // precision @@ -198,6 +200,7 @@ class GGMLPrec(enum.IntEnum): # GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors # GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors # GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors +# GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors # }; class GGMLFType(enum.IntEnum): GGML_FTYPE_UNKNOWN = -1 @@ -226,6 +229,7 @@ class GGMLFType(enum.IntEnum): GGML_FTYPE_MOSTLY_BF16 = 24 GGML_FTYPE_MOSTLY_MXFP4 = 25 GGML_FTYPE_MOSTLY_NVFP4 = 26 + GGML_FTYPE_MOSTLY_Q1_0 = 27 # // available tensor operations: diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 5d7fcd5fd2..1e1e40b70f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -351,6 +351,7 @@ # LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -391,6 +392,7 @@ LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_MOSTLY_NVFP4 = 39 +LLAMA_FTYPE_MOSTLY_Q1_0 = 40 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { From abe789f0223b9d930292c7a54bb87170e088b9eb Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 9 Apr 2026 04:34:05 +0800 Subject: [PATCH 011/304] Implement `Step3VLChatHandler` for Step3-VL-10B Signed-off-by: JamePeng --- README.md | 1 + llama_cpp/llama_chat_format.py | 137 +++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) diff --git a/README.md b/README.md index e316972b78..77eb0e661c 100644 --- a/README.md +++ b/README.md @@ -744,6 +744,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | | [qwen3-vl](https://huggingface.co/unsloth/Qwen3-VL-8B-Thinking-GGUF) | `Qwen3VLChatHandler` | `qwen3-vl` | | [qwen3.5](https://huggingface.co/unsloth/Qwen3.5-27B-GGUF) | `Qwen35ChatHandler` | `qwen3.5` | +| [step3-vl](https://huggingface.co/JamePeng2023/Step3-VL-10B-GGUF) | `Step3VLChatHandler` | `step3-vl` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 3c426b38f5..73c186aa8d 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -5512,6 +5512,143 @@ def __call__(self, **kwargs): return super().__call__(**kwargs) +class Step3VLChatHandler(MTMDChatHandler): + """ + Handler for Step3-VL models. + """ + + STEP3VL_BOS_TOKEN = "<|im_start|>" + STEP3VL_EOS_TOKEN = "<|im_end|>" + STEP3VL_PAD_TOKEN = "<|endoftext|>" + STEP3VL_IMAGE_TOKEN = "" + + CHAT_FORMAT = ( + "{%- macro render_content(content) -%}\n" + " {%- if content is none -%}{{- '' -}}\n" + " {%- elif content is string -%}{{- content -}}\n" + " {%- elif content is mapping -%}{{- content['value'] if 'value' in content else content['text'] -}}\n" + " {%- elif content is iterable -%}\n" + " {%- for item in content -%}\n" + " {%- if item.type == 'text' -%}\n" + " {{- item['value'] if 'value' in item else item['text'] -}}\n" + " {%- elif item.type in ['image', 'image_url'] -%}\n" + " {%- set url_val = '' -%}\n" + " {%- if item.image_url -%}\n" + " {%- set url_val = item.image_url if item.image_url is string else item.image_url.url -%}\n" + " {%- endif -%}\n" + " {{- '' + url_val -}}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + "{%- endmacro -%}\n" + "\n" + "{%- if tools -%}\n" + " {{- '<|im_start|>system\\n' -}}\n" + " {%- if messages[0].role == 'system' -%}\n" + " {{- render_content(messages[0].content) + '\\n\\n' -}}\n" + " {%- endif -%}\n" + " {{- '# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n' -}}\n" + " {%- for tool in tools -%}\n" + " {{- '\\n' -}}\n" + " {{- tool | tojson -}}\n" + " {%- endfor -%}\n" + " {{- '\\n\\n\\nAlways adhere to this exact format for tool use:\\n\\n\\n{\"name\": , \"arguments\": }\\n\\n{additional_tool_calls}\\n\\nNote:\\n- For each function call, return a json object with function name and arguments within XML tags.\\n- `` must be an exact match to one of the available tools.\\n- `` must be valid JSON that strictly follows the tool\\'s parameters schema.<|im_end|>\\n' -}}\n" + "{%- else -%}\n" + " {%- if messages[0].role == 'system' -%}\n" + " {{- '<|im_start|>system\\n' + render_content(messages[0].content) + '<|im_end|>\\n' -}}\n" + " {%- endif -%}\n" + "{%- endif -%}\n" + "\n" + "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) -%}\n" + "{%- for message in messages[::-1] -%}\n" + " {%- set index = (messages|length - 1) - loop.index0 -%}\n" + " {%- if ns.multi_step_tool and message.role == 'user' and render_content(message.content) is string and not(render_content(message.content).startswith('') and render_content(message.content).endswith('')) -%}\n" + " {%- set ns.multi_step_tool = false -%}\n" + " {%- set ns.last_query_index = index -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "\n" + "{%- for message in messages -%}\n" + " {%- set content = render_content(message.content) -%}\n" + " {%- if (message.role == 'user') or (message.role == 'system' and not loop.first) -%}\n" + " {%- set role_name = 'observation' if (message.role == 'system' and not loop.first and message.name == 'observation') else message.role -%}\n" + " {{- '<|im_start|>' + role_name + '\\n' + content + '<|im_end|>' + '\\n' -}}\n" + " {%- elif message.role == 'assistant' -%}\n" + " {%- if message.reasoning_content is string -%}\n" + " {%- set reasoning_content = render_content(message.reasoning_content) -%}\n" + " {%- else -%}\n" + " {%- if '' in content -%}\n" + " {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') -%}\n" + " {%- set content = content.split('')[-1].lstrip('\\n') -%}\n" + " {%- else -%}\n" + " {%- set reasoning_content = '' -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- if loop.index0 > ns.last_query_index -%}\n" + " {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n' + content -}}\n" + " {%- else -%}\n" + " {{- '<|im_start|>' + message.role + '\\n' + content -}}\n" + " {%- endif -%}\n" + " {%- if message.tool_calls -%}\n" + " {{- '\\n' -}}\n" + " {%- for tool_call in message.tool_calls -%}\n" + " {{- '\\n' -}}\n" + " {%- if tool_call.function -%}\n" + " {%- set tool_call = tool_call.function -%}\n" + " {%- endif -%}\n" + " {{- '\\n{\"name\": \"' -}}\n" + " {{- tool_call.name -}}\n" + " {{- '\", \"arguments\": ' -}}\n" + " {%- if tool_call.arguments is string -%}\n" + " {{- tool_call.arguments -}}\n" + " {%- else -%}\n" + " {{- tool_call.arguments | tojson -}}\n" + " {%- endif -%}\n" + " {{- '}\\n' -}}\n" + " {%- endfor -%}\n" + " {{- '\\n' -}}\n" + " {%- endif -%}\n" + " {{- '<|im_end|>\\n' -}}\n" + " {%- elif message.role == 'tool' -%}\n" + " {%- if loop.first or (messages[loop.index0 - 1].role != 'tool') -%}\n" + " {{- '<|im_start|>tool_response' -}}\n" + " {%- endif -%}\n" + " {{- '\\n\\n' -}}\n" + " {{- content -}}\n" + " {{- '\\n' -}}\n" + " {%- if loop.last or (messages[loop.index0 + 1].role != 'tool') -%}\n" + " {{- '<|im_end|>\\n' -}}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "{%- if add_generation_prompt -%}\n" + " {{- '<|im_start|>assistant\\n\\n\\n\\n' if (enable_thinking is defined and not enable_thinking) else '<|im_start|>assistant\\n' -}}\n" + "{%- endif -%}\n" + ) + + def __init__(self, enable_thinking: bool = True, **kwargs): + """ + Initializes the Step3-VL Handler. + + Args: + enable_thinking (bool): If False, injects an empty block to bypass reasoning. + """ + self.enable_thinking = enable_thinking + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Pass thinking toggle into Jinja + self.extra_template_arguments["enable_thinking"] = self.enable_thinking + + # Step3 uses standard <|im_end|> ChatML stop formatting + kwargs['stop'] = [self.STEP3VL_PAD_TOKEN, self.STEP3VL_EOS_TOKEN] + + if self.verbose: + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") + + return super().__call__(**kwargs) + + @register_chat_completion_handler("chatml-function-calling") def chatml_function_calling( llama: llama_core.Llama, From 5a47267c1c45b5ea6002963325e51622038d04f1 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Apr 2026 03:40:52 +0800 Subject: [PATCH 012/304] Update Submodule vendor/llama.cpp d12cc3d..d132f22 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index d12cc3d1ca..d132f22fc9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit d12cc3d1ca6bba741cd77887ac9c9ee18c8415c7 +Subproject commit d132f22fc92f36848f7ccf2fc9987cd0b0120825 From 6866eba30deb12bf665339ea55ac8dce192872b5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Apr 2026 03:43:24 +0800 Subject: [PATCH 013/304] Sync ggml: backend-agnostic tensor parallelism (experimental) (#19378) --- llama_cpp/llama_cpp.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1e1e40b70f..a527904637 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -460,14 +460,16 @@ def llama_flash_attn_type_name( """ # enum llama_split_mode { -# LLAMA_SPLIT_MODE_NONE = 0, // single GPU -# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs -# LLAMA_SPLIT_MODE_ROW = 2, // split rows across GPUs +# LLAMA_SPLIT_MODE_NONE = 0, // single GPU +# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs +# LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported +# LLAMA_SPLIT_MODE_TENSOR = 3, # }; -LLAMA_SPLIT_MODE_NONE = 0 -LLAMA_SPLIT_MODE_LAYER = 1 -LLAMA_SPLIT_MODE_ROW = 2 - +class llama_split_mode(enum.IntEnum): + LLAMA_SPLIT_MODE_NONE = 0 + LLAMA_SPLIT_MODE_LAYER = 1 + LLAMA_SPLIT_MODE_ROW = 2 + LLAMA_SPLIT_MODE_TENSOR = 3 # typedef struct llama_token_data { # llama_token id; // token id From 7d2c4ba6c49ff4f3eec81d6eb84b0aab9421e451 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Apr 2026 04:05:29 +0800 Subject: [PATCH 014/304] Update Makefile --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 2e1cd1c71b..db99016262 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,9 @@ build.musa: build.openblas: CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" python3 -m pip install --verbose -e . +build.openvino: + CMAKE_ARGS="-DGGML_OPENVINO=ON" python3 -m pip install --verbose -e . + build.rpc: CMAKE_ARGS="-DGGML_RPC=on" python3 -m pip install --verbose -e . @@ -63,6 +66,9 @@ build.webgpu: build.zdnn: CMAKE_ARGS="-DGGML_ZDNN=ON" python3 -m pip install --verbose -e . +build.zendnn : + CMAKE_ARGS="-DGGML_ZENDNN=ON" python3 -m pip install --verbose -e . + build.sdist: python3 -m build --sdist --verbose From f22b819d8129d4ffe917429def81310e28d2967d Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Apr 2026 04:11:27 +0800 Subject: [PATCH 015/304] fix missing change of llama_cpp.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, --- llama_cpp/llama.py | 2 +- llama_cpp/server/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index d6c6926e60..59e636b56e 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -71,7 +71,7 @@ def __init__( *, # Model Params n_gpu_layers: int = 0, - split_mode: int = llama_cpp.LLAMA_SPLIT_MODE_LAYER, + split_mode: int = llama_cpp.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, vocab_only: bool = False, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index db96a41705..350ccc2323 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -31,7 +31,7 @@ class ModelSettings(BaseSettings): description="The number of layers to put on the GPU. The rest will be on the CPU. Set -1 to move all to GPU.", ) split_mode: int = Field( - default=llama_cpp.LLAMA_SPLIT_MODE_LAYER, + default=llama_cpp.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, description="The split mode to use.", ) main_gpu: int = Field( From 634a712e970f86bebec5ced94d04042fffacbe9b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Apr 2026 04:58:57 +0800 Subject: [PATCH 016/304] Update README.md for OpenVINO/Metal/Vulkan/SYCL Signed-off-by: JamePeng --- README.md | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 77eb0e661c..2639556ab0 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,8 @@ Installing a CUDA-supported version requires the `CUDA Toolkit` environment to b See here: https://developer.nvidia.com/cuda-toolkit-archive +More Information see: https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md#cuda + Then, set the `GGML_CUDA=on` environment variable before installing: ```bash @@ -169,13 +171,64 @@ CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" pip install "llama-cpp-p ``` +
+OpenVINO + +### Install OpenVINO Runtime + +Follow the guide to install OpenVINO Runtime from an archive file: [Linux](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-linux.html) | [Windows](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-windows.html) + +- **Linux:** + +
+ šŸ“¦ Click to expand OpenVINO installation from an archive file on Ubuntu +
+ + ```bash + wget https://raw.githubusercontent.com/ravi9/misc-scripts/main/openvino/ov-archive-install/install-openvino-from-archive.sh + chmod +x install-openvino-from-archive.sh + ./install-openvino-from-archive.sh + ``` + + Verify OpenVINO is initialized properly: + ```bash + echo $OpenVINO_DIR + ``` +
+ +### Supported Devices + +OpenVINO backend supports the following hardware: + +- Intel CPUs +- Intel GPUs (integrated and discrete) +- Intel NPUs + +Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), the llama.cpp OpenVINO backend has been validated specifically on AI PCs such as the IntelĀ® Coreā„¢ Ultra Series 1 and Series 2. + +More Information see: https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md + +To install with OpenVINO, set the `GGML_OPENVINO=ON` environment variable before installing: + +```bash +# Linux +source /opt/intel/openvino/setupvars.sh +# Windows +"C:\Program Files (x86)\Intel\openvino_2026.0\setupvars.bat" +# Build +CMAKE_ARGS="-DGGML_OPENVINO=ON" pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` +
+
Metal -To install with Metal (MPS), set the `GGML_METAL=on` environment variable before installing: +On MacOS, Metal is enabled by default(`GGML_METAL=ON`). Using Metal makes the computation run on the GPU. + +To disable the Metal build at compile time use the `CMAKE_ARGS="-DGGML_METAL=OFF"` cmake option. ```bash -CMAKE_ARGS="-DGGML_METAL=on -DGGML_METAL_USE_BF16=on" pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" ``` **Pre-built Wheel (New)** @@ -213,7 +266,20 @@ More details see here: https://github.com/ggml-org/llama.cpp/blob/master/docs/bu - For Windows User: Download and install the [`Vulkan SDK`](https://vulkan.lunarg.com/sdk/home#windows) with the default settings. -- For Linux User: Follow the official LunarG instructions for the installation and setup of the Vulkan SDK in the [Getting Started with the Linux Tarball Vulkan SDK](https://vulkan.lunarg.com/doc/sdk/latest/linux/getting_started.html) guide. +- For Linux User: + * First, follow the official LunarG instructions for the installation and setup of the Vulkan SDK in the [Getting Started with the Linux Tarball Vulkan SDK](https://vulkan.lunarg.com/doc/sdk/latest/linux/getting_started.html) guide. + + * After completing the first step, ensure that you have used the `source` command on the `setup_env.sh` file inside of the Vulkan SDK in your current terminal session. Otherwise, the build won't work. Additionally, if you close out of your terminal, you must perform this step again if you intend to perform a build. However, there are ways to make this persistent. Refer to the Vulkan SDK guide linked in the first step for more information about any of this. + +- For Mac User: + * Generally, follow LunarG's [Getting Started with the MacOS Vulkan SDK](https://vulkan.lunarg.com/doc/sdk/latest/mac/getting_started.html) guide for installation and setup of the Vulkan SDK. There are two options of Vulkan drivers on macOS, both of which implement translation layers to map Vulkan to Metal. They can be hot-swapped by setting the `VK_ICD_FILENAMES` environment variable to point to the respective ICD JSON file. Check the box for "KosmicKrisp" during the LunarG Vulkan SDK installation. + + * Set environment variable for the LunarG Vulkan SDK after installation (and optionally add to your shell profile for persistence): + ```bash + source /path/to/vulkan-sdk/setup-env.sh + ``` + +More Information see: https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md#vulkan Then install with Vulkan support by set the `GGML_VULKAN=on` environment variable before installing: @@ -226,11 +292,35 @@ CMAKE_ARGS="-DGGML_VULKAN=on" pip install "llama-cpp-python @ git+https://github
SYCL +### Supported OS + +| OS | Status | Verified | +|---------|---------|------------------------------------------------| +| Linux | Support | Ubuntu 22.04, Fedora Silverblue 39, Arch Linux | +| Windows | Support | Windows 11 | + +### Intel GPU + +SYCL backend supports Intel GPU Family: + +- Intel Data Center Max Series +- Intel Flex Series, Arc Series +- Intel Built-in Arc GPU +- Intel iGPU in Core CPU (11th Generation Core CPU and newer, refer to [oneAPI supported GPU](https://www.intel.com/content/www/us/en/developer/articles/system-requirements/intel-oneapi-base-toolkit-system-requirements.html#inpage-nav-1-1)). + +On older Intel GPUs, you may try [OpenCL](/docs/backend/OPENCL.md) although the performance is not optimal, and some GPUs may not support OpenCL nor have any GPGPU capabilities. + +More Information see here: https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/SYCL.md + To install with SYCL support, set the `GGML_SYCL=on` environment variable before installing: ```bash -source /opt/intel/oneapi/setvars.sh +# Export relevant ENV variables +source /opt/intel/oneapi/setvars.sh +# Option 1: Use FP32 (recommended for better performance in most cases) CMAKE_ARGS="-DGGML_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx" pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +# Option 2: Use FP16 +CMAKE_ARGS="-DGGML_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON" pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" ```
From e73ad59ee9b2c967f2a1012635e380d03bd5aecb Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 11 Apr 2026 06:06:21 +0800 Subject: [PATCH 017/304] Update Submodule vendor/llama.cpp d132f22..073bb2c --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index d132f22fc9..073bb2c20b 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit d132f22fc92f36848f7ccf2fc9987cd0b0120825 +Subproject commit 073bb2c20b5b2c919469653214aaa1a9895816a2 From 122d8dbb37ff60fe267d314e9744f67134dbfdfe Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 11 Apr 2026 23:58:05 +0800 Subject: [PATCH 018/304] refactor: update Gemma4ChatHandler with latest google/gemma-4-31B-it chat template from huggingface - Sync `Gemma4ChatHandler` logic with the upstream chat template, incorporating the new `format_tool_response_block` and OpenAI-compatible forward-scan tool resolution. Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 179 ++++++++++++++++++++++----------- 1 file changed, 123 insertions(+), 56 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 73c186aa8d..c0936883fc 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -4376,34 +4376,15 @@ class Gemma4ChatHandler(MTMDChatHandler): " description:<|\"|>{{ value['description'] }}<|\"|>\n" " {%- set add_comma = true -%}\n" " {%- endif -%}\n" - " {%- if value['nullable'] %}\n" - " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" - " nullable:true\n" - " {%- endif -%}\n" " {%- if value['type'] | upper == 'STRING' -%}\n" " {%- if value['enum'] -%}\n" " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" " enum:{{ format_argument(value['enum']) }}\n" " {%- endif -%}\n" - " {%- elif value['type'] | upper == 'OBJECT' -%}\n" - " ,properties:{\n" - " {%- if value['properties'] is defined and value['properties'] is mapping -%}\n" - " {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n" - " {%- elif value is mapping -%}\n" - " {{- format_parameters(value, value['required'] | default([])) -}}\n" - " {%- endif -%}\n" - " }\n" - " {%- if value['required'] -%}\n" - " ,required:[\n" - " {%- for item in value['required'] | default([]) -%}\n" - " <|\"|>{{- item -}}<|\"|>\n" - " {%- if not loop.last %},{% endif -%}\n" - " {%- endfor -%}\n" - " ]\n" - " {%- endif -%}\n" " {%- elif value['type'] | upper == 'ARRAY' -%}\n" " {%- if value['items'] is mapping and value['items'] -%}\n" - " ,items:{\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " items:{\n" " {%- set ns_items = namespace(found_first=false) -%}\n" " {%- for item_key, item_value in value['items'] | dictsort -%}\n" " {%- if item_value is not none -%}\n" @@ -4436,6 +4417,32 @@ class Gemma4ChatHandler(MTMDChatHandler): " }\n" " {%- endif -%}\n" " {%- endif -%}\n" + " {%- if value['nullable'] %}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " nullable:true\n" + " {%- endif -%}\n" + " {%- if value['type'] | upper == 'OBJECT' -%}\n" + " {%- if value['properties'] is defined and value['properties'] is mapping -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " properties:{\n" + " {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n" + " }\n" + " {%- elif value is mapping -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " properties:{\n" + " {{- format_parameters(value, value['required'] | default([])) -}}\n" + " }\n" + " {%- endif -%}\n" + " {%- if value['required'] -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " required:[\n" + " {%- for item in value['required'] | default([]) -%}\n" + " <|\"|>{{- item -}}<|\"|>\n" + " {%- if not loop.last %},{% endif -%}\n" + " {%- endfor -%}\n" + " ]\n" + " {%- endif -%}\n" + " {%- endif -%}\n" " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" " type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n" " {%- endif -%}\n" @@ -4514,25 +4521,35 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor -%}\n" " {{- ns.result | trim -}}\n" "{%- endmacro -%}\n" - "\n" + "{%- macro format_tool_response_block(tool_name, response) -%}\n" + " {{- '<|tool_response>' -}}\n" + " {%- if response is mapping -%}\n" + " {{- 'response:' + tool_name + '{' -}}\n" + " {%- for key, value in response | dictsort -%}\n" + " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" + " {%- if not loop.last %},{% endif -%}\n" + " {%- endfor -%}\n" + " {{- '}' -}}\n" + " {%- else -%}\n" + " {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}\n" + " {%- endif -%}\n" + " {{- '' -}}\n" + "{%- endmacro -%}\n" "{%- set ns = namespace(prev_message_type=None) -%}\n" "{%- set loop_messages = messages -%}\n" - "{{ bos_token }}\n" + "{{- bos_token -}}\n" "{#- Handle System/Tool Definitions Block -#}\n" "{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n" " {{- '<|turn>system\\n' -}}\n" - "\n" " {#- Inject Thinking token at the very top of the FIRST system turn -#}\n" " {%- if enable_thinking is defined and enable_thinking -%}\n" - " {{- '<|think|>' -}}\n" + " {{- '<|think|>\\n' -}}\n" " {%- set ns.prev_message_type = 'think' -%}\n" " {%- endif -%}\n" - "\n" " {%- if messages[0]['role'] in ['system', 'developer'] -%}\n" " {{- messages[0]['content'] | trim -}}\n" " {%- set loop_messages = messages[1:] -%}\n" " {%- endif -%}\n" - "\n" " {%- if tools -%}\n" " {%- for tool in tools %}\n" " {{- '<|tool>' -}}\n" @@ -4541,16 +4558,41 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor %}\n" " {%- set ns.prev_message_type = 'tool' -%}\n" " {%- endif -%}\n" - "\n" " {{- '\\n' -}}\n" "{%- endif %}\n" - "\n" + "{#- Pre-scan: find last user message index for reasoning guard -#}\n" + "{%- set ns_turn = namespace(last_user_idx=-1) -%}\n" + "{%- for i in range(loop_messages | length) -%}\n" + " {%- if loop_messages[i]['role'] == 'user' -%}\n" + " {%- set ns_turn.last_user_idx = i -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" "{#- Loop through messages -#}\n" "{%- for message in loop_messages -%}\n" + " {%- if message['role'] != 'tool' -%}\n" " {%- set ns.prev_message_type = None -%}\n" " {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n" + " {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n" + " {%- set prev_nt = namespace(role=None, found=false) -%}\n" + " {%- if loop.index0 > 0 -%}\n" + " {%- for j in range(loop.index0 - 1, -1, -1) -%}\n" + " {%- if not prev_nt.found -%}\n" + " {%- if loop_messages[j]['role'] != 'tool' -%}\n" + " {%- set prev_nt.role = loop_messages[j]['role'] -%}\n" + " {%- set prev_nt.found = true -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + " {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n" + " {%- if not continue_same_model_turn -%}\n" " {{- '<|turn>' + role + '\\n' }}\n" - "\n" + " {%- endif -%}\n" + " {#- Render reasoning/reasoning_content as thinking channel -#}\n" + " {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n" + " {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n" + " {{- '<|channel>thought\\n' + thinking_text + '\\n' -}}\n" + " {%- endif -%}\n" " {%- if message['tool_calls'] -%}\n" " {%- for tool_call in message['tool_calls'] -%}\n" " {%- set function = tool_call['function'] -%}\n" @@ -4569,26 +4611,50 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor -%}\n" " {%- set ns.prev_message_type = 'tool_call' -%}\n" " {%- endif -%}\n" - "\n" - " {%- if message['tool_responses'] -%}\n" - " {#- Tool Response handling -#}\n" + " {%- set ns_tr_out = namespace(flag=false) -%}\n" + " {%- if message.get('tool_responses') -%}\n" + " {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n" " {%- for tool_response in message['tool_responses'] -%}\n" - " {{- '<|tool_response>' -}}\n" - " {%- if tool_response['response'] is mapping -%}\n" - " {{- 'response:' + tool_response['name'] | default('unknown') + '{' -}}\n" - " {%- for key, value in tool_response['response'] | dictsort -%}\n" - " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" - " {%- if not loop.last %},{% endif -%}\n" - " {%- endfor -%}\n" - " {{- '}' -}}\n" + " {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n" + " {%- set ns_tr_out.flag = true -%}\n" + " {%- set ns.prev_message_type = 'tool_response' -%}\n" + " {%- endfor -%}\n" + " {%- elif message.get('tool_calls') -%}\n" + " {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}\n" + " {%- set ns_tool_scan = namespace(stopped=false) -%}\n" + " {%- for k in range(loop.index0 + 1, loop_messages | length) -%}\n" + " {%- if ns_tool_scan.stopped -%}\n" + " {%- elif loop_messages[k]['role'] != 'tool' -%}\n" + " {%- set ns_tool_scan.stopped = true -%}\n" " {%- else -%}\n" - " {{- 'response:' + tool_response['name'] | default('unknown') + '{value:' + format_argument(tool_response['response'], escape_keys=False) + '}' -}}\n" + " {%- set follow = loop_messages[k] -%}\n" + " {#- Resolve tool_call_id to function name -#}\n" + " {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n" + " {%- for tc in message['tool_calls'] -%}\n" + " {%- if tc.get('id') == follow.get('tool_call_id') -%}\n" + " {%- set ns_tname.name = tc['function']['name'] -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {#- Handle content as string or content-parts array -#}\n" + " {%- set tool_body = follow.get('content') -%}\n" + " {%- if tool_body is string -%}\n" + " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n" + " {%- elif tool_body is sequence and tool_body is not string -%}\n" + " {%- set ns_txt = namespace(s='') -%}\n" + " {%- for part in tool_body -%}\n" + " {%- if part.get('type') == 'text' -%}\n" + " {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n" + " {%- else -%}\n" + " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n" + " {%- endif -%}\n" + " {%- set ns_tr_out.flag = true -%}\n" + " {%- set ns.prev_message_type = 'tool_response' -%}\n" " {%- endif -%}\n" - " {{- '' -}}\n" " {%- endfor -%}\n" - " {%- set ns.prev_message_type = 'tool_response' -%}\n" " {%- endif -%}\n" - "\n" " {%- if message['content'] is string -%}\n" " {%- if role == 'model' -%}\n" " {{- strip_thinking(message['content']) -}}\n" @@ -4605,35 +4671,36 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endif -%}\n" " {%- elif item['type'] == 'image_url' -%}\n" " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" - " {{- '\\n\\n<|image|>' + url_val + '\\n\\n' -}}\n" + " {{- '<|image|>' + url_val -}}\n" " {%- set ns.prev_message_type = 'image' -%}\n" " {%- elif item['type'] == 'audio_url' -%}\n" " {%- set audio_val = item['audio_url'] if item['audio_url'] is string else item['audio_url']['url'] -%}\n" - " {{- '\\n\\n<|audio|>' + audio_val + '\\n\\n' -}}\n" + " {{- '<|audio|>' + audio_val -}}\n" " {%- set ns.prev_message_type = 'audio' -%}\n" " {%- elif item['type'] == 'input_audio' -%}\n" " {%- set audio_val = item['input_audio'] if item['input_audio'] is string else ('data:audio/' + item['input_audio']['format'] + ';base64,' + item['input_audio']['data']) -%}\n" - " {{- '\\n\\n<|audio|>' + audio_val + '\\n\\n' -}}\n" + " {{- '<|audio|>' + audio_val -}}\n" " {%- set ns.prev_message_type = 'audio' -%}\n" # " {%- elif item['type'] == 'video_url' -%}\n" # " {%- set video_val = item['video_url'] if item['video_url'] is string else item['video_url']['url'] -%}\n" - # " {{- '\\n\\n<|video|>' + video_val + '\\n\\n' -}}\n" + # " {{- '<|video|>' + video_val -}}\n" # " {%- set ns.prev_message_type = 'video' -%}\n" " {%- endif -%}\n" " {%- endfor -%}\n" " {%- endif -%}\n" - "\n" - " {%- if not (message['tool_responses'] and not message['content']) -%}\n" + " {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n" + " {{- '<|tool_response>' -}}\n" + " {%- elif not (ns_tr_out.flag and not message.get('content')) -%}\n" " {{- '\\n' -}}\n" " {%- endif -%}\n" + " {%- endif -%}\n" "{%- endfor -%}\n" - "\n" "{%- if add_generation_prompt -%}\n" - " {%- if ns.prev_message_type != 'tool_response' -%}\n" + " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n" " {{- '<|turn>model\\n' -}}\n" - " {%- endif -%}\n" - " {%- if not enable_thinking | default(false) -%}\n" - " {{- '<|channel>thought\\n' -}}\n" + " {%- if not enable_thinking | default(false) -%}\n" + " {{- '<|channel>thought\\n' -}}\n" + " {%- endif -%}\n" " {%- endif -%}\n" "{%- endif -%}\n" ) From 4ec15acb4249c18aa613d72e32af9d24478ae82a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 13 Apr 2026 22:16:18 +0800 Subject: [PATCH 019/304] Update Submodule vendor/llama.cpp 073bb2c..75f3bc9 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 073bb2c20b..75f3bc94e6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 073bb2c20b5b2c919469653214aaa1a9895816a2 +Subproject commit 75f3bc94e649616162981c322e8e6b88ca5491e8 From 5e6529ea024644a00c3a3da48e137d8f6e849124 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 14 Apr 2026 06:24:55 +0800 Subject: [PATCH 020/304] docs: add audio processing recommendation to Gemma4ChatHandler - Recommend BF16 mmproj for Gemma4 E2B and E4B models. - Note known degraded audio performance with other quantizations. - Add reference link to the relevant llama.cpp PR/issue comment. Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index c0936883fc..1149c7677c 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -4342,6 +4342,11 @@ class Gemma4ChatHandler(MTMDChatHandler): Note on `enable_thinking`: The `enable_thinking` toggle is currently ONLY supported by Gemma4 31B and 26BA4B models. It is NOT supported by Gemma4 E2B and E4B models. + + [Important Note for Audio Processing!] + It is recommended to use BF16 mmproj for Gemma4 E2B and E4B models. + Other quantizations are known to have degraded performance; + ref comment: https://github.com/ggml-org/llama.cpp/pull/21421#issuecomment-4230306463 """ # The special token in Gemma 4 From e70304be471e0fe4fbcb3bcc08e6b2bf0e98262d Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 14 Apr 2026 21:41:26 +0800 Subject: [PATCH 021/304] Update Submodule vendor/llama.cpp 75f3bc9..1f30ac0 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 75f3bc94e6..1f30ac0cea 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 75f3bc94e649616162981c322e8e6b88ca5491e8 +Subproject commit 1f30ac0ceac0e2b4400069d81857089b6e04872a From 711577248c6d4928bfa3b79111a3ef876803a272 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 14 Apr 2026 21:47:25 +0800 Subject: [PATCH 022/304] docs: add comprehensive omni multimodal example for Gemma-4 - Wrapped the existing Qwen3-VL image loading example in a `
` block to improve README readability and save vertical space. - Introduced a complete, production-ready "Omni MultiModal" example demonstrating simultaneous Vision and Audio processing using the `Gemma4ChatHandler`. - Added a universal `build_media_payload` helper function to dynamically route and encode local files into OpenAI-compatible `image_url` and `input_audio` payload structures. - Added crucial documentation clarifying multimodal capability differences across Gemma-4 variants (E2B/E4B supporting full audio/vision vs. 31B/26BA4B supporting vision only). Signed-off-by: JamePeng --- README.md | 165 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 164 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2639556ab0..7dcd26576c 100644 --- a/README.md +++ b/README.md @@ -900,9 +900,13 @@ print(response["choices"][0]["text"]) **Note**: Multi-modal models also support tool calling and JSON mode. + ## Loading a Local Image With Qwen3VL(Thinking/Instruct) -This script demonstrates how to load a local image, encode it as a base64 Data URI, and pass it to a local Qwen3-VL model (with the 'force_reasoning' parameter enabled for thinking model, disabled for instruct model) for processing using the llama-cpp-python library. +This script demonstrates how to load a local image, encode it as a base64 Data URI, and pass it to a local Qwen3-VL model (with the 'force_reasoning' parameter enabled for thinking model, disabled for instruct model) for processing using the llama-cpp-python library.
+ + +**Example Code**:
```python # Import necessary libraries @@ -1052,6 +1056,165 @@ print(res["choices"][0]["message"]["content"]) ``` +
+ +## Comprehensive Omni MultiModal Example: Gemma-4 (Vision + Audio + Text) + +Below is a complete, production-ready example demonstrating how to dynamically route and process both image and audio files. It includes a universal media processor that automatically converts local files into the correct payload structure (Data URIs for images, and `input_audio` for audio files). + +> **āš ļø IMPORTANT: GEMMA-4 MODEL CAPABILITIES & LIMITATIONS** +> * **Gemma4 E2B / E4B:** Supports Full Multimodal (Vision + Audio + Text). `enable_thinking` **MUST** be `True`(default). +> * **Gemma4 31B / 26BA4B:** Supports Vision + Text ONLY (Audio is NOT supported). `enable_thinking` can be toggled (`True` or `False`). + +```python +from llama_cpp import Llama +from llama_cpp.llama_chat_format import Gemma4ChatHandler +import base64 +import os + +# Model and multimodal projection paths +MODEL_PATH = r"/path/to/Gemma-4-E4B-It-BF16.gguf" +# BF16 mmproj is required for audio. Other quantizations are known to have degraded performance. +MMPROJ_PATH = r"/path/to/mmproj-Gemma-4-E4B-It-BF16.gguf" + +# Initialize the Llama model with multimodal support +# Note: Since we are using E4B here, enable_thinking MUST be True, and audio is supported. +llm = Llama( + model_path=MODEL_PATH, + chat_handler=Gemma4ChatHandler( + clip_model_path=MMPROJ_PATH, + enable_thinking=True, # MUST be True for E2B/E4B models + verbose=True, # Enable Debug Info + ), + n_gpu_layers=-1, + n_ctx=10240, + verbose=True, # Enable Debug Info +) + +# 1. Extend the MIME dictionary to support audio formats +_MEDIA_MIME_TYPES = { + # ------ Image formats ------ + '.png': ('image', 'image/png'), + '.jpg': ('image', 'image/jpeg'), + '.jpeg': ('image', 'image/jpeg'), + '.gif': ('image', 'image/gif'), + '.webp': ('image', 'image/webp'), + '.bmp': ('image', 'image/bmp'), + + # ------ Audio formats ------ + '.wav': ('audio', 'wav'), # OpenAI standard usually uses raw format names for audio + '.mp3': ('audio', 'mp3'), + # '.flac': ('audio', 'flac'), +} + +def build_media_payload(file_path: str) -> dict: + """ + Read a local media file (image or audio) and convert it into a valid input payload for the LLM. + """ + if not os.path.isfile(file_path): + raise FileNotFoundError(f"Media file not found: {file_path}") + + extension = os.path.splitext(file_path)[1].lower() + media_category, mime_or_format = _MEDIA_MIME_TYPES.get(extension, ('unknown', 'application/octet-stream')) + + if media_category == 'unknown': + print(f"Warning: Unknown extension '{extension}'. It might not be processed correctly.") + + # Read and Base64 encode the file + with open(file_path, "rb") as f: + encoded_data = base64.b64encode(f.read()).decode("utf-8") + + # 2. Return the appropriate dictionary structure based on the media type + if media_category == 'image': + # Image format: Data URI (OpenAI compatible) + data_uri = f"data:{mime_or_format};base64,{encoded_data}" + return { + "type": "image_url", + "image_url": {"url": data_uri} + } + + elif media_category == 'audio': + # Audio format: input_audio (OpenAI compatible) + return { + "type": "input_audio", + "input_audio": { + "data": encoded_data, + "format": mime_or_format + } + } + else: + # Fallback for unsupported formats + return {"type": "text", "text": f"[Attached unsupported file: {file_path}]"} + + +def run_inference(media_paths: list, text_prompt: str): + """ + Helper function to dynamically build the payload and run inference. + """ + # 3. Build the user_content list + user_content = [] + + # Automatically parse each file and append to the payload + for path in media_paths: + payload = build_media_payload(path) + user_content.append(payload) + + # Append the final text instruction + user_content.append({ + "type": "text", + "text": text_prompt + }) + + print(f"\n--- Running Inference with {len(media_paths)} media file(s) ---") + + # 4. Send to the model for inference + response = llm.create_chat_completion( + messages=[ + {"role": "system", "content": """ + You are a highly capable multimodal assistant that can process both text, vision and audio. + + """}, # Note: Supported ONLY by Gemma4 E2B / E4B. + {"role": "user", "content": user_content} + ], + temperature=1.0, + top_p=0.95, + top_k=64, + max_tokens=8192, + ) + + print("\n[Model Response]:") + print(response["choices"][0]["message"]["content"]) + print("-" * 60) + + +# ============================================================================== +# Main Inference Examples +# Uncomment the example block you wish to execute. +# ============================================================================== + +# --- Example A: Image + Audio (Full Multimodal) --- +# Note: Supported ONLY by Gemma4 E2B / E4B. +run_inference( + media_paths=[r"/path/to/test.png", r"/path/to/test.wav"], + text_prompt="Introduce the content by combining the images and converting the audio to text." +) + +# --- Example B: Image Only (Vision + Text) --- +# Note: Supported by all Gemma4 variants (E2B, E4B, 31B, 26BA4B). +# run_inference( +# media_paths=[r"/path/to/test.png"], +# text_prompt="Describe the contents of this image in detail." +# ) + +# --- Example C: Audio Only (Audio + Text) --- +# Note: Supported ONLY by Gemma4 E2B / E4B. +# run_inference( +# media_paths=[r"/path/to/test.wav"], +# text_prompt="Transcribe this audio and summarize the main points." +# ) +``` + + --- ## Embeddings & Reranking (GGUF) From 701195e63a4a09b48dfd28936980532a03598ba3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 15 Apr 2026 20:23:27 +0800 Subject: [PATCH 023/304] Update Submodule vendor/llama.cpp 1f30ac0..8dc530b --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1f30ac0cea..8dc530b86d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1f30ac0ceac0e2b4400069d81857089b6e04872a +Subproject commit 8dc530b86d44cd0667a685539f29ded70a08ae0a From 9d2b2cb0b2a3c175681a5f720bdbb0eafd26e957 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 15 Apr 2026 23:10:56 +0800 Subject: [PATCH 024/304] Sync mtmd: add mtmd_image_tokens_get_decoder_pos() API (#21851) Signed-off-by: JamePeng --- llama_cpp/mtmd_cpp.py | 115 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 100 insertions(+), 15 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 3368f848e8..ba64b21ef6 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -82,15 +82,59 @@ class mtmd_input_chunk_type(enum.IntEnum): mtmd_context_p = NewType("mtmd_context_p", int) mtmd_context_p_ctypes = c_void_p -# struct mtmd_bitmap; +# // represents raw image data, layout is RGBRGBRGB... +# // length of data must be nx * ny * 3 +# struct mtmd_bitmap { +# uint32_t nx; +# uint32_t ny; +# std::vector data; +# std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking +# bool is_audio = false; // true if the bitmap is audio +# }; mtmd_bitmap_p = NewType("mtmd_bitmap_p", int) mtmd_bitmap_p_ctypes = c_void_p -# struct mtmd_image_tokens; +# struct mtmd_image_tokens { +# uint32_t nx; // number of tokens in x direction +# uint32_t ny; // number of tokens in y direction +# bool use_mrope_pos = false; // use M-RoPE position counting (the whole image is 1 temporal position) +# uint32_t n_tokens() const { return nx * ny; } +# clip_image_f32_batch batch_f32; // preprocessed image patches +# std::string id; // optional user-defined ID, useful for KV cache tracking +# mtmd_image_tokens clone() { +# return mtmd_image_tokens{ +# nx, +# ny, +# use_mrope_pos, +# batch_f32.clone(), +# id +# }; +# } +# }; mtmd_image_tokens_p = NewType("mtmd_image_tokens_p", int) mtmd_image_tokens_p_ctypes = c_void_p -# struct mtmd_input_chunk; +# struct mtmd_audio_tokens { +# uint32_t n_tokens; // number of tokens +# clip_image_f32_batch batch_f32; // preprocessed image patches +# std::string id; // optional user-defined ID, useful for KV cache tracking +# mtmd_audio_tokens clone() { +# return mtmd_audio_tokens{ +# n_tokens, +# batch_f32.clone(), +# id +# }; +# } +# }; +mtmd_audio_tokens_p = NewType("mtmd_audio_tokens_p", int) +mtmd_audio_tokens_p_ctypes = c_void_p + +# struct mtmd_input_chunk { +# mtmd_input_chunk_type type; +# std::vector tokens_text; +# mtmd_image_tokens_ptr tokens_image; +# mtmd_audio_tokens_ptr tokens_audio; +# }; mtmd_input_chunk_p = NewType("mtmd_input_chunk_p", int) mtmd_input_chunk_p_ctypes = c_void_p @@ -487,18 +531,6 @@ def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p): def mtmd_image_tokens_get_n_tokens(image_tokens: mtmd_image_tokens_p) -> c_size_t: ... -# MTMD_API size_t mtmd_image_tokens_get_nx (const mtmd_image_tokens * image_tokens); -@ctypes_function_mtmd( - "mtmd_image_tokens_get_nx", [mtmd_image_tokens_p_ctypes], c_size_t) -def mtmd_image_tokens_get_nx(image_tokens: mtmd_image_tokens_p) -> c_size_t: - ... - -# MTMD_API size_t mtmd_image_tokens_get_ny (const mtmd_image_tokens * image_tokens); -@ctypes_function_mtmd( - "mtmd_image_tokens_get_ny", [mtmd_image_tokens_p_ctypes], c_size_t) -def mtmd_image_tokens_get_ny(image_tokens: mtmd_image_tokens_p) -> c_size_t: - ... - # MTMD_API const char * mtmd_image_tokens_get_id (const mtmd_image_tokens * image_tokens); // TODO: deprecate @ctypes_function_mtmd( "mtmd_image_tokens_get_id", [mtmd_image_tokens_p_ctypes], c_char_p) @@ -513,6 +545,59 @@ def mtmd_image_tokens_get_n_pos(image_tokens: mtmd_image_tokens_p) -> c_int32: """number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise)""" ... +# DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens), +# "use mtmd_image_tokens_get_decoder_pos() instead"); +@ctypes_function_mtmd( + "mtmd_image_tokens_get_nx", [mtmd_image_tokens_p_ctypes], c_size_t) +def mtmd_image_tokens_get_nx(image_tokens: mtmd_image_tokens_p) -> c_size_t: + """ + use mtmd_image_tokens_get_decoder_pos() instead + """ + ... + +# DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens), +# "use mtmd_image_tokens_get_decoder_pos() instead"); +@ctypes_function_mtmd( + "mtmd_image_tokens_get_ny", [mtmd_image_tokens_p_ctypes], c_size_t) +def mtmd_image_tokens_get_ny(image_tokens: mtmd_image_tokens_p) -> c_size_t: + """ + use mtmd_image_tokens_get_decoder_pos() instead + """ + ... + +# struct mtmd_decoder_pos { +# uint32_t t; +# uint32_t x; +# uint32_t y; +# }; +class mtmd_decoder_pos(Structure): + _fields_ = [ + ("t", c_uint32), + ("x", c_uint32), + ("y", c_uint32), + ] + + if TYPE_CHECKING: + t: c_uint32 + x: c_uint32 + y: c_uint32 + +# // get position for decoder attention, to be used by M-RoPE models +# // i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 +# // return relative position (for example, embedding 0 will have position (0, 0, 0); +# // remember to adjust it to the current absolute position) +# MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, size_t i); +@ctypes_function_mtmd( + "mtmd_image_tokens_get_decoder_pos", [mtmd_image_tokens_p_ctypes, c_size_t], mtmd_decoder_pos) +def mtmd_image_tokens_get_decoder_pos(image_tokens: mtmd_image_tokens_p, i: c_size_t) -> mtmd_decoder_pos: + """ + get position for decoder attention, to be used by M-RoPE models + i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 + return relative position (for example, embedding 0 will have position (0, 0, 0); + remember to adjust it to the current absolute position) + """ + ... + # // tokenize an input text prompt and a list of bitmaps (images/audio) # // the prompt must have the input image marker (default: "<__media__>") in it # // the default marker is defined by mtmd_default_marker() From 3ee7ff8912466485d27325645717176e4626610a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 16 Apr 2026 23:45:02 +0800 Subject: [PATCH 025/304] chore(ci): upgrade softprops/action-gh-release to v3 (Node 24 runtime) --- .github/workflows/build-wheels-cu124-linux.yml | 2 +- .github/workflows/build-wheels-cu124-win.yml | 2 +- .github/workflows/build-wheels-cu126-linux.yml | 2 +- .github/workflows/build-wheels-cu126-win.yml | 2 +- .github/workflows/build-wheels-cu128-linux.yml | 2 +- .github/workflows/build-wheels-cu128-win.yml | 2 +- .github/workflows/build-wheels-cu130-linux.yml | 2 +- .github/workflows/build-wheels-cu130-win.yml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-wheels-cu124-linux.yml b/.github/workflows/build-wheels-cu124-linux.yml index f14684289d..42b3d13169 100644 --- a/.github/workflows/build-wheels-cu124-linux.yml +++ b/.github/workflows/build-wheels-cu124-linux.yml @@ -120,7 +120,7 @@ jobs: # Store the date in environment variable for the release step echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV - - uses: softprops/action-gh-release@v2.2.2 # Action to create a GitHub Release + - uses: softprops/action-gh-release@v3 # Action to create a GitHub Release with: files: dist/* # Upload the generated wheel files from the dist directory # Define the release tag name using the collected environment variables diff --git a/.github/workflows/build-wheels-cu124-win.yml b/.github/workflows/build-wheels-cu124-win.yml index 0989afc8dd..135b847d32 100644 --- a/.github/workflows/build-wheels-cu124-win.yml +++ b/.github/workflows/build-wheels-cu124-win.yml @@ -125,7 +125,7 @@ jobs: - name: Create Release if: always() && env.TAG_VERSION != '' - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: dist/* # Set tag_name to -cu--win- diff --git a/.github/workflows/build-wheels-cu126-linux.yml b/.github/workflows/build-wheels-cu126-linux.yml index 1eda5d10f2..f60eb5f878 100644 --- a/.github/workflows/build-wheels-cu126-linux.yml +++ b/.github/workflows/build-wheels-cu126-linux.yml @@ -120,7 +120,7 @@ jobs: # Store the date in environment variable for the release step echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV - - uses: softprops/action-gh-release@v2.2.2 # Action to create a GitHub Release + - uses: softprops/action-gh-release@v3 # Action to create a GitHub Release with: files: dist/* # Upload the generated wheel files from the dist directory # Define the release tag name using the collected environment variables diff --git a/.github/workflows/build-wheels-cu126-win.yml b/.github/workflows/build-wheels-cu126-win.yml index 19474b530d..be7bfdc72c 100644 --- a/.github/workflows/build-wheels-cu126-win.yml +++ b/.github/workflows/build-wheels-cu126-win.yml @@ -125,7 +125,7 @@ jobs: - name: Create Release if: always() && env.TAG_VERSION != '' - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: dist/* # Set tag_name to -cu--win- diff --git a/.github/workflows/build-wheels-cu128-linux.yml b/.github/workflows/build-wheels-cu128-linux.yml index a4ab9e8eb2..0bfe971eea 100644 --- a/.github/workflows/build-wheels-cu128-linux.yml +++ b/.github/workflows/build-wheels-cu128-linux.yml @@ -120,7 +120,7 @@ jobs: # Store the date in environment variable for the release step echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV - - uses: softprops/action-gh-release@v2.2.2 # Action to create a GitHub Release + - uses: softprops/action-gh-release@v3 # Action to create a GitHub Release with: files: dist/* # Upload the generated wheel files from the dist directory # Define the release tag name using the collected environment variables diff --git a/.github/workflows/build-wheels-cu128-win.yml b/.github/workflows/build-wheels-cu128-win.yml index 0d87e09a45..80dd9f2f74 100644 --- a/.github/workflows/build-wheels-cu128-win.yml +++ b/.github/workflows/build-wheels-cu128-win.yml @@ -125,7 +125,7 @@ jobs: - name: Create Release if: always() && env.TAG_VERSION != '' - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: dist/* # Set tag_name to -cu--win- diff --git a/.github/workflows/build-wheels-cu130-linux.yml b/.github/workflows/build-wheels-cu130-linux.yml index dbc710e18a..23cd668c8d 100644 --- a/.github/workflows/build-wheels-cu130-linux.yml +++ b/.github/workflows/build-wheels-cu130-linux.yml @@ -120,7 +120,7 @@ jobs: # Store the date in environment variable for the release step echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV - - uses: softprops/action-gh-release@v2.2.2 # Action to create a GitHub Release + - uses: softprops/action-gh-release@v3 # Action to create a GitHub Release with: files: dist/* # Upload the generated wheel files from the dist directory # Define the release tag name using the collected environment variables diff --git a/.github/workflows/build-wheels-cu130-win.yml b/.github/workflows/build-wheels-cu130-win.yml index 5b8b9c1992..b995f4f5f4 100644 --- a/.github/workflows/build-wheels-cu130-win.yml +++ b/.github/workflows/build-wheels-cu130-win.yml @@ -125,7 +125,7 @@ jobs: - name: Create Release if: always() && env.TAG_VERSION != '' - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: files: dist/* # Set tag_name to -cu--win- From 9e00017812148a755058854d6f62458af8cdbe8c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 17 Apr 2026 00:11:53 +0800 Subject: [PATCH 026/304] Update Submodule vendor/llama.cpp 8dc530b..9db77a0 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8dc530b86d..9db77a020c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8dc530b86d44cd0667a685539f29ded70a08ae0a +Subproject commit 9db77a020c97ac3b13b7c1bf4e0c5787001533e7 From 7820677e65827b6f3356f651da9be8d510ba10e5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 17 Apr 2026 00:15:13 +0800 Subject: [PATCH 027/304] feat: enhance Qwen35ChatHandler with preserve_thinking and Qwen3.6 template fixes - Add `preserve_thinking` parameter to optionally retain `` reasoning blocks across all historical conversational turns (defaults to False to save tokens). - Improve template robustness by adding an `is defined` safety check for `enable_thinking`. - Simplify JSON serialization logic for tool call arguments in the Jinja template. - Update class docstring to explicitly indicate support for Qwen 3.5 and Qwen 3.6 models. - Include `preserve_thinking` state in verbose processing logs. --- README.md | 1 + llama_cpp/llama_chat_format.py | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7dcd26576c..ab643fe427 100644 --- a/README.md +++ b/README.md @@ -834,6 +834,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | | [qwen3-vl](https://huggingface.co/unsloth/Qwen3-VL-8B-Thinking-GGUF) | `Qwen3VLChatHandler` | `qwen3-vl` | | [qwen3.5](https://huggingface.co/unsloth/Qwen3.5-27B-GGUF) | `Qwen35ChatHandler` | `qwen3.5` | +| [qwen3.6](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF) | `Qwen35ChatHandler` | `qwen3.6` | | [step3-vl](https://huggingface.co/JamePeng2023/Step3-VL-10B-GGUF) | `Step3VLChatHandler` | `step3-vl` | Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 1149c7677c..af068f5535 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -5387,6 +5387,9 @@ def __call__(self, **kwargs): return super().__call__(**kwargs) class Qwen35ChatHandler(MTMDChatHandler): + """ + Handler for Qwen3.5/Qwen3.6 models. + """ CHAT_FORMAT = ( "{%- set image_count = namespace(value=0) -%}" "{%- set video_count = namespace(value=0) -%}" @@ -5494,7 +5497,7 @@ class Qwen35ChatHandler(MTMDChatHandler): " {%- set content = content.split('')[-1].lstrip('\n') -%}" " {%- endif -%}" " {%- set reasoning_content = reasoning_content | trim -%}" - " {%- if loop.index0 > ns.last_query_index -%}" + " {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) -%}" " {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content -}}" " {%- else -%}" " {{- '<|im_start|>' + message.role + '\n' + content -}}" @@ -5516,7 +5519,7 @@ class Qwen35ChatHandler(MTMDChatHandler): " {%- if tool_call.arguments is defined -%}" " {%- for (args_name, args_value) in tool_call.arguments | items -%}" " {{- '\n' -}}" - " {%- set args_value = args_value | tojson | safe if args_value is mapping or args_value is sequence and args_value is not string else args_value | string -%}" + " {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}" " {{- args_value -}}" " {{- '\n' -}}" " {%- endfor -%}" @@ -5543,7 +5546,7 @@ class Qwen35ChatHandler(MTMDChatHandler): "{%- endfor -%}" "{%- if add_generation_prompt -%}" " {{- '<|im_start|>assistant\n' -}}" - " {%- if enable_thinking is false -%}" + " {%- if enable_thinking is defined and enable_thinking is false -%}" " {{- '\n\n\n\n' -}}" " {%- else -%}" " {{- '\n' -}}" @@ -5553,23 +5556,29 @@ class Qwen35ChatHandler(MTMDChatHandler): def __init__( self, - enable_thinking: bool = True, add_vision_id: bool = True, + enable_thinking: bool = True, + preserve_thinking: bool = False, **kwargs, ): """ Parameters: - - enable_thinking (bool): - - True (default): Enables reasoning for better results. - - False: Disables reasoning for faster results. - add_vision_id (bool): - True (default): Count all the images. Recommended for multi-image. - False: Doesn't count the images. Can save tokens with single-image. + - enable_thinking (bool): + - True (default): Enables reasoning for better results. + - False: Disables reasoning for faster results. + - preserve_thinking (bool): + - True: Keeps reasoning process for ALL historical conversational turns. + - False (default): Only keeps for the latest assistant reply to save tokens. """ super().__init__(**kwargs) self.enable_thinking = enable_thinking - self.extra_template_arguments["enable_thinking"] = enable_thinking + self.preserve_thinking = preserve_thinking self.extra_template_arguments["add_vision_id"] = add_vision_id + self.extra_template_arguments["enable_thinking"] = enable_thinking + self.extra_template_arguments["preserve_thinking"] = preserve_thinking def __call__(self, **kwargs): llama = kwargs['llama'] @@ -5578,7 +5587,7 @@ def __call__(self, **kwargs): llama.input_ids.fill(0) if self.verbose: - print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}, preserve_thinking={self.preserve_thinking}) - Start processing") # Use parent implementation return super().__call__(**kwargs) From b97cb637cd6124fc47f569721b1716014bd856a8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 17 Apr 2026 02:36:09 +0800 Subject: [PATCH 028/304] Bump version to 0.3.36 Signed-off-by: JamePeng --- CHANGELOG.md | 55 +++++++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 156cbd334e..480b3c24cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,61 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.36] Gemma-4 Omni-Multimodal and ToolCall Improved, Qwen3.6 / Step3-VL Support, Compilation workflow optimization + +- feat: enhance `Qwen35ChatHandler` with preserve_thinking and `Qwen3.6` Support + - Add `preserve_thinking` parameter to optionally retain `` reasoning + blocks across all historical conversational turns (defaults to False to save tokens). + - Improve template robustness by adding an `is defined` safety check for `enable_thinking`. + - Simplify JSON serialization logic for tool call arguments in the Jinja template. + - Update class docstring to explicitly indicate support for `Qwen 3.5` and `Qwen 3.6` models. + - Include `preserve_thinking` state in verbose processing logs. + +- docs: add comprehensive omni multimodal example for Gemma-4 (See here: [Gemma 4 Omni Example](https://github.com/JamePeng/llama-cpp-python?tab=readme-ov-file#comprehensive-omni-multimodal-example-gemma-4-vision--audio--text)) + - Wrapped the existing Qwen3-VL image loading example in a `
` block to improve README readability and save vertical space. + - Introduced a complete, production-ready "Omni MultiModal" example demonstrating simultaneous Vision and Audio processing using the `Gemma4ChatHandler`. + - Added a universal `build_media_payload` helper function to dynamically route and encode local files into OpenAI-compatible `image_url` and `input_audio` payload structures. + - Added crucial documentation clarifying multimodal capability differences across Gemma-4 variants (E2B/E4B supporting full audio/vision vs. 31B/26BA4B supporting vision only). + + +- docs: add audio processing recommendation to Gemma4ChatHandler + - Recommend BF16 mmproj for Gemma4 E2B and E4B models. + - Note known degraded audio performance with other quantizations. + - Add reference link to the relevant llama.cpp PR/issue comment. + +- refactor: update Gemma4ChatHandler with latest google/gemma-4-31B-it chat template from huggingface + - Sync `Gemma4ChatHandler` logic with the upstream chat template, incorporating the new `format_tool_response_block` and OpenAI-compatible forward-scan tool resolution. + +- Update README.md for OpenVINO/Metal/Vulkan/SYCL + +- Implement `Step3VLChatHandler` for `Step3-VL-10B` + +- feat(types): align with latest OpenAI API spec and fix type issues + - Expand `CompletionUsage` with `PromptTokensDetails` and `CompletionTokensDetails` for granular token tracking. + - Add `usage` to `CreateChatCompletionStreamResponse` to support usage reporting in streaming mode. + - Fix duplicate `object` field in `CreateCompletionResponse`. + - Update `ChatCompletionRequestAssistantMessage` to accept `None` for `content` and introduce the new `refusal` field. + - Clean up `ChatCompletionRequestMessage` Union by removing the duplicate user message type. + - Broaden `ChatCompletionToolChoiceOption` to fully support `allowed_tools` and `custom` tool choice behaviors. + +- feat(ci): Optimizing the GitHub build workflow for CUDA and METAL + - Update CI Action runner version + - microsoft/setup-msbuild@v2 -> v3 + - actions/checkout@v5 -> v6 + - actions/upload-artifact@v4 -> v6 + - actions/download-artifact@v4 -> v6 + - softprops/action-gh-release@v2 -> v3 + - ci: restrict cudaarch to Volta-Hopper to fix GitHub Actions timeout + - Using the `all` option for `cudaarch` on CUDA 12.4-12.6 causes the compilation process to exceed the 6-hour maximum execution limit on GitHub Actions, leading to cancelled jobs. + + - To resolve this and reduce build times, the target architectures are now restricted to explicitly support compute capabilities 7.0 through 9.0 (`70-real` to `90-real`). This maintains support for all modern NVIDIA GPUs equipped with Tensor Cores (from Volta up to Hopper architectures) while keeping the build time safely within CI constraints. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/9db77a020c97ac3b13b7c1bf4e0c5787001533e7](https://github.com/ggml-org/llama.cpp/commit/9db77a020c97ac3b13b7c1bf4e0c5787001533e7) + +- feat: Sync llama.cpp llama/mtmd API Binding 20260415 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/e1ade17c6330e3cc46a2b08f9b48b1540521b231...7820677e65827b6f3356f651da9be8d510ba10e5 + ## [0.3.35] Gemma 4 series & LFM 2.5-VL Support, OpenAI OpenAPI Alignment and Logging Architecture Migration - fix: expand stop sequences for `Gemma4ChatHandler` diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index fb263e7825..a02ec5af51 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.35" +__version__ = "0.3.36" From 7a19575ec579901c9718ec0aa77cb656bba9c47c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 18 Apr 2026 05:04:07 +0800 Subject: [PATCH 029/304] Update Submodule vendor/llama.cpp 9db77a0..45cac7c --- llama_cpp/llama_cpp.py | 20 ++++++++++---------- llama_cpp/mtmd_cpp.py | 38 ++++++++++++++++++++++++++++++-------- vendor/llama.cpp | 2 +- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a527904637..7e2e32b4a0 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1014,7 +1014,7 @@ class llama_model_imatrix_data(ctypes.Structure): if TYPE_CHECKING: name: ctypes.c_char_p - data: ctypes.POINTER(ctypes.c_float) + data: ctypes.POINTER(ctypes.c_float) # type: ignore size: ctypes.c_size_t llama_model_imatrix_data_p = ctypes.POINTER(llama_model_imatrix_data) @@ -1068,10 +1068,10 @@ class llama_model_quantize_params(ctypes.Structure): pure: bool keep_split: bool dry_run: bool - imatrix: ctypes.POINTER(llama_model_imatrix_data) - kv_overrides: ctypes.POINTER(llama_model_kv_override) - tensor_types: ctypes.POINTER(llama_model_tensor_override) - prune_layers: ctypes.POINTER(ctypes.c_int32) + imatrix: ctypes.POINTER(llama_model_imatrix_data) # type: ignore + kv_overrides: ctypes.POINTER(llama_model_kv_override) # type: ignore + tensor_types: ctypes.POINTER(llama_model_tensor_override) # type: ignore + prune_layers: ctypes.POINTER(ctypes.c_int32) # type: ignore _fields_ = [ ("nthread", ctypes.c_int32), @@ -1345,7 +1345,7 @@ def llama_numa_init(numa: int, /): ) def llama_model_init_from_user( metadata: ctypes.c_void_p, - set_tensor_data: llama_model_set_tensor_data_t, + set_tensor_data: llama_model_set_tensor_data_t, # type: ignore set_tensor_data_ud: ctypes.c_void_p, params: llama_model_params, / @@ -4548,7 +4548,7 @@ def llama_sampler_init_grammar_lazy_patterns( vocab: llama_vocab_p, grammar_str: bytes, grammar_root: bytes, - trigger_patterns: CtypesArray[bytes], + trigger_patterns: CtypesArray[bytes], # type: ignore num_trigger_patterns: int, trigger_tokens: CtypesArray[llama_token], num_trigger_tokens: int, @@ -4803,8 +4803,8 @@ def llama_print_system_info() -> bytes: None, ) def llama_log_get( - log_callback: Optional[ctypes.pointer(ggml_log_callback)], - user_data: ctypes.pointer(ctypes.c_void_p), + log_callback: Optional[ctypes.pointer(ggml_log_callback)], # type: ignore + user_data: ctypes.pointer(ctypes.c_void_p), # type: ignore /, ): """Get callback for all future logging events. @@ -4819,7 +4819,7 @@ def llama_log_get( None, ) def llama_log_set( - log_callback: Optional[ggml_log_callback], + log_callback: Optional[ggml_log_callback], # type: ignore user_data: ctypes.c_void_p, /, ): diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index ba64b21ef6..57f8414b80 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -345,7 +345,7 @@ def mtmd_bitmap_init( ) def mtmd_bitmap_init_from_audio( n_samples: c_uint, - data: POINTER(c_float), + data: POINTER(c_float), # type: ignore /, ) -> mtmd_bitmap_p: ... @@ -582,6 +582,9 @@ class mtmd_decoder_pos(Structure): x: c_uint32 y: c_uint32 +mtmd_decoder_pos_p = POINTER(mtmd_decoder_pos) +mtmd_decoder_pos_p_ctypes = c_void_p + # // get position for decoder attention, to be used by M-RoPE models # // i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 # // return relative position (for example, embedding 0 will have position (0, 0, 0); @@ -633,7 +636,7 @@ def mtmd_tokenize( ctx: mtmd_context_p, output: mtmd_input_chunks_p, text: mtmd_input_text_p, - bitmaps: POINTER(mtmd_bitmap_p), + bitmaps: POINTER(mtmd_bitmap_p), # type: ignore n_bitmaps: c_uint, /, ) -> c_int32: @@ -691,7 +694,7 @@ def mtmd_encode_chunk( # MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx); @ctypes_function_mtmd( "mtmd_get_output_embd", [mtmd_context_p_ctypes], POINTER(c_float)) -def mtmd_get_output_embd(ctx: mtmd_context_p) -> POINTER(c_float): +def mtmd_get_output_embd(ctx: mtmd_context_p) -> POINTER(c_float): # type: ignore """ get output embeddings from the last encode pass """ @@ -703,7 +706,7 @@ def mtmd_get_output_embd(ctx: mtmd_context_p) -> POINTER(c_float): # MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function_mtmd( "mtmd_log_set", [ggml_log_callback, c_void_p], None) -def mtmd_log_set(log_callback: ggml_log_callback, user_data: c_void_p): +def mtmd_log_set(log_callback: ggml_log_callback, user_data: c_void_p): # type: ignore """ Set callback for all future logging events. """ @@ -735,7 +738,7 @@ def mtmd_test_create_input_chunks() -> mtmd_input_chunk_p: # MTMD_API void mtmd_helper_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function_mtmd( "mtmd_helper_log_set", [ggml_log_callback, c_void_p], None) -def mtmd_helper_log_set(log_callback: ggml_log_callback, user_data: c_void_p): +def mtmd_helper_log_set(log_callback: ggml_log_callback, user_data: c_void_p): # type: ignore """ Set callback for all future logging events. """ @@ -810,6 +813,25 @@ def mtmd_helper_get_n_pos(chunks: mtmd_input_chunk_p) -> c_int32: ... +# // helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE +# // out_pos must have length == mtmd_helper_get_n_tokens(image) +# MTMD_API void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * image, struct mtmd_decoder_pos * out_pos); +@ctypes_function_mtmd("mtmd_helper_image_get_decoder_pos", [ + mtmd_image_tokens_p_ctypes, + mtmd_decoder_pos_p_ctypes + ], + None) +def mtmd_helper_image_get_decoder_pos( + image: mtmd_image_tokens_p, + out_pos: mtmd_decoder_pos_p # type: ignore +) -> c_int32: + """ + helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE + out_pos must have length == mtmd_helper_get_n_tokens(image) + """ + ... + + # // helper function that automatically: # // 1. run llama_decode() on text chunks # // 2. run mtmd_encode() on image chunks, then mtmd_get_output_embd() and then llama_decode() @@ -844,7 +866,7 @@ def mtmd_helper_eval_chunks( seq_id: c_int32, n_batch: c_int32, logits_last: c_bool, - new_n_past: POINTER(c_int32), + new_n_past: POINTER(c_int32), # type: ignore /, ) -> c_int32: """ @@ -887,7 +909,7 @@ def mtmd_helper_eval_chunk_single( seq_id: c_int32, n_batch: c_int32, logits_last: c_bool, - new_n_past: POINTER(c_int32), + new_n_past: POINTER(c_int32), # type: ignore /, ) -> c_int32: """ @@ -923,7 +945,7 @@ def mtmd_helper_decode_image_chunk( ctx: mtmd_context_p, lctx: llama_cpp.llama_context_p, chunks: mtmd_input_chunk_p, - encoded_embd: POINTER(c_float), + encoded_embd: POINTER(c_float), # type: ignore n_past: c_int32, seq_id: c_int32, n_batch: c_int32, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 9db77a020c..45cac7ca70 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 9db77a020c97ac3b13b7c1bf4e0c5787001533e7 +Subproject commit 45cac7ca703fb9085eae62b9121fca01d20177f6 From 33ce052ec868c1a301ced325ed9c5afbe041fb95 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 18 Apr 2026 06:05:03 +0800 Subject: [PATCH 030/304] chore(ci): upgrade astral-sh/setup-uv@v7 and Jimver/cuda-toolkit@v0.2.35 (Node 24 runtime) --- .github/workflows/build-wheels-cu124-linux.yml | 2 +- .github/workflows/build-wheels-cu124-win.yml | 4 ++-- .github/workflows/build-wheels-cu126-linux.yml | 2 +- .github/workflows/build-wheels-cu126-win.yml | 4 ++-- .github/workflows/build-wheels-cu128-linux.yml | 2 +- .github/workflows/build-wheels-cu128-win.yml | 4 ++-- .github/workflows/build-wheels-cu130-linux.yml | 2 +- .github/workflows/build-wheels-cu130-win.yml | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-wheels-cu124-linux.yml b/.github/workflows/build-wheels-cu124-linux.yml index 42b3d13169..889a1679a4 100644 --- a/.github/workflows/build-wheels-cu124-linux.yml +++ b/.github/workflows/build-wheels-cu124-linux.yml @@ -40,7 +40,7 @@ jobs: # from astral-sh/setup-uv - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true diff --git a/.github/workflows/build-wheels-cu124-win.yml b/.github/workflows/build-wheels-cu124-win.yml index 135b847d32..01bd48e7de 100644 --- a/.github/workflows/build-wheels-cu124-win.yml +++ b/.github/workflows/build-wheels-cu124-win.yml @@ -42,7 +42,7 @@ jobs: # from kingbri1/flash-attention build-wheels.yml - name: Install CUDA ${{ matrix.cuda }} - uses: N-Storm/cuda-toolkit@v0.2.28 + uses: Jimver/cuda-toolkit@v0.2.35 id: cuda-toolkit with: cuda: "${{ matrix.cuda }}" @@ -50,7 +50,7 @@ jobs: # from astral-sh/setup-uv - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true diff --git a/.github/workflows/build-wheels-cu126-linux.yml b/.github/workflows/build-wheels-cu126-linux.yml index f60eb5f878..568824c642 100644 --- a/.github/workflows/build-wheels-cu126-linux.yml +++ b/.github/workflows/build-wheels-cu126-linux.yml @@ -40,7 +40,7 @@ jobs: # from astral-sh/setup-uv - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true diff --git a/.github/workflows/build-wheels-cu126-win.yml b/.github/workflows/build-wheels-cu126-win.yml index be7bfdc72c..9330cb130b 100644 --- a/.github/workflows/build-wheels-cu126-win.yml +++ b/.github/workflows/build-wheels-cu126-win.yml @@ -42,7 +42,7 @@ jobs: # from kingbri1/flash-attention build-wheels.yml - name: Install CUDA ${{ matrix.cuda }} - uses: N-Storm/cuda-toolkit@v0.2.28 + uses: Jimver/cuda-toolkit@v0.2.35 id: cuda-toolkit with: cuda: "${{ matrix.cuda }}" @@ -50,7 +50,7 @@ jobs: # from astral-sh/setup-uv - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true diff --git a/.github/workflows/build-wheels-cu128-linux.yml b/.github/workflows/build-wheels-cu128-linux.yml index 0bfe971eea..d1c387c52a 100644 --- a/.github/workflows/build-wheels-cu128-linux.yml +++ b/.github/workflows/build-wheels-cu128-linux.yml @@ -40,7 +40,7 @@ jobs: # from astral-sh/setup-uv - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true diff --git a/.github/workflows/build-wheels-cu128-win.yml b/.github/workflows/build-wheels-cu128-win.yml index 80dd9f2f74..98ebbc4127 100644 --- a/.github/workflows/build-wheels-cu128-win.yml +++ b/.github/workflows/build-wheels-cu128-win.yml @@ -42,7 +42,7 @@ jobs: # from kingbri1/flash-attention build-wheels.yml - name: Install CUDA ${{ matrix.cuda }} - uses: N-Storm/cuda-toolkit@v0.2.28 + uses: Jimver/cuda-toolkit@v0.2.35 id: cuda-toolkit with: cuda: "${{ matrix.cuda }}" @@ -50,7 +50,7 @@ jobs: # from astral-sh/setup-uv - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true diff --git a/.github/workflows/build-wheels-cu130-linux.yml b/.github/workflows/build-wheels-cu130-linux.yml index 23cd668c8d..4f4305ad3e 100644 --- a/.github/workflows/build-wheels-cu130-linux.yml +++ b/.github/workflows/build-wheels-cu130-linux.yml @@ -40,7 +40,7 @@ jobs: # from astral-sh/setup-uv - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true diff --git a/.github/workflows/build-wheels-cu130-win.yml b/.github/workflows/build-wheels-cu130-win.yml index b995f4f5f4..d6187d7bf4 100644 --- a/.github/workflows/build-wheels-cu130-win.yml +++ b/.github/workflows/build-wheels-cu130-win.yml @@ -42,7 +42,7 @@ jobs: # from kingbri1/flash-attention build-wheels.yml - name: Install CUDA ${{ matrix.cuda }} - uses: N-Storm/cuda-toolkit@v0.2.29 + uses: Jimver/cuda-toolkit@v0.2.35 id: cuda-toolkit with: cuda: "${{ matrix.cuda }}" @@ -50,7 +50,7 @@ jobs: # from astral-sh/setup-uv - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true From 3984ab5c81a31f4fec7c3c0366f7ffa39c957e1e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 19 Apr 2026 15:34:42 +0800 Subject: [PATCH 031/304] Update Submodule vendor/llama.cpp 45cac7c..037bfe3 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 45cac7ca70..037bfe38d0 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 45cac7ca703fb9085eae62b9121fca01d20177f6 +Subproject commit 037bfe38d0297001869df87150286952ae94cb1c From 15f8a36f00dd24dd92a8290b2a06f35fe51bcad4 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 19 Apr 2026 16:07:05 +0800 Subject: [PATCH 032/304] docs: initialize LLM Wiki structure for better documentation maintenance - Create docs/wiki/ directory with full folder structure - Add SCHEMA.md, index.md and contributing guidelines - Set up core/, features/, examples/, types/ and subdirectories - Prepare for LLM-powered living documentation (Llama class, multi-modal chat handlers, vision/audio examples, etc.) - Include .gitkeep files to preserve empty directories This lays the foundation for a modern, maintainable wiki that will replace outdated static docs. Future commits will populate pages with up-to-date content generated from latest source code. Signed-off-by: JamePeng --- docs/wiki/.gitkeep | 0 docs/wiki/SCHEMA.md | 45 +++++++++++++++++++++ docs/wiki/contributing-to-wiki.md | 0 docs/wiki/core/.gitkeep | 0 docs/wiki/core/ChatHandler.md | 0 docs/wiki/core/Llama.md | 0 docs/wiki/core/LlamaChatFormat.md | 0 docs/wiki/core/LlamaCppBindings.md | 0 docs/wiki/core/MTMDCppBindings.md | 0 docs/wiki/development/.gitkeep | 0 docs/wiki/examples/.gitkeep | 0 docs/wiki/examples/audio/.gitkeep | 0 docs/wiki/examples/audio/audio-gemma.md | 0 docs/wiki/examples/audio/audio-qwen-omni.md | 0 docs/wiki/examples/basic-completion.md | 0 docs/wiki/examples/chat-completion.md | 0 docs/wiki/examples/speculative-decoding.md | 0 docs/wiki/examples/vision/.gitkeep | 0 docs/wiki/examples/vision/video/.gitkeep | 0 docs/wiki/examples/vision/vision-gemma.md | 0 docs/wiki/examples/vision/vision-glmv.md | 0 docs/wiki/examples/vision/vision-ocr.md | 0 docs/wiki/examples/vision/vision-qwen.md | 0 docs/wiki/features/.gitkeep | 0 docs/wiki/features/caching.md | 0 docs/wiki/features/embeddings-rerank.md | 0 docs/wiki/features/grammar.md | 0 docs/wiki/features/multi-model.md | 0 docs/wiki/features/tool-calls.md | 0 docs/wiki/index.md | 0 docs/wiki/install.md | 0 docs/wiki/troubleshooting.md | 0 docs/wiki/types/.gitkeep | 0 docs/wiki/types/common-types.md | 0 docs/wiki/types/mcp-types.md | 0 35 files changed, 45 insertions(+) create mode 100644 docs/wiki/.gitkeep create mode 100644 docs/wiki/SCHEMA.md create mode 100644 docs/wiki/contributing-to-wiki.md create mode 100644 docs/wiki/core/.gitkeep create mode 100644 docs/wiki/core/ChatHandler.md create mode 100644 docs/wiki/core/Llama.md create mode 100644 docs/wiki/core/LlamaChatFormat.md create mode 100644 docs/wiki/core/LlamaCppBindings.md create mode 100644 docs/wiki/core/MTMDCppBindings.md create mode 100644 docs/wiki/development/.gitkeep create mode 100644 docs/wiki/examples/.gitkeep create mode 100644 docs/wiki/examples/audio/.gitkeep create mode 100644 docs/wiki/examples/audio/audio-gemma.md create mode 100644 docs/wiki/examples/audio/audio-qwen-omni.md create mode 100644 docs/wiki/examples/basic-completion.md create mode 100644 docs/wiki/examples/chat-completion.md create mode 100644 docs/wiki/examples/speculative-decoding.md create mode 100644 docs/wiki/examples/vision/.gitkeep create mode 100644 docs/wiki/examples/vision/video/.gitkeep create mode 100644 docs/wiki/examples/vision/vision-gemma.md create mode 100644 docs/wiki/examples/vision/vision-glmv.md create mode 100644 docs/wiki/examples/vision/vision-ocr.md create mode 100644 docs/wiki/examples/vision/vision-qwen.md create mode 100644 docs/wiki/features/.gitkeep create mode 100644 docs/wiki/features/caching.md create mode 100644 docs/wiki/features/embeddings-rerank.md create mode 100644 docs/wiki/features/grammar.md create mode 100644 docs/wiki/features/multi-model.md create mode 100644 docs/wiki/features/tool-calls.md create mode 100644 docs/wiki/index.md create mode 100644 docs/wiki/install.md create mode 100644 docs/wiki/troubleshooting.md create mode 100644 docs/wiki/types/.gitkeep create mode 100644 docs/wiki/types/common-types.md create mode 100644 docs/wiki/types/mcp-types.md diff --git a/docs/wiki/.gitkeep b/docs/wiki/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/SCHEMA.md b/docs/wiki/SCHEMA.md new file mode 100644 index 0000000000..4a8f700e2c --- /dev/null +++ b/docs/wiki/SCHEMA.md @@ -0,0 +1,45 @@ +# LLM Wiki Schema ā€“ llama-cpp-python + +**Purpose**: Maintain a living, always-up-to-date, structured documentation wiki for the llama-cpp-python library using LLMs as the primary maintainer. + +**Core Principles**: +- The source of truth is the latest code in `llama_cpp/` (especially `llama.py`, `llama_chat_format.py`, `llama_cpp.py`, `llama_types.py`, `mtmd_cpp.py`, `_internals.py`, `_ggml.py`). +- Never invent parameters or behavior. Always read the current source code before writing/updating a page. +- All examples must be complete, runnable with the latest API, and include necessary imports. +- Clearly mark any deprecated/old usage with a warning and show the modern replacement. +- Use internal wiki links (e.g. [[Llama]], [[Qwen35ChatHandler]]) for cross-referencing. +- Keep pages concise, professional, and user-friendly. + +**Page Types and Templates**: + +1. **Class / Module Page** (e.g. core/Llama.md) + - Frontmatter (YAML): + ```yaml + --- + title: Llama Class + class_name: Llama + last_updated: YYYY-MM-DD + version_target: "latest" + --- + ``` + - Sections (in order): + - Overview + - Constructor (`__init__`) ā€“ full parameter table with types, defaults, and explanations + - Core Methods (with signatures and examples) + - Best Practices & Common Patterns + - Deprecated / Changed APIs (with migration notes) + - Related Links + +2. **Feature Page** (features/xxx.md) + - Overview, When to use, Code examples, Limitations, Related features + +3. **Example Page** (examples/xxx.md) + - Goal, Prerequisites, Complete runnable code block, Expected output, Tips + +**Update Rules**: +- Before updating any page, the LLM must read the relevant source files. +- Update the `last_updated` date. +- If a new feature (e.g. new ChatHandler, new sampler) appears in code, create or expand the corresponding page. +- Maintain a high standard of readability and accuracy. + +This schema is the contract. All generated content must follow it. \ No newline at end of file diff --git a/docs/wiki/contributing-to-wiki.md b/docs/wiki/contributing-to-wiki.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/core/.gitkeep b/docs/wiki/core/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/core/ChatHandler.md b/docs/wiki/core/ChatHandler.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/core/LlamaChatFormat.md b/docs/wiki/core/LlamaChatFormat.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/core/LlamaCppBindings.md b/docs/wiki/core/LlamaCppBindings.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/core/MTMDCppBindings.md b/docs/wiki/core/MTMDCppBindings.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/development/.gitkeep b/docs/wiki/development/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/.gitkeep b/docs/wiki/examples/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/audio/.gitkeep b/docs/wiki/examples/audio/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/audio/audio-gemma.md b/docs/wiki/examples/audio/audio-gemma.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/audio/audio-qwen-omni.md b/docs/wiki/examples/audio/audio-qwen-omni.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/basic-completion.md b/docs/wiki/examples/basic-completion.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/chat-completion.md b/docs/wiki/examples/chat-completion.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/speculative-decoding.md b/docs/wiki/examples/speculative-decoding.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/vision/.gitkeep b/docs/wiki/examples/vision/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/vision/video/.gitkeep b/docs/wiki/examples/vision/video/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/vision/vision-gemma.md b/docs/wiki/examples/vision/vision-gemma.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/vision/vision-glmv.md b/docs/wiki/examples/vision/vision-glmv.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/vision/vision-ocr.md b/docs/wiki/examples/vision/vision-ocr.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/examples/vision/vision-qwen.md b/docs/wiki/examples/vision/vision-qwen.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/features/.gitkeep b/docs/wiki/features/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/features/caching.md b/docs/wiki/features/caching.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/features/embeddings-rerank.md b/docs/wiki/features/embeddings-rerank.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/features/grammar.md b/docs/wiki/features/grammar.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/features/multi-model.md b/docs/wiki/features/multi-model.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/features/tool-calls.md b/docs/wiki/features/tool-calls.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/index.md b/docs/wiki/index.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/install.md b/docs/wiki/install.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/troubleshooting.md b/docs/wiki/troubleshooting.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/types/.gitkeep b/docs/wiki/types/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/types/common-types.md b/docs/wiki/types/common-types.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/wiki/types/mcp-types.md b/docs/wiki/types/mcp-types.md new file mode 100644 index 0000000000..e69de29bb2 From e3b7ad67901b0091662dc05079526c79d4f6ee6f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 20 Apr 2026 20:28:17 +0800 Subject: [PATCH 033/304] Update Submodule vendor/llama.cpp 037bfe3..81df3f7 --- llama_cpp/mtmd_cpp.py | 18 ++++++++++-------- vendor/llama.cpp | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 57f8414b80..4dd6d6d05f 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -587,17 +587,17 @@ class mtmd_decoder_pos(Structure): # // get position for decoder attention, to be used by M-RoPE models # // i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 -# // return relative position (for example, embedding 0 will have position (0, 0, 0); -# // remember to adjust it to the current absolute position) -# MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, size_t i); +# // pos_0 is the absolute position of the first token +# // return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position) +# MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i); @ctypes_function_mtmd( - "mtmd_image_tokens_get_decoder_pos", [mtmd_image_tokens_p_ctypes, c_size_t], mtmd_decoder_pos) -def mtmd_image_tokens_get_decoder_pos(image_tokens: mtmd_image_tokens_p, i: c_size_t) -> mtmd_decoder_pos: + "mtmd_image_tokens_get_decoder_pos", [mtmd_image_tokens_p_ctypes, c_int32, c_size_t], mtmd_decoder_pos) +def mtmd_image_tokens_get_decoder_pos(image_tokens: mtmd_image_tokens_p, pos_0: c_int32, i: c_size_t) -> mtmd_decoder_pos: """ get position for decoder attention, to be used by M-RoPE models i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 - return relative position (for example, embedding 0 will have position (0, 0, 0); - remember to adjust it to the current absolute position) + pos_0 is the absolute position of the first token + return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position) """ ... @@ -815,14 +815,16 @@ def mtmd_helper_get_n_pos(chunks: mtmd_input_chunk_p) -> c_int32: # // helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE # // out_pos must have length == mtmd_helper_get_n_tokens(image) -# MTMD_API void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * image, struct mtmd_decoder_pos * out_pos); +# MTMD_API void mtmd_helper_image_get_decoder_pos(const mtmd_image_tokens * image, llama_pos pos_0, struct mtmd_decoder_pos * out_pos); @ctypes_function_mtmd("mtmd_helper_image_get_decoder_pos", [ mtmd_image_tokens_p_ctypes, + c_int32, mtmd_decoder_pos_p_ctypes ], None) def mtmd_helper_image_get_decoder_pos( image: mtmd_image_tokens_p, + pos_0: c_int32, out_pos: mtmd_decoder_pos_p # type: ignore ) -> c_int32: """ diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 037bfe38d0..81df3f7cfa 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 037bfe38d0297001869df87150286952ae94cb1c +Subproject commit 81df3f7cfaa6f99de14e792b38d5771bf427383e From 8625836b703cfbd17f38105be5561e0147728a05 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 21 Apr 2026 22:07:43 +0800 Subject: [PATCH 034/304] Update Submodule vendor/llama.cpp 81df3f7..82209ef --- llama_cpp/_internals.py | 4 --- llama_cpp/llama_cpp.py | 58 ----------------------------------------- llama_cpp/mtmd_cpp.py | 26 ++++++++++++------ vendor/llama.cpp | 2 +- 4 files changed, 19 insertions(+), 71 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 20648cf74d..27dd4f80d3 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -758,10 +758,6 @@ def reset_timings(self): def print_timings(self): llama_cpp.llama_perf_context_print(self.ctx) - def print_memory_breakdown(self): - """print a breakdown of per-device memory use via LLAMA_LOG""" - llama_cpp.llama_memory_breakdown_print(self.ctx) - # LoRA / ALoRA Dynamic Routing Methods def clear_loras(self): diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 7e2e32b4a0..416e8b9357 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1519,53 +1519,6 @@ class llama_params_fit_status(enum.IntEnum): LLAMA_PARAMS_FIT_STATUS_ERROR = 2 -# // fits mparams and cparams to free device memory (assumes system memory is unlimited) -# // - returns true if the parameters could be successfully modified to fit device memory -# // - this function is NOT thread safe because it modifies the global llama logger state -# // - only parameters that have the same value as in llama_default_model_params are modified -# // with the exception of the context size which is modified if and only if equal to 0 -# LLAMA_API enum llama_params_fit_status llama_params_fit( -# const char * path_model, -# struct llama_model_params * mparams, -# struct llama_context_params * cparams, -# float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements -# struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements -# size_t margin, // margin of memory to leave per device in bytes -# uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use -# enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log -@ctypes_function( - "llama_params_fit", - [ - ctypes.c_char_p, - llama_model_params_p, - llama_context_params_p, - ctypes.POINTER(ctypes.c_float), - ctypes.POINTER(llama_model_tensor_buft_override), - ctypes.c_size_t, - ctypes.c_uint32, - ctypes.c_int, - ], - ctypes.c_int, -) -def llama_params_fit( - path_model: ctypes.c_char_p, - mparams: CtypesPointer[llama_model_params], - cparams: CtypesPointer[llama_context_params], - tensor_split: CtypesPointer[ctypes.c_float], - tensor_buft_overrides: CtypesPointer[llama_model_tensor_buft_override], - margin: ctypes.c_size_t, - n_ctx_min: ctypes.c_uint32, - log_level: int, - /, -) -> int: - """ - fits mparams and cparams to free device memory (assumes system memory is unlimited) - returns true if the parameters could be successfully modified to fit device memory - this function is NOT thread safe because it modifies the global llama logger state - """ - ... - - # LLAMA_API int64_t llama_time_us(void); @ctypes_function( "llama_time_us", @@ -4930,17 +4883,6 @@ def llama_perf_sampler_reset(chain: llama_sampler_p, /): ... -# // print a breakdown of per-device memory use via LLAMA_LOG: -# LLAMA_API void llama_memory_breakdown_print(const struct llama_context * ctx); -@ctypes_function( - "llama_memory_breakdown_print", - [llama_context_p_ctypes], - None, -) -def llama_memory_breakdown_print(ctx: llama_context_p, /): - ... - - # // # // training # // diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 4dd6d6d05f..574d90e2bf 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -94,10 +94,19 @@ class mtmd_input_chunk_type(enum.IntEnum): mtmd_bitmap_p = NewType("mtmd_bitmap_p", int) mtmd_bitmap_p_ctypes = c_void_p +# // position indexing for decoder model +# enum mtmd_pos_type { +# MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens +# MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes +# }; +class mtmd_pos_type(enum.IntEnum): + MTMD_POS_TYPE_NORMAL = 0 # number of positions equals to number of tokens + MTMD_POS_TYPE_MROPE = 1 # qwen-vl mrope style, each image takes max(t,h,w) position indexes + # struct mtmd_image_tokens { # uint32_t nx; // number of tokens in x direction # uint32_t ny; // number of tokens in y direction -# bool use_mrope_pos = false; // use M-RoPE position counting (the whole image is 1 temporal position) +# mtmd_pos_type pos = MTMD_POS_TYPE_NORMAL; # uint32_t n_tokens() const { return nx * ny; } # clip_image_f32_batch batch_f32; // preprocessed image patches # std::string id; // optional user-defined ID, useful for KV cache tracking @@ -269,28 +278,29 @@ def mtmd_free(ctx: mtmd_context_p): ... # // whether we need to set non-causal mask before llama_decode -# MTMD_API bool mtmd_decode_use_non_causal(mtmd_context * ctx); +# // if chunk is nullptr, we assume the default case where chunk is an image chunk +# MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk); @ctypes_function_mtmd( - "mtmd_decode_use_non_causal", [mtmd_context_p_ctypes], c_bool) -def mtmd_decode_use_non_causal(ctx: mtmd_context_p) -> c_bool: + "mtmd_decode_use_non_causal", [mtmd_context_p_ctypes, mtmd_input_chunk_p_ctypes], c_bool) +def mtmd_decode_use_non_causal(ctx: mtmd_context_p, chunk: mtmd_input_chunk_p) -> c_bool: ... # // whether the current model use M-RoPE for llama_decode -# MTMD_API bool mtmd_decode_use_mrope(mtmd_context * ctx); +# MTMD_API bool mtmd_decode_use_mrope(const mtmd_context * ctx); @ctypes_function_mtmd( "mtmd_decode_use_mrope", [mtmd_context_p_ctypes], c_bool) def mtmd_decode_use_mrope(ctx: mtmd_context_p) -> c_bool: ... # // whether the current model supports vision input -# MTMD_API bool mtmd_support_vision(mtmd_context * ctx); +# MTMD_API bool mtmd_support_vision(const mtmd_context * ctx); @ctypes_function_mtmd( "mtmd_support_vision", [mtmd_context_p_ctypes], c_bool) def mtmd_support_vision(ctx: mtmd_context_p) -> c_bool: ... # // whether the current model supports audio input -# MTMD_API bool mtmd_support_audio(mtmd_context * ctx); +# MTMD_API bool mtmd_support_audio(const mtmd_context * ctx); @ctypes_function_mtmd( "mtmd_support_audio", [mtmd_context_p_ctypes], c_bool) def mtmd_support_audio(ctx: mtmd_context_p) -> c_bool: @@ -298,7 +308,7 @@ def mtmd_support_audio(ctx: mtmd_context_p) -> c_bool: # // get audio sample rate in Hz, for example 16000 for Whisper # // return -1 if audio is not supported -# MTMD_API int mtmd_get_audio_sample_rate(mtmd_context * ctx); +# MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx); @ctypes_function_mtmd( "mtmd_get_audio_sample_rate", [mtmd_context_p_ctypes], c_int) def mtmd_get_audio_sample_rate(ctx: mtmd_context_p) -> c_int: diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 81df3f7cfa..82209efb7e 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 81df3f7cfaa6f99de14e792b38d5771bf427383e +Subproject commit 82209efb7eab9a741923897c74fbb8fd71cd17ba From dbb740780b03abe7fdabf7d4656f54b36132c765 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 23 Apr 2026 00:49:26 +0800 Subject: [PATCH 035/304] Update Submodule vendor/llama.cpp 82209ef..8bccdbb --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 82209efb7e..8bccdbbff9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 82209efb7eab9a741923897c74fbb8fd71cd17ba +Subproject commit 8bccdbbff9d0d91d54838471f6eea182b9ab1b79 From afd038f96c59419f9c720450ed28c31e1c171682 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 23 Apr 2026 00:58:31 +0800 Subject: [PATCH 036/304] feat(types): introduce MCP definitions and align with latest OpenAI spec - Add comprehensive Model Context Protocol (MCP) type definitions, including `MCPTool`, `MCPToolCall`, `MCPListTools`, connector IDs, and approval filters to support remote server tool calling. - Add `ServiceTier` literal ("auto", "default", etc.) and include the `service_tier` field in `CreateChatCompletionResponse`. - Restrict `finish_reason` in completion responses to strict standard literals (`stop`, `length`, `tool_calls`, `content_filter`, `function_call`). - Introduce `ChatCompletionMessageCustomToolCall` to support custom tool calls generated by the model. - Update `ChatCompletionRequestAssistantMessage` to include the `name` field and add descriptive docstrings to message types. Signed-off-by: JamePeng --- llama_cpp/llama_types.py | 113 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 60202ae8f0..37b041ee87 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -144,7 +144,10 @@ class ChatCompletionResponseChoice(TypedDict): index: int message: "ChatCompletionResponseMessage" logprobs: Optional[ChatCompletionLogprobs] - finish_reason: Optional[str] + finish_reason: Optional[Literal["stop", "length", "tool_calls", "content_filter", "function_call"]] + + +ServiceTier = Literal["auto", "default", "flex", "scale", "priority"] class CreateChatCompletionResponse(TypedDict): @@ -152,6 +155,7 @@ class CreateChatCompletionResponse(TypedDict): object: Literal["chat.completion"] created: int model: str + service_tier: NotRequired[ServiceTier] choices: List["ChatCompletionResponseChoice"] usage: CompletionUsage @@ -300,27 +304,130 @@ class ChatCompletionRequestUserMessage(TypedDict): content: Optional[Union[str, List[ChatCompletionRequestMessageContentPart]]] +# Function tool call + class ChatCompletionMessageToolCallFunction(TypedDict): + """The function that the model called.""" name: str arguments: str class ChatCompletionMessageToolCall(TypedDict): + """A call to a function tool created by the model.""" id: str type: Literal["function"] function: ChatCompletionMessageToolCallFunction +# Custom tool call -ChatCompletionMessageToolCalls = List[ChatCompletionMessageToolCall] +class ChatCompletionMessageCustomToolCallCustom(TypedDict): + """The custom tool that the model called.""" + name: str + input: str +class ChatCompletionMessageCustomToolCall(TypedDict): + """A call to a custom tool created by the model.""" + id: str + type: Literal["custom"] + custom: ChatCompletionMessageCustomToolCallCustom -class ChatCompletionRequestAssistantMessageFunctionCall(TypedDict): +# The tool calls generated by the model, such as function calls. +ChatCompletionMessageToolCalls = Union[ + ChatCompletionMessageToolCall, + ChatCompletionMessageCustomToolCall +] + + +# MCP ToolCall + +MCPConnectorID = Literal[ + "connector_dropbox", + "connector_gmail", + "connector_googlecalendar", + "connector_googledrive", + "connector_microsoftteams", + "connector_outlookcalendar", + "connector_outlookemail", + "connector_sharepoint" +] + +MCPToolCallStatus = Literal["in_progress", "completed", "incomplete", "calling", "failed"] + +class MCPToolCall(TypedDict): + """An invocation of a tool on an MCP server.""" + type: Literal["mcp_call"] + id: str + server_label: str name: str + arguments: str # JSON string + output: NotRequired[Optional[str]] + error: NotRequired[Optional[str]] + status: NotRequired[MCPToolCallStatus] + approval_request_id: NotRequired[Optional[str]] + + +class MCPListToolsTool(TypedDict): + """A tool available on an MCP server.""" + name: str + description: Optional[str] + input_schema: Dict[str, Any] # The JSON schema describing the tool's input + annotations: Optional[Dict[str, Any]] + + +class MCPListTools(TypedDict): + """A list of tools available on an MCP server.""" + type: Literal["mcp_list_tools"] + id: str + server_label: str + tools: List[MCPListToolsTool] + error: Optional[str] + + +class MCPToolFilter(TypedDict): + """A filter object to specify which tools are allowed.""" + tool_names: NotRequired[List[str]] + read_only: NotRequired[bool] + + +class MCPToolApprovalFilter(TypedDict, total=False): + """Specify which of the MCP server's tools require approval based on filters.""" + always: MCPToolFilter + never: MCPToolFilter + + +class MCPTool(TypedDict): + """ + Give the model access to additional tools via remote Model Context Protocol (MCP) servers. + """ + # The type of the MCP tool. Always `mcp`. + type: Literal["mcp"] + # A label for this MCP server, used to identify it in tool calls. + server_label: str + # The URL for the MCP server. One of `server_url` or `connector_id` must be provided. + server_url: NotRequired[str] + connector_id: NotRequired[MCPConnectorID] + authorization: NotRequired[str] + server_description: NotRequired[str] + headers: NotRequired[Optional[Dict[str, str]]] + # List of allowed tool names or a filter object. + allowed_tools: NotRequired[Optional[Union[List[str], MCPToolFilter]]] + # Specify which of the MCP server's tools require approval. + require_approval: NotRequired[Optional[Union[Literal["always", "never"], MCPToolApprovalFilter]]] + # Whether this MCP tool is deferred and discovered via tool search. + defer_loading: NotRequired[bool] + + +# Assistant message + +class ChatCompletionRequestAssistantMessageFunctionCall(TypedDict): arguments: str + name: str class ChatCompletionRequestAssistantMessage(TypedDict): + """Messages sent by the model in response to user messages.""" role: Literal["assistant"] + name: Optional[str] content: NotRequired[Optional[str]] refusal: NotRequired[Optional[str]] tool_calls: NotRequired[ChatCompletionMessageToolCalls] From 852a2e972cb1521ca374de4b60f9ca1dffd0962c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 23 Apr 2026 01:36:03 +0800 Subject: [PATCH 037/304] Update /docs/wiki/core/Llama.md Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 205 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 205 insertions(+) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index e69de29bb2..ce2f1ba5d7 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -0,0 +1,205 @@ +```yaml +--- +title: Llama Class +class_name: Llama +last_updated: 2026-04-23 +version_target: "latest" +--- +``` + +## Overview +The `Llama` class is the core, high-level Python wrapper for a `llama.cpp` model. It handles model loading, memory management (KV cache), tokenization, and generation (both base text completion and chat formatting). It includes advanced features like dynamic LoRA routing, hybrid model checkpointing, speculative decoding, and context shifting. + +## Constructor (`__init__`) + +Initialize the model and context. Note that model loading will immediately allocate RAM/VRAM based on the selected offloading parameters. + +### Core Model & Hardware Parameters +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `model_path` | `str` | **Required** | Path to the `.gguf` model file. | +| `n_gpu_layers` | `int` | `0` | Number of layers to offload to GPU. Set to `-1` for all layers. | +| `split_mode` | `int` | `LLAMA_SPLIT_MODE_LAYER` | How to split the model across GPUs (e.g., `LLAMA_SPLIT_MODE_ROW`). | +| `main_gpu` | `int` | `0` | The primary GPU to use for intermediate results or the entire model. | +| `tensor_split` | `List[float]` | `None` | Proportional split of tensors across GPUs (max `LLAMA_MAX_DEVICES`). | +| `use_mmap` | `bool` | `True` | Whether to use memory mapping (mmap) if possible. | +| `use_mlock` | `bool` | `False` | Force the system to keep the model in RAM, preventing swapping. | +| `kv_overrides` | `Dict` | `None` | Key-value overrides for the model metadata (supports bool, int, float, str). | +| `numa` | `Union[bool, int]`| `False` | NUMA strategy (e.g., `GGML_NUMA_STRATEGY_DISTRIBUTE`). | + +### Context & Performance Parameters +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `n_ctx` | `int` | `512` | Text context size. Set to `0` to load from model metadata. | +| `n_batch` | `int` | `2048` | Maximum batch size for prompt processing. | +| `n_ubatch` | `int` | `512` | Physical batch size. | +| `n_threads` | `int` | `None` | Number of threads for generation (defaults to CPU count // 2). | +| `n_threads_batch`| `int` | `None` | Number of threads for batch processing (defaults to CPU count). | +| `flash_attn_type`| `int` | `AUTO` | Controls Flash Attention activation (`LLAMA_FLASH_ATTN_TYPE_AUTO`). | +| `swa_full` | `bool` | `None` | Whether to use full-size SWA cache | +| `kv_unified` | `bool` | `None` | Use single unified KV buffer for the KV cache of all sequences | +| `type_k` / `type_v`| `int` | `None` | KV cache data type for K and V (defaults to `f16`). | +| `offload_kqv` | `bool` | `True` | Whether to offload K, Q, V tensors to GPU. | + +### Advanced & Chat Parameters +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `chat_format` | `str` | `None` | String specifying the chat template (e.g., `"llama-2"`, `"chatml"`). Guessed from GGUF if None. | +| `chat_handler` | `LlamaChatCompletionHandler` | `None` | Optional custom handler. See [[ChatHandlers]]. | +| `draft_model` | `LlamaDraftModel` | `None` | Optional draft model for speculative decoding. | +| `ctx_checkpoints` | `int` | `32` | Max context checkpoints per slot (Hybrid/SWA models). | +| `checkpoint_interval`| `int`| `4096` | Token interval for saving Hybrid model checkpoints. | + +*(Note: There are numerous additional RoPE/YaRN scaling parameters available for specialized context extension. Refer to the source code for the full list).* + +--- + +## Core Methods + +### `create_chat_completion` +Generates a chat response using the configured `chat_format` or `chat_handler`. +```python +import llama_cpp + +model = llama_cpp.Llama(model_path="models/qwen2.5-7b-instruct.gguf", n_gpu_layers=-1) + +response = model.create_chat_completion( + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Explain KV caching."} + ], + temperature=0.7, + max_tokens=2048 +) +print(response["choices"][0]["message"]["content"]) +``` + +### `create_completion` / `__call__` +Generates standard text completion from a raw string prompt. +```python +import llama_cpp + +model = llama_cpp.Llama(model_path="models/llama-3-8b.gguf") +output = model("The capital of Japan is", max_tokens=10, stop=["\n"]) +print(output["choices"][0]["text"]) +``` + +### `generate` +A low-level generator yielding token IDs one by one. Highly customizable with sampling parameters, dynamic LoRA mounting, and control vectors. +```python +import llama_cpp + +model = llama_cpp.Llama(model_path="models/llama-3-8b.gguf") +tokens = model.tokenize(b"def fibonacci(n):") + +for token in model.generate(tokens, top_k=40, top_p=0.95, temp=0.2): + print(model.detokenize([token]).decode('utf-8'), end="", flush=True) +``` + +### `eval` +Low-level method to ingest and evaluate a sequence of tokens. Used internally to update the KV cache and logits. Handles **Context Shifting** automatically to prevent OOM when the token count exceeds `n_ctx`. +```python +# Evaluates a chunk of tokens and updates internal state +model.eval(tokens=[1, 453, 234, 987], active_loras=[{"name": "coding_adapter", "scale": 1.0}]) +``` + +### Dynamic LoRA Management +The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dynamically per-generation or per-eval. +* `load_lora(name: str, path: str)`: Loads an adapter into VRAM (does not apply it yet). +* `unload_lora(name: str)`: Releases the specific LoRA from VRAM. +* `list_loras() -> List[str]`: Returns names of all registered LoRAs. +* `unload_all_loras()`: Forces VRAM release for all loaded adapters. + +--- + +## Best Practices & Common Patterns + +1. **Context Shifting & Prompt Caching**: + + By default, when calling `.generate()` or `.create_completion(reset=True)`, the engine checks for the longest matching prefix in the existing KV cache. To maximize speed, keep system prompts static and only append new dialogue to avoid re-evaluating the entire history. If the context limit is reached during `eval`, the model will automatically trigger a Context Shift (discarding older tokens while attempting to keep `n_keep` tokens, usually the system prompt). + +2. **Basic Chat with JSON Mode**: + Forces the model to output valid JSON by using the `response_format` parameter. + ```python + from llama_cpp import Llama + + llm = Llama(model_path="path/to/model.gguf", n_gpu_layers=-1) + + response = llm.create_chat_completion( + messages=[{"role": "user", "content": "Extract name and age from: John is 30."}], + response_format={"type": "json_object"}, + temperature=0.0 + ) + print(response["choices"][0]["message"]["content"]) + ``` + +3. **Speculative Decoding**: + + Accelerates generation by using a small "draft" model to predict tokens, which the larger model then validates in parallel. + ```python + from llama_cpp import Llama + from llama_cpp.llama_speculative import LlamaDraftModel + + draft = LlamaDraftModel.from_model(Llama(model_path="tiny_draft.gguf", n_gpu_layers=-1)) + main_llm = Llama(model_path="large_model.gguf", n_gpu_layers=-1, draft_model=draft) + + for chunk in main_llm.create_completion("Explain quantum physics", stream=True): + print(chunk["choices"][0]["text"], end="") + ``` + +4. **Dynamic LoRA Routing**: + + You can load multiple LoRAs using `load_lora()` at startup. Then, pass the `active_loras` parameter to `.generate()`, `.create_completion()`, or `.create_chat_completion()` to dynamically apply them to specific queries without reloading the base model. + + Multi-LoRA Dynamic Switching Example:
+ + Load multiple adapters and apply them selectively without reloading the base model. + ```python + llm = Llama(model_path="base_model.gguf") + llm.load_lora("coding", "codellama_adapter.gguf") + llm.load_lora("story", "storywriter_adapter.gguf") + llm.load_lora("sql_expert", "adapters/sql_lora.gguf") + + # Use coding adapter + llm.create_completion("def sort:", active_loras=[{"name": "coding", "scale": 1.0}]) + + # Use story adapter + llm.create_completion("Once upon a time", active_loras=[{"name": "story", "scale": 0.9}]) + + # Use sql adapter + llm.create_completion("SELECT *", active_loras=[{"name": "sql_expert", "scale": 0.8}])v + ``` + +5. **Hybrid & Recurrent Architectures**: + + The class natively detects Hybrid/Recurrent models (like LFM2VL/LFM2.5VL, Qwen3.5/3.6, Mamba or specialized SWA models(Gemma3/4)) and automatically enables the `HybridCheckpointCache`. This creates periodic save-states during large context pre-filling, allowing the model to roll back seamlessly if a generation is rejected (e.g., speculative decoding mismatches) without corrupting the recurrent state. + + * Tips: If you are using hybrid multimodal model for building ComfyUI nodes or running single-turn API wrappers where you do not need multi-turn state rollbacks, simply initialize your Llama instance with `ctx_checkpoints=0`: + + ```python + llm = Llama( + model_path="./Qwen3.5-VL-9B.gguf", + chat_handler=MTMDChatHandler(clip_model_path="./mmproj.gguf"), + n_ctx=4096, + ctx_checkpoints=0 # <-- SET THIS TO 0 TO ENABLE ZERO-LATENCY FAST PATH + ) + ``` + +--- + +## Deprecated / Changed APIs + +> āš ļø **Warning:** The internal embedding methods on the `Llama` class are deprecated and will be removed. + +* `embed()` āž” **Deprecated.** +* `create_embedding()` āž” **Deprecated.** + +**Migration Note:** Do not use `Llama(..., embeddings=True)` combined with `model.create_embedding(...)`. Instead, use the dedicated `LlamaEmbedding` class, which offers optimized batching and reranking support. +*See: [[LlamaEmbedding]]* + +--- + +## Related Links +* [[LlamaEmbedding]] - Dedicated class for text embeddings and reranking. +* [[ChatHandlers]] - Customizing `LlamaChatCompletionHandler` for function calling and vision/omni models (e.g., `[[Gemma4ChatHandler]]`, `[[Qwen35ChatHandler]]`). +* [[LlamaCache]] - Implementing disk or RAM-based prompt caching (LlamaRAMCache, **TrieCache**, **HybridCheckpointCache**). \ No newline at end of file From 7a02d4d213a420d70c091019367e45759a94ff9a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 25 Apr 2026 01:55:58 +0800 Subject: [PATCH 038/304] Update Submodule vendor/llama.cpp 8bccdbb..13d36cf --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8bccdbbff9..13d36cf891 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8bccdbbff9d0d91d54838471f6eea182b9ab1b79 +Subproject commit 13d36cf89178354d9aa6732e5930d89d64caf718 From ddf577dd5bcf46657e4429d8ab774261b683b647 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 25 Apr 2026 04:02:50 +0800 Subject: [PATCH 039/304] feat(speculative): introduce O(1) hash-based N-Gram speculative decoding - Add `LlamaNGramMapDecoding` to `llama_speculative.py`, implementing an ultra-fast speculative decoder based on a hash inverted index and incremental updates. - Achieve O(1) time complexity for draft token generation, completely eliminating the CPU bottleneck present in the legacy Numpy sliding window approach. - Update `README.md` and `docs/wiki/core/Llama.md` to recommend `LlamaNGramMapDecoding` as the default and fastest speculative decoding method, along with updated initialization examples. - Add docs comment to the speculative decoding classes for better developer experience. - Add warnings to the legacy `LlamaPromptLookupDecoding` class regarding its high computational overhead for long contexts. --- README.md | 18 ++++- docs/wiki/core/Llama.md | 17 ++++- llama_cpp/llama_speculative.py | 129 ++++++++++++++++++++++++++++++++- 3 files changed, 153 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index ab643fe427..8873c39039 100644 --- a/README.md +++ b/README.md @@ -1370,19 +1370,29 @@ emb = llm.create_embedding("text") `llama-cpp-python` supports speculative decoding which allows the model to generate completions based on a draft model. -The fastest way to use speculative decoding is through the `LlamaPromptLookupDecoding` class. +The fastest way to use speculative decoding is through the `LlamaNGramMapDecoding`(**Recommend**) or `LlamaPromptLookupDecoding` class. Just pass this as a draft model to the `Llama` class during initialization. ```python from llama_cpp import Llama -from llama_cpp.llama_speculative import LlamaPromptLookupDecoding +from llama_cpp.llama_speculative import LlamaNGramMapDecoding llama = Llama( - model_path="path/to/model.gguf", - draft_model=LlamaPromptLookupDecoding(num_pred_tokens=10) # num_pred_tokens is the number of tokens to predict 10 is the default and generally good for gpu, 2 performs better for cpu-only machines. + model_path="path/to/qwen-3.6-27b.gguf", + n_ctx=4096, + n_gpu_layers=-1, + draft_model=LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10 + ) +) + +response = llama.create_chat_completion( + messages=[{"role": "user", "content": "Write a python script..."}] ) ``` +Note: `LlamaPromptLookupDecoding.num_pred_tokens` is the number of tokens to predict 10 is the default and generally good for gpu, 2 performs better for cpu-only machines. Now, `LlamaNGramMapDecoding` with the new Hash Map algorithm, draft generation becomes instantaneous $O(1)$, and the time consumption is almost 0 regardless of whether you set the prediction to 2 or 10 words. ### Adjusting the Context Window diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index ce2f1ba5d7..6c8c47b7fe 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -136,16 +136,25 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn 3. **Speculative Decoding**: Accelerates generation by using a small "draft" model to predict tokens, which the larger model then validates in parallel. + The fastest way to use speculative decoding is through the `LlamaNGramMapDecoding`(**Recommend**) or `LlamaPromptLookupDecoding` class. ```python from llama_cpp import Llama - from llama_cpp.llama_speculative import LlamaDraftModel - - draft = LlamaDraftModel.from_model(Llama(model_path="tiny_draft.gguf", n_gpu_layers=-1)) - main_llm = Llama(model_path="large_model.gguf", n_gpu_layers=-1, draft_model=draft) + from llama_cpp.llama_speculative import LlamaNGramMapDecoding + + llama = Llama( + model_path="path/to/qwen-3.6-27b.gguf", + n_ctx=4096, + n_gpu_layers=-1, + draft_model=LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10 + ) + ) for chunk in main_llm.create_completion("Explain quantum physics", stream=True): print(chunk["choices"][0]["text"], end="") ``` + Note: `LlamaPromptLookupDecoding.num_pred_tokens` is the number of tokens to predict 10 is the default and generally good for gpu, 2 performs better for cpu-only machines. Now, `LlamaNGramMapDecoding` with the new Hash Map algorithm, draft generation becomes instantaneous $O(1)$, and the time consumption is almost 0 regardless of whether you set the prediction to 2 or 10 words. 4. **Dynamic LoRA Routing**: diff --git a/llama_cpp/llama_speculative.py b/llama_cpp/llama_speculative.py index 39dfb903ba..c3814aaf42 100644 --- a/llama_cpp/llama_speculative.py +++ b/llama_cpp/llama_speculative.py @@ -1,6 +1,7 @@ import abc +import collections -from typing import Any +from typing import Any, Dict, List, Tuple import numpy as np import numpy.typing as npt @@ -14,10 +15,120 @@ def __call__( raise NotImplementedError() +class LlamaNGramMapDecoding(LlamaDraftModel): + """ + Ultra-fast speculative decoder based on hash inverted index and incremental updates. + O(1) time complexity, aligned with llama.cpp's underlying ngram-map algorithm. + """ + + def __init__(self, ngram_size: int = 3, num_pred_tokens: int = 10): + """ + Initializes the N-Gram Map speculative decoder. + + Args: + ngram_size (int): The length of the token sequence used as the search key. + Larger values provide strictly accurate context matching but may result + in fewer cache hits. Defaults to 3. + num_pred_tokens (int): The maximum number of future tokens to draft (predict) + and return once a match is found in the history. Defaults to 10. + """ + self.ngram_size = ngram_size + self.num_pred_tokens = num_pred_tokens + + # Core state cache + # Mapping format: (token_1, ..., token_N) -> [index_1, index_2, ...] + self._ngram_map: Dict[Tuple[int, ...], List[int]] = collections.defaultdict(list) + self._history: List[int] = [] + + def _update_cache(self, input_ids: npt.NDArray[np.intc]) -> None: + """ + Smart state synchronization and incremental build (Extreme O(1) optimization). + + Args: + input_ids (npt.NDArray[np.intc]): The complete sequence of current token IDs + generated or processed so far. + """ + new_len = len(input_ids) + old_len = len(self._history) + + # Check if it's a perfect incremental append (verify if the previous token matches) + is_incremental = False + if new_len > old_len and old_len > 0: + if self._history[-1] == input_ids[old_len - 1]: + is_incremental = True + + if is_incremental: + # Only extract, convert, and append new tokens. + # Never copy or touch the entire historical array! + new_tokens = input_ids[old_len:].tolist() + self._history.extend(new_tokens) + start_idx = max(0, old_len - self.ngram_size) + else: + # Rollback occurred (wrong prediction) or a completely new Prompt. Trigger full rebuild. + self._ngram_map.clear() + self._history = input_ids.tolist() + start_idx = 0 + + # Build/update the hash inverted index + for i in range(start_idx, new_len - self.ngram_size): + key = tuple(self._history[i : i + self.ngram_size]) + self._ngram_map[key].append(i) + + def __call__( + self, input_ids: npt.NDArray[np.intc], /, **kwargs: Any + ) -> npt.NDArray[np.intc]: + """ + Generates draft tokens based on historical N-Gram frequency. + + Args: + input_ids (npt.NDArray[np.intc]): The current sequence of token IDs. + **kwargs: Additional generation arguments (ignored in this implementation). + + Returns: + npt.NDArray[np.intc]: An array of predicted draft tokens. Returns an empty + array if no matching context is found. + """ + # 1. Ultra-fast state synchronization + self._update_cache(input_ids) + + # 2. Cannot speculate if the history is too short + if len(self._history) < self.ngram_size: + return np.array([], dtype=np.intc) + + # 3. Extract the Search Key (the last N tokens) + search_key = tuple(self._history[-self.ngram_size:]) + + # 4. O(1) instant lookup + match_indices = self._ngram_map.get(search_key) + + if not match_indices: + return np.array([], dtype=np.intc) + + # 5. Get the context of the last match and extract draft tokens + best_match_idx = match_indices[-1] + draft_start = best_match_idx + self.ngram_size + draft_end = min(draft_start + self.num_pred_tokens, len(self._history)) + + return np.array(self._history[draft_start:draft_end], dtype=np.intc) + + +# Legacy Numpy sliding window implementation class LlamaPromptLookupDecoding(LlamaDraftModel): - """Based on https://github.com/apoorvumang/prompt-lookup-decoding""" + """ + Stateless speculative decoding based on Numpy sliding window + Warning: High computational overhead for long contexts. + + Based on https://github.com/apoorvumang/prompt-lookup-decoding + """ + + def __init__(self, max_ngram_size: int = 3, num_pred_tokens: int = 10): + """ + Initializes the legacy sliding window speculative decoder. - def __init__(self, max_ngram_size: int = 2, num_pred_tokens: int = 10): + Args: + max_ngram_size (int): The maximum n-gram size to search for. Defaults to 3. + num_pred_tokens (int): The maximum number of tokens to predict. Defaults to 10. + """ self.max_ngram_size = max_ngram_size self.num_pred_tokens = num_pred_tokens @@ -27,6 +138,17 @@ def find_candidate_pred_tokens( max_ngram_size: int, num_pred_tokens: int, ): + """ + Linearly scans the input_ids using sliding windows to find pattern matches. + + Args: + input_ids (npt.NDArray[np.intc]): The complete sequence of token IDs. + max_ngram_size (int): Maximum size of the n-gram window. + num_pred_tokens (int): Maximum draft tokens to return. + + Returns: + npt.NDArray[np.intc]: The predicted draft tokens. + """ input_length = input_ids.shape[0] for ngram_size in range(min(max_ngram_size, input_length - 1), 0, -1): @@ -57,6 +179,7 @@ def find_candidate_pred_tokens( def __call__( self, input_ids: npt.NDArray[np.intc], /, **kwargs: Any ) -> npt.NDArray[np.intc]: + """Generates draft tokens using the legacy sliding window search.""" return self.find_candidate_pred_tokens( input_ids=input_ids, max_ngram_size=self.max_ngram_size, From 3ec636d3a98f9b1d62f38cd90ca82d51293b8c5c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Apr 2026 02:35:13 +0800 Subject: [PATCH 040/304] Update Submodule vendor/llama.cpp 13d36cf..b760272 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 13d36cf891..b760272f1a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 13d36cf89178354d9aa6732e5930d89d64caf718 +Subproject commit b760272f1a25fcae065d827ce2cbcaa035597b02 From 2cccd2eee3c7ce7476080d8ff06870542722add3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Apr 2026 11:29:10 +0800 Subject: [PATCH 041/304] feat(core): implement thread-safe generation abort mechanism - Add `AbortCriteria` class and a thread-safe `Llama.abort()` method to allow graceful interruption of ongoing text generation from external threads (e.g., UI or async environments). - Automatically inject `AbortCriteria` into the stopping criteria sequence at the start of `_create_completion`. - Ensure that when an abort is triggered, the partially generated `completion_tokens` are correctly detokenized and preserved. - Set `finish_reason` to `"abort"` when generation is interrupted, allowing downstream streaming clients to correctly identify manual cancellations. - Simplify and optimize the stopping criteria evaluation logic within the core `generate` loop. - Reorganize and sort module imports for better readability. Signed-off-by: JamePeng --- llama_cpp/llama.py | 81 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 59e636b56e..be5d259868 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1,17 +1,20 @@ from __future__ import annotations +import contextlib +import ctypes +import fnmatch +import json +import multiprocessing import os import sys -import uuid import time -import json -import ctypes +import threading import typing -import random -import fnmatch +import uuid import warnings -import contextlib -import multiprocessing + +import numpy as np +import numpy.typing as npt from typing import ( Any, @@ -29,7 +32,6 @@ from collections import deque from pathlib import Path - from .llama_types import * from .llama_grammar import LlamaGrammar from .llama_cache import ( @@ -46,9 +48,6 @@ from llama_cpp.llama_speculative import LlamaDraftModel -import numpy as np -import numpy.typing as npt - import llama_cpp._internals as internals from ._internals import ( LlamaSamplingContext, @@ -60,6 +59,19 @@ from ._utils import suppress_stdout_stderr +class AbortCriteria: + """ + Listen for external interruption signals to trigger a stop condition. + When an external thread calls `llama.abort()`, a loop interrupt is generated. + """ + def __init__(self, abort_event: threading.Event): + self.abort_event = abort_event + + def __call__(self, _input_ids: npt.NDArray[np.intc], _logits: npt.NDArray[np.single]) -> bool: + # Note: _input_ids and _logits are required by the signature but unused here. + return self.abort_event.is_set() + + class Llama: """High-level Python wrapper for a llama.cpp model.""" @@ -623,6 +635,9 @@ def __init__( self._sampling_ctx: Optional[LlamaSamplingContext] = None + # Create a thread-safe interrupt event + self._abort_event = threading.Event() + def close(self) -> None: """Explicitly free the model from memory.""" if getattr(self, "_sampling_ctx", None) is not None: @@ -769,6 +784,15 @@ def reset(self): """Reset the model state.""" self.n_tokens = 0 + def abort(self) -> None: + """ + Safely aborts any ongoing text generation. + Useful for async API environments or UI interruption buttons. + """ + if self.verbose: + print(f"Llama.abort: Abort signal received. Terminating generation...", file=sys.stderr) + self._abort_event.set() + def eval( self, tokens: Sequence[int], @@ -1442,26 +1466,18 @@ def adapter(token_data_array: llama_cpp.llama_token_data_array): # Sample loop while sample_idx < self.n_tokens: + if self._abort_event.is_set(): + return + token = self._sampling_ctx.sample(self._ctx, idx=-1) self._sampling_ctx.accept(token, False if grammar is None else True) sample_idx += 1 - # Halt generation if custom stopping criteria are met if stopping_criteria is not None: - if self._logits_all: - logits_idx = sample_idx - self.n_tokens - check_stopping = True - else: - if sample_idx == self.n_tokens: - logits_idx = 0 - check_stopping = True - else: - check_stopping = False - - if check_stopping and stopping_criteria( + if stopping_criteria( self._input_ids[: sample_idx], - self._scores[logits_idx, :] + self._scores[0 if not self._logits_all else sample_idx - self.n_tokens, :] ): return @@ -1746,6 +1762,8 @@ def _create_completion( Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse] ]: assert suffix is None or suffix.__class__ is str + # Each time a new request is initiated, the previous abort state must be cleared. + self._abort_event.clear() completion_id: str = f"cmpl-{str(uuid.uuid4())}" created: int = int(time.time()) @@ -1885,6 +1903,11 @@ def _create_completion( if self.verbose: print("Llama._create_completion: cache miss", file=sys.stderr) + if stopping_criteria is None: + stopping_criteria = StoppingCriteriaList([AbortCriteria(self._abort_event)]) + else: + stopping_criteria.append(AbortCriteria(self._abort_event)) + finish_reason = "length" multibyte_fix = 0 for token in self.generate( @@ -1929,6 +1952,11 @@ def _create_completion( finish_reason = "stop" break + if self._abort_event.is_set(): + text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) + finish_reason = "abort" + break + completion_tokens.append(token) all_text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) @@ -2108,6 +2136,11 @@ def _create_completion( text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) finish_reason = "stop" + # If the abort is triggered externally, force the `finish_reason` to be changed to "abort". + if self._abort_event.is_set(): + text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) + finish_reason = "abort" + if self.verbose: self._ctx.print_timings() From 7a4dd5b1e5dfa2a58108e79e337fcc0c606d5c0a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Apr 2026 18:02:16 +0800 Subject: [PATCH 042/304] Update /docs/wiki/core/Llama.md for abort() and example code Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 102 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 6c8c47b7fe..eacda189e3 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -2,7 +2,7 @@ --- title: Llama Class class_name: Llama -last_updated: 2026-04-23 +last_updated: 2026-04-26 version_target: "latest" --- ``` @@ -103,6 +103,10 @@ Low-level method to ingest and evaluate a sequence of tokens. Used internally to model.eval(tokens=[1, 453, 234, 987], active_loras=[{"name": "coding_adapter", "scale": 1.0}]) ``` +### `abort` +Immediately halts an active generation loop safely. +* **Usage**: Typically called from a separate monitoring thread (like a timer). When triggered, the running stream will exit and the final chunk will contain `"finish_reason": "abort"`. + ### Dynamic LoRA Management The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dynamically per-generation or per-eval. * `load_lora(name: str, path: str)`: Loads an adapter into VRAM (does not apply it yet). @@ -193,6 +197,102 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn ctx_checkpoints=0 # <-- SET THIS TO 0 TO ENABLE ZERO-LATENCY FAST PATH ) ``` +6. **Assistant Prefill**: + + `llama-cpp-python` supports native **Assistant Prefill** for seamless message continuation. You can now simply use the `assistant_prefill=True` parameter in the `create_chat_completion` function. + + This safely renders the `N-1` conversation history using standard Jinja templates (preserving exact control tokens) and flawlessly appends your partial text directly to the prompt. + + ```python + from llama_cpp import Llama + + llm = Llama(model_path="path/to/model.gguf") + + # An interrupted/partial conversation + messages = [ + {"role": "user", "content": "What are the first 5 planets in the solar system?"}, + {"role": "assistant", "content": "The first 5 planets in our solar system are:\n1. Mercury\n2."} + ] + + # Seamlessly continue the generation + response = llm.create_chat_completion( + messages=messages, + max_tokens=50, + assistant_prefill=True # <--- Enables seamless continuation + ) + + prefilled_text = messages[-1]["content"] + # The model will flawlessly continue from " Venus\n3. Earth..." + generated_text = response["choices"][0]["message"]["content"] + + print(prefilled_text + generated_text) + ``` + +7. **Interrupting Reasoning & Assistant Prefill (Time-boxing)**: + + Use the `abort()` method alongside `assistant_prefill=True` to forcefully stop a reasoning model (like Qwen or DeepSeek) if it thinks for too long, inject a bridge text, and force it to output the final answer. + ```python + import threading + from llama_cpp import Llama + + llm = Llama(model_path="Qwen3.6-27B.gguf", n_ctx=4096, n_gpu_layers=-1) + + def run_controlled_generation(prompt: str, timeout_seconds: int = 10): + messages = [{"role": "user", "content": prompt}] + + # 1. Set a time bomb to interrupt long phases + def timeout_handler(): + llm.abort() + + timer = threading.Timer(timeout_seconds, timeout_handler) + timer.start() + + stream = llm.create_chat_completion( + messages=messages, max_tokens=2048, stream=True + ) + + partial_response = "" + finish_reason = None + + for chunk in stream: + finish_reason = chunk["choices"][0].get("finish_reason") + + if finish_reason is not None and finish_reason != "abort": + timer.cancel() + break + + if finish_reason == "abort": + break + + delta = chunk["choices"][0]["delta"].get("content", "") + if delta: + partial_response += delta + print(delta, end="", flush=True) + + # 2. Forced Intervention and Prefill Continuation + if finish_reason == "abort": + # Inject bridge text to forcefully close the reasoning tag + bridge_text = "\n...Wait, I have thought long enough, let's start answering the user.\n\n\n" + print(bridge_text, end="", flush=True) + + prefilled_content = partial_response + bridge_text + messages.append({"role": "assistant", "content": prefilled_content}) + + # Use assistant_prefill=True to seamlessly continue the text block + stream_part2 = llm.create_chat_completion( + messages=messages, + max_tokens=2048, + stream=True, + assistant_prefill=True + ) + + for chunk in stream_part2: + delta = chunk["choices"][0]["delta"].get("content", "") + if delta: + print(delta, end="", flush=True) + + run_controlled_generation("Explain quantum mechanics in a way that relates to bugs in code.", timeout_seconds=8) + ``` --- From 820d1839ef4303c26b9785a8e5eca684cb07e33c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Apr 2026 18:23:06 +0800 Subject: [PATCH 043/304] Update README.md Signed-off-by: JamePeng --- README.md | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 8873c39039..1d56558d23 100644 --- a/README.md +++ b/README.md @@ -4,28 +4,38 @@ # Python Bindings for [`llama.cpp`](https://github.com/ggml-org/llama.cpp) -[![Documentation Status](https://readthedocs.org/projects/llama-cpp-python/badge/?version=latest)](https://llama-cpp-python.readthedocs.io/en/latest/?badge=latest) [![Tests](https://github.com/JamePeng/llama-cpp-python/actions/workflows/test.yaml/badge.svg?branch=main)](https://github.com/JamePeng/llama-cpp-python/actions/workflows/test.yaml) ![GitHub Tag](https://img.shields.io/github/v/tag/JamePeng/llama-cpp-python) [![PyPI - License](https://img.shields.io/pypi/l/llama-cpp-python)](https://pypi.org/project/llama-cpp-python/) [![PyPI - Downloads](https://static.pepy.tech/badge/llama-cpp-python/month)](https://pepy.tech/projects/llama-cpp-python) [![Github All Releases](https://img.shields.io/github/downloads/abetlen/llama-cpp-python/total.svg?label=Github%20Downloads)]() -Simple Python bindings for **@ggerganov's** [`llama.cpp`](https://github.com/ggml-org/llama.cpp) library. +Efficiency Python bindings for **ggml-org's** [`llama.cpp`](https://github.com/ggml-org/llama.cpp) library. This package provides: - Low-level access to C API via `ctypes` interface. + - [llama_cpp_lib](https://github.com/JamePeng/llama-cpp-python/blob/main/llama_cpp/llama_cpp.py) + - [mtmd_cpp_lib](https://github.com/JamePeng/llama-cpp-python/blob/main/llama_cpp/mtmd_cpp.py) - High-level Python API for text completion - - OpenAI-like API - - [LangChain compatibility](https://python.langchain.com/docs/integrations/llms/llamacpp) - - [LlamaIndex compatibility](https://docs.llamaindex.ai/en/stable/examples/llm/llama_2_llama_cpp.html) -- OpenAI compatible web server - - [Local Copilot replacement](https://llama-cpp-python.readthedocs.io/en/latest/server/#code-completion) - - [Function Calling support](https://llama-cpp-python.readthedocs.io/en/latest/server/#function-calling) - - [Vision API support](https://llama-cpp-python.readthedocs.io/en/latest/server/#multimodal-models) - - [Multiple Models](https://llama-cpp-python.readthedocs.io/en/latest/server/#configuration-and-multi-model-support) - -Documentation is available at [https://llama-cpp-python.readthedocs.io/en/latest](https://llama-cpp-python.readthedocs.io/en/latest). + - OpenAI-like API and Type([llama_types.py](https://github.com/JamePeng/llama-cpp-python/blob/main/llama_cpp/llama_types.py)) + - [High-level API](https://github.com/JamePeng/llama-cpp-python#high-level-api) + - [Continuing Assistant Responses (Prefill)](https://github.com/JamePeng/llama-cpp-python#continuing-assistant-responses-prefill) + - [Dynamic LoRA Routing & Control Vectors (Multi-Tenant Serving)](https://github.com/JamePeng/llama-cpp-python#dynamic-lora-routing--control-vectors-multi-tenant-serving) + - [Dynamic LoRA Example](https://github.com/JamePeng/llama-cpp-python#dynamic-lora-example) + - [Control Vector Injection (Representation Engineering)](https://github.com/JamePeng/llama-cpp-python#control-vector-injection-representation-engineering) + - [Sampling Configuration & Usage (LlamaSamplingParams)](https://github.com/JamePeng/llama-cpp-python#sampling-configuration--usage-llamasamplingparams) + - [Multi-modal Models Support](https://github.com/JamePeng/llama-cpp-python#multi-modal-models) + - Support Models Lists + - [Loading a Local Image With Qwen3VL(Thinking/Instruct)](https://github.com/JamePeng/llama-cpp-python#loading-a-local-image-with-qwen3vlthinkinginstruct) + - [Comprehensive Omni MultiModal Example: Gemma-4 (Vision + Audio + Text)](https://github.com/JamePeng/llama-cpp-python#comprehensive-omni-multimodal-example-gemma-4-vision--audio--text) + - [Embeddings & Reranking (GGUF)](https://github.com/JamePeng/llama-cpp-python#embeddings--reranking-gguf) + - [1. Text Embeddings (Vector Search)](https://github.com/JamePeng/llama-cpp-python#1-text-embeddings-vector-search) + - [2. Reranking (Cross-Encoder Scoring)](https://github.com/JamePeng/llama-cpp-python#2-reranking-cross-encoder-scoring) + - [3. Normalization](https://github.com/JamePeng/llama-cpp-python#3-normalization) + - [Speculative Decoding](https://github.com/JamePeng/llama-cpp-python#speculative-decoding) +- [FAQ](https://github.com/JamePeng/llama-cpp-python#faq) + +The new documentation will be maintained in the [docs/wiki](https://github.com/JamePeng/llama-cpp-python/tree/main/docs/wiki) directory based on the LLM Wiki approach. Interested volunteers are welcome to participate in its maintenance and updates :) ## Discussions From 9d231991242a2c4288719786db664a705b1d8bed Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 27 Apr 2026 22:58:36 +0800 Subject: [PATCH 044/304] Update Submodule vendor/llama.cpp b760272..4414c04 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b760272f1a..4414c04b9a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b760272f1a25fcae065d827ce2cbcaa035597b02 +Subproject commit 4414c04b9a23bed188e0318fb1e8812cf175b5b4 From 5068a8072e44aebd2ad124b2db5defa2b102f3c3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 27 Apr 2026 22:59:26 +0800 Subject: [PATCH 045/304] Upload /docs/wiki/LlamaEmbedding.md for llama_embedding.py Signed-off-by: JamePeng --- docs/wiki/SCHEMA.md | 2 +- docs/wiki/core/ChatHandler.md | 0 docs/wiki/core/LlamaEmbedding.md | 263 +++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+), 1 deletion(-) delete mode 100644 docs/wiki/core/ChatHandler.md create mode 100644 docs/wiki/core/LlamaEmbedding.md diff --git a/docs/wiki/SCHEMA.md b/docs/wiki/SCHEMA.md index 4a8f700e2c..f5676442cb 100644 --- a/docs/wiki/SCHEMA.md +++ b/docs/wiki/SCHEMA.md @@ -3,7 +3,7 @@ **Purpose**: Maintain a living, always-up-to-date, structured documentation wiki for the llama-cpp-python library using LLMs as the primary maintainer. **Core Principles**: -- The source of truth is the latest code in `llama_cpp/` (especially `llama.py`, `llama_chat_format.py`, `llama_cpp.py`, `llama_types.py`, `mtmd_cpp.py`, `_internals.py`, `_ggml.py`). +- The source of truth is the latest code in `llama_cpp/` (especially `llama.py`, `llama_chat_format.py`, `llama_cpp.py`, `llama_types.py`, `llama_embedding.py`, `mtmd_cpp.py`, `_internals.py`, `_ggml.py`). - Never invent parameters or behavior. Always read the current source code before writing/updating a page. - All examples must be complete, runnable with the latest API, and include necessary imports. - Clearly mark any deprecated/old usage with a warning and show the modern replacement. diff --git a/docs/wiki/core/ChatHandler.md b/docs/wiki/core/ChatHandler.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/wiki/core/LlamaEmbedding.md b/docs/wiki/core/LlamaEmbedding.md new file mode 100644 index 0000000000..b28772b45e --- /dev/null +++ b/docs/wiki/core/LlamaEmbedding.md @@ -0,0 +1,263 @@ +--- +title: LlamaEmbedding +class_name: LlamaEmbedding +last_updated: 2026-04-27 +version_target: "latest" +--- + +# LlamaEmbedding + +## Overview + +`LlamaEmbedding` is a specialized class for high-performance Text Embedding and Reranking. It inherits from the base `Llama` class but is optimized for vector operations. + +### Support Embeddings & Rerank Model: + + +| Model | Type | Link | Status | +|--------------------|-----------|--------------------------------------------------------|--------------| +| `bge-m3` | Embedding |[bge-m3-GGUF](https://huggingface.co/gpustack/bge-m3-GGUF) | Useful āœ… | +|`bge-reranker-v2-m3`| Rerank |[bge-reranker-v2-m3-GGUF](https://huggingface.co/gpustack/bge-reranker-v2-m3-GGUF) | Useful āœ… | +|`qwen3-reranker`| Rerank |[Qwen3-Reranker-GGUF](https://huggingface.co/JamePeng2023/Qwen3-Reranker-GGUF) | Useful āœ… | + +**Core Features:** +1. **Auto-configuration**: Automatically sets `embeddings=True`. +2. **Streaming Batch**: Handles massive datasets without OOM (Out Of Memory). +3. **Native Reranking Support**: Specifically handles `LLAMA_POOLING_TYPE_RANK` models (like BGE-Reranker, Qwen3-Reranker). It correctly identifies classification heads to output scalar relevance scores instead of high-dimensional vectors. +4. **Advanced Normalization**: Implements MaxInt16, Taxicab (L1), and Euclidean (L2) normalization strategies using NumPy for optimal performance and compatibility with various vector databases. + +## Constructor `__init__` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `model_path` | str | Required | Path to the GGUF model file. | +| `n_ctx` | int | 0 | Text context window size (0 = model default). | +| `n_batch` | int | 512 | Maximum prompt processing batch size. | +| `n_ubatch` | int | 512 | Physical batch size. | +| `pooling_type` | int | `LLAMA_POOLING_TYPE_UNSPECIFIED` (-1) | Pooling strategy used by the model: `LLAMA_POOLING_TYPE_RANK` (4) for rerankers, `LLAMA_POOLING_TYPE_UNSPECIFIED` (-1) for embeddings. | +| `n_gpu_layers` | int | 0 | Number of layers offloaded to GPU (0 = CPU only, -1 = all layers). | +| `verbose` | bool | True | Whether to print debug information. | +| `**kwargs` | Any | ā€” | Extra arguments passed to the `Llama` base class (e.g., `n_batch`, `n_ctx`, `verbose`). | + +### Initialization Logic + +1. Forces `embeddings=True` to enable embedding support. +2. Sets `kv_unified=True` to enable unified KV Cache, allowing arbitrary sequence IDs in a batch without "invalid seq_id" errors. +3. Passes `pooling_type` to the parent class constructor. + +## Core Methods + +### `embed(input, normalize=NORM_MODE_EUCLIDEAN, truncate=True, separator=None, return_count=False)` + +**Description**: Computes embedding vectors for input text (standard embeddings or reranking scores). + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input` | `Union[str, List[str], List[List[int]]]` | ā€” | Input format: string (can be split), list of strings, or list of integer lists (token IDs). | +| `normalize` | int | `NORM_MODE_EUCLIDEAN` (2) | Vector normalization mode (see below). | +| `truncate` | bool | True | Whether to truncate input. | +| `separator` | str | None | Separator for splitting string input into multiple documents. | +| `return_count` | bool | False | If True, returns `(embeddings, token_count)`. | + +**Normalization Modes:** +- `NORM_MODE_NONE` (-1): No normalization. +- `NORM_MODE_MAX_INT16` (0): Max absolute value normalization (scaled to 32760). +- `NORM_MODE_TAXICAB` (1): L1 Taxicab norm. +- `NORM_MODE_EUCLIDEAN` (2): L2 Euclidean norm. +- `NORM_MODE_PNORM` (>2): p-norm normalization. + +**Returns:** +- `return_count=False`: List of embedding vectors. +- `return_count=True`: Tuple `(embeddings, token_count)`. + +**Internal Logic:** +1. Determines mode based on `pooling_type`: `LLAMA_POOLING_TYPE_NONE` (token-level), `LLAMA_POOLING_TYPE_RANK` (rerank), or other (sequence-level). +2. Uses streaming batch decoding to process embeddings in chunks. +3. For token-level mode, extracts and normalizes per-token vectors. +4. For sequence-level mode, extracts sequence vectors and normalizes. +5. Supports `separator` for splitting input into multiple documents. + +### `rank(query, documents)` + +**Description**: Calculates relevance scores for a list of documents against a query using a Reranking model. + +| Parameter | Type | Description | +|-----------|------|-------------| +| `query` | str | Search query string. | +| `documents` | `List[str]` | List of candidate document strings to be scored. | + +**Returns**: List of float scores, where higher values indicate greater relevance. + +**Internal Logic:** +1. Checks if model is a reranker (`pooling_type == LLAMA_POOLING_TYPE_RANK`). +2. Attempts to retrieve the built-in 'rerank' chat template. +3. If template exists: dynamically replaces `{query}` and `{document}` and tokenizes; otherwise, manually constructs `[BOS] Query [SEP] Doc [EOS]` sequence. +4. Executes embedding inference (`embed`), returning raw logits/scores. +5. For generative rerankers (e.g., Qwen3-Reranker, output dim = 2), uses `yes_logit` as relevance score. + +### `create_embedding(input, model=None, normalize=NORM_MODE_EUCLIDEAN, output_format="json")` + +**Description**: High-level API compatible with OpenAI format. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input` | `Union[str, List[str]]` | ā€” | Input text or list of texts. | +| `model` | str | None | Model name (optional, uses `self.model_path` if None). | +| `normalize` | int | `NORM_MODE_EUCLIDEAN` (2) | Normalization mode. | +| `output_format` | str | "json" | Output format: 'json', 'json+', or 'array'. | + +**Output Formats:** +- `'json'`: OpenAI-style dictionary list. +- `'json+'`: OpenAI dictionary list + cosine similarity matrix. +- `'array'`: Raw Python list (`List[float]` or `List[List[float]]`). + +**Returns**: Data structure according to `output_format`. + +## Deprecated / Changed APIs + +- **Note**: The TODO comments `# TODO(JamePeng): Needs more extensive testing with various embedding and reranking models.` indicate that support for various embedding and reranking models may be incomplete. Further testing is recommended. + +## Best Practices & Common Patterns + +1. **Select Correct `pooling_type`**: + - Standard embeddings: `LLAMA_POOLING_TYPE_UNSPECIFIED (-1)`. + - Reranker models: `LLAMA_POOLING_TYPE_RANK (4)`. + - Token-level embeddings: `LLAMA_POOLING_TYPE_NONE (0)`. + +2. **Batch Optimization for Large Datasets**: + - Adjust `n_batch` and `n_ubatch` to balance performance and memory. + - Streaming processing avoids OOM for large datasets. + +3. **Normalization Selection**: + - Vector databases typically prefer L2 normalization (Euclidean), but other norms may be needed in specific scenarios. + +4. **Reranker Models**: + - Ensure `pooling_type` is set to `LLAMA_POOLING_TYPE_RANK`. + - Note that output is scalar scores, not vectors. + +5. **Performance Tuning**: + - For GPU acceleration, set `n_gpu_layers` to -1 (recommended). + - Use `verbose=True` for debugging configuration. + +## Example Code + +### 1. Text Embeddings (Vector Search) + +To generate embeddings, use the `LlamaEmbedding` class. It automatically configures the model for vector generation. + +```python +from llama_cpp.llama_embedding import LlamaEmbedding, LLAMA_POOLING_TYPE_NONE + +# Initialize the model (automatically sets embeddings=True) +llm = LlamaEmbedding(model_path="path/to/bge-m3.gguf", n_gpu_layers=-1, pooling_type=LLAMA_POOLING_TYPE_NONE) + +# 1. Simple usage (OpenAI-compatible format) +response = llm.create_embedding("Hello, world!") +print(response['data'][0]['embedding']) + +# 2. Batch processing (High Performance) +# You can pass a large list of strings; the streaming batcher handles memory automatically. +documents = ["Hello, world!", "Goodbye, world!", "Llama is cute."] * 100 +embeddings = llm.embed(documents) # Returns a list of lists (vectors) + +print(f"Generated {len(embeddings)} vectors.") +``` + +**Advanced Output Formats:** +You can request raw arrays or cosine similarity matrices directly: + +```python +from llama_cpp.llama_embedding import LlamaEmbedding, LLAMA_POOLING_TYPE_NONE + +# Initialize the model (automatically sets embeddings=True) +llm = LlamaEmbedding(model_path="path/to/bge-m3.gguf", n_gpu_layers=-1, pooling_type=LLAMA_POOLING_TYPE_NONE) + +# Returns raw List[float] instead of a dictionary wrapper +vector = llm.create_embedding("Text", output_format="array") + +# Returns a similarity matrix (A @ A.T) in the response +# Note: Requires numpy installed +response = llm.create_embedding( + ["apple", "fruit", "car"], + output_format="json+" +) +print(response["cosineSimilarity"]) +``` + +### 2. Reranking (Cross-Encoder Scoring) + +Reranking models (like `bge-reranker`) take a **Query** and a list of **Documents** as input and output a relevance score (scalar) for each document. + +> **Important:** You must explicitly set `pooling_type` to `LLAMA_POOLING_TYPE_RANK` (4) when initializing the model. + +```python +import llama_cpp +from llama_cpp.llama_embedding import LlamaEmbedding + +# Initialize a Reranking model +ranker = LlamaEmbedding( + model_path="path/to/qwen3-reranker-0.6b-q8_0.gguf", + pooling_type=llama_cpp.LLAMA_POOLING_TYPE_RANK, # Crucial for Rerankers! + n_gpu_layers=-1, + n_ctx=0 +) + +query = "What causes Rain?" +docs = [ + "Clouds are made of water droplets...", # Relevant + "To bake a cake you need flour...", # Irrelevant + "Rain is liquid water in the form of droplets..." # Highly Relevant +] + +# Calculate relevance scores +# Logic: Constructs inputs like "[BOS] query [SEP] doc [EOS]" automatically +scores = ranker.rank(query, docs) + +# Result: List of floats (higher means more relevant) +print(scores) +# e.g., [0.0011407170677557588, 5.614783731289208e-05, 0.7173627614974976] -> The 3rd doc is the best match +``` + +### 3. Normalization + +The `embed` method supports various mathematical normalization strategies via the `normalize` parameter. + +| Normalization modes | $Integer$ | Description | Formula | +|---------------------|-----------|---------------------|---------| +| NORM_MODE_NONE | $-1$ | none | +| NORM_MODE_MAX_INT16 | $0$ | max absolute int16 | $\Large{{32760 * x_i} \over\max \lvert x_i\rvert}$ +| NORM_MODE_TAXICAB | $1$ | taxicab | $\Large{x_i \over\sum \lvert x_i\rvert}$ +| NORM_MODE_EUCLIDEAN | $2$ | euclidean (default) | $\Large{x_i \over\sqrt{\sum x_i^2}}$ +| NORM_MODE_PNORM | $>2$ | p-norm | $\Large{x_i \over\sqrt[p]{\sum \lvert x_i\rvert^p}}$ + +This is useful for optimizing storage or preparing vectors for cosine similarity search (which requires L2 normalization). + +```python +from llama_cpp.llama_embedding import ( + LLAMA_POOLING_TYPE_NONE, + NORM_MODE_MAX_INT16, + NORM_MODE_TAXICAB, + NORM_MODE_EUCLIDEAN +) + +# Initialize the model (automatically sets embeddings=True) +llm = LlamaEmbedding(model_path="path/to/bge-m3.gguf", n_gpu_layers=-1, pooling_type=LLAMA_POOLING_TYPE_NONE) + +# Taxicab (L1) +vec_l1 = llm.embed("text", normalize=NORM_MODE_TAXICAB) + +# Default is Euclidean (L2) - Standard for vector databases +vec_l2 = llm.embed("text", normalize=NORM_MODE_EUCLIDEAN) + +# Max Absolute Int16 - Useful for quantization/compression +vec_int16 = llm.embed("text", normalize=NORM_MODE_MAX_INT16) + +# Raw Output (No Normalization) - Get the raw floating point values from the model +embeddings_raw = llm.embed(["search query", "document text"], normalize=NORM_MODE_NONE) +``` + +## Notes + +- This class is in development; some features may be unstable, especially reranking model support. +- Performance issues can be addressed by adjusting `n_batch`, `n_ubatch`, and `n_gpu_layers`. +- For custom models, manual `pooling_type` configuration may be required to match model behavior. \ No newline at end of file From 969b45f4a8fbd4f1e760988c1947b4b3328bcb45 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 28 Apr 2026 00:12:43 +0800 Subject: [PATCH 046/304] feat(handler): Support `add_generation_prompt` parameter pass to MTMDChatHandler - supports disabling assistant part injection, used to support the multimodal `assistant_prefill` functionality. Signed-off-by: JamePeng --- llama_cpp/llama.py | 2 ++ llama_cpp/llama_chat_format.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index be5d259868..5ad8364a9b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2717,6 +2717,7 @@ def create_chat_completion( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, assistant_prefill: bool = False, + add_generation_prompt: bool = True, ) -> Union[ CreateChatCompletionResponse, Iterator[CreateChatCompletionStreamResponse] ]: @@ -2829,6 +2830,7 @@ def create_chat_completion( active_loras=active_loras, control_vector=control_vector, assistant_prefill=assistant_prefill, + add_generation_prompt=add_generation_prompt, ) def create_chat_completion_openai_v1( diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index af068f5535..a0d8d25db4 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3079,6 +3079,7 @@ def _process_mtmd_prompt( function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + add_generation_prompt: bool = True, ) -> Tuple[List[int], List[tuple], Any, List[Any]]: """ Core multimodal preprocessing pipeline. @@ -3106,7 +3107,7 @@ def _process_mtmd_prompt( # 2. Render the chat template and replace actual URLs with C++ media markers text = self.chat_template.render( messages=messages, - add_generation_prompt=True, + add_generation_prompt=add_generation_prompt, eos_token=self.mtmd_eos_token, bos_token=self.mtmd_bos_token, functions=functions, @@ -3306,6 +3307,7 @@ def __call__( logit_bias: Optional[Dict[str, float]] = None, logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, + add_generation_prompt: bool = True, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, @@ -3322,7 +3324,8 @@ def __call__( functions=functions, function_call=function_call, tools=tools, - tool_choice=tool_choice + tool_choice=tool_choice, + add_generation_prompt=add_generation_prompt, ) if self.verbose: From db8bf453c980776433dd5f23075711a5896a7b5c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 30 Apr 2026 03:02:03 +0800 Subject: [PATCH 047/304] Update Submodule vendor/llama.cpp 4414c04..660b1b4 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4414c04b9a..660b1b4bdc 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4414c04b9a23bed188e0318fb1e8812cf175b5b4 +Subproject commit 660b1b4bdc6fedc18e8c3d87a945ffb51f91c547 From 66dd88b0ad8126c322a110d22fd90811a8cb64b6 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 1 May 2026 05:55:23 +0800 Subject: [PATCH 048/304] feat(_ggml): implement ggml-backend API bindings and fix type hints - Introduces extensive ctypes bindings for `ggml-backend.h` (devices, buffers, registries, and CPU buffer types) to support advanced memory routing like MoE CPU offloading. Also fixes various static typing warnings by adding `# type: ignore` to pointer annotations. Signed-off-by: JamePeng --- llama_cpp/_ggml.py | 525 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 499 insertions(+), 26 deletions(-) diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 733a3520c1..8f4cb1187f 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -548,7 +548,7 @@ class ggml_object(ctypes.Structure): if TYPE_CHECKING: offs: ctypes.c_size_t size: ctypes.c_size_t - next: "ctypes.POINTER(ggml_object)" + next: "ctypes.POINTER(ggml_object)" # type: ignore type: int padding: ctypes.Array[ctypes.c_char] @@ -586,8 +586,8 @@ class ggml_context(ctypes.Structure): mem_buffer_owned: bool no_alloc: bool n_objects: int - objects_begin: ggml_object_p - objects_end: ggml_object_p + objects_begin: ggml_object_p # type: ignore + objects_end: ggml_object_p # type: ignore _fields_ = [ ("mem_size", ctypes.c_size_t), @@ -694,8 +694,8 @@ class ggml_tensor(ctypes.Structure): op: int op_params: ctypes.Array[ctypes.c_int32] flags: int - src: "ctypes.Array[ctypes.POINTER(ggml_tensor)]" - view_src: "ctypes.POINTER(ggml_tensor)" + src: "ctypes.Array[ctypes.POINTER(ggml_tensor)]" # type: ignore + view_src: "ctypes.POINTER(ggml_tensor)" # type: ignore view_offs: ctypes.c_size_t data: ctypes.c_void_p name: ctypes.Array[ctypes.c_char] @@ -744,8 +744,8 @@ class ggml_tensor(ctypes.Structure): None, ) def ggml_log_get( - log_callback: Optional[ctypes.POINTER(ggml_log_callback)], - user_data: ctypes.POINTER(ctypes.c_void_p), + log_callback: Optional[ctypes.POINTER(ggml_log_callback)], # type: ignore + user_data: ctypes.POINTER(ctypes.c_void_p), # type: ignore /, ): """ @@ -763,7 +763,7 @@ def ggml_log_get( None, ) def ggml_log_set( - log_callback: Optional[ggml_log_callback], + log_callback: Optional[ggml_log_callback], # type: ignore user_data: ctypes.c_void_p, /, ): @@ -875,36 +875,509 @@ class ggml_opt_optimizer_params(ctypes.Structure): ) -# from ggml-backend.h -# // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback) -# // when ask == true, the scheduler wants to know if the user wants to observe this node -# // this allows the scheduler to batch nodes together in order to evaluate them in a single call # // -# // when ask == false, the scheduler is passing the node tensor to the user for observation -# // if the user returns false, the scheduler will cancel the graph compute +# // GGML Backend from ggml-backend.h # // -# typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); -ggml_backend_sched_eval_callback = ctypes.CFUNCTYPE( - ctypes.c_bool, ctypes.c_void_p, ctypes.c_bool, ctypes.c_void_p + +# typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; +ggml_backend_buffer_type_t = NewType( + "ggml_backend_buffer_type_t", + ctypes.c_void_p, +) + +# typedef struct ggml_backend_buffer * ggml_backend_buffer_t; +ggml_backend_buffer_t = NewType( + "ggml_backend_buffer_t", + ctypes.c_void_p, +) + +# typedef struct ggml_backend_event * ggml_backend_event_t; +ggml_backend_event_t = NewType( + "ggml_backend_event_t", + ctypes.c_void_p, +) + +# typedef struct ggml_backend * ggml_backend_t; +ggml_backend_t = NewType( + "ggml_backend_t", + ctypes.c_void_p, +) + +# typedef struct ggml_backend_reg * ggml_backend_reg_t; +ggml_backend_reg_t = NewType( + "ggml_backend_reg_t", + ctypes.c_void_p, +) + +# typedef struct ggml_backend_device * ggml_backend_dev_t; +ggml_backend_dev_t = NewType( + "ggml_backend_dev_t", + ctypes.c_void_p, +) + +# // +# // Backend buffer type +# // + +# GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); +@ggml_base_function("ggml_backend_buft_name", [ctypes.c_void_p], ctypes.c_char_p) +def ggml_backend_buft_name(buft: ggml_backend_buffer_type_t) -> ctypes.c_char_p: + """ + Get ggml_backend_buffer name + """ + ... + + +# GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); +@ggml_base_function("ggml_backend_buft_alloc_buffer", [ctypes.c_void_p, ctypes.c_size_t], ctypes.c_void_p) +def ggml_backend_buft_alloc_buffer( + buft: ggml_backend_buffer_type_t, + size: ctypes.c_size_t +) -> ggml_backend_buffer_t: + """ + Alloc ggml_backend_buffer with size + """ + ... + + +# GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); +@ggml_base_function("ggml_backend_buft_get_alignment", [ctypes.c_void_p], ctypes.c_size_t) +def ggml_backend_buft_get_alignment(buft: ggml_backend_buffer_type_t) -> ctypes.c_size_t: + """ + Get tensor alignment by ggml_backend_buffer + """ + ... + + +# GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft); +@ggml_base_function("ggml_backend_buft_get_max_size", [ctypes.c_void_p], ctypes.c_size_t) +def ggml_backend_buft_get_max_size(buft: ggml_backend_buffer_type_t) -> ctypes.c_size_t: + """ + Get ggml_backend_buffer max buffer size that can be allocated (defaults to SIZE_MAX) + """ + ... + + +# GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); +@ggml_base_function("ggml_backend_buft_get_alloc_size", [ + ctypes.c_void_p, + ggml_tensor_p, +], ctypes.c_size_t) +def ggml_backend_buft_get_alloc_size( + buft: ggml_backend_buffer_type_t, + tensor: ggml_tensor_p, # type: ignore +) -> ctypes.c_size_t: + """ + Get alloc data size needed to allocate the tensor, including padding (defaults to ggml_nbytes) + """ + ... + + +# GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); +@ggml_base_function("ggml_backend_buft_is_host", [ctypes.c_void_p], ctypes.c_bool) +def ggml_backend_buft_is_host(buft: ggml_backend_buffer_type_t) -> ctypes.c_bool: + """ + Check if ggml_backend_buffer is host + """ + ... + + +# GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft); +@ggml_base_function("ggml_backend_buft_get_device", [ctypes.c_void_p], ctypes.c_void_p) +def ggml_backend_buft_get_device(buft: ggml_backend_buffer_type_t) -> ggml_backend_dev_t: + """ + Get device by ggml_backend_buffer + """ + ... + +# // +# // Backend buffer +# // + +# enum ggml_backend_buffer_usage { +# GGML_BACKEND_BUFFER_USAGE_ANY = 0, +# GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1, +# GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2, +# }; +class GGMLBackendBufferUsage(enum.IntEnum): + GGML_BACKEND_BUFFER_USAGE_ANY = 0 + GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1 + GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2 + + +# GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_name", [ctypes.c_void_p], ctypes.c_char_p) +def ggml_backend_buffer_name(buffer: ggml_backend_buffer_t) -> ctypes.c_char_p: + """ + Get ggml_backend_buffer name + """ + ... + + +# GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_free", [ctypes.c_void_p], None) +def ggml_backend_buffer_free(buffer: ggml_backend_buffer_t): + """ + Free ggml_backend_buffer + """ + ... + + +# GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_get_base", [ctypes.c_void_p], None) +def ggml_backend_buffer_get_base(buffer: ggml_backend_buffer_t): + """ + Get ggml_backend_buffer base address + """ + ... + + +# GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_get_size", [ctypes.c_void_p], ctypes.c_size_t) +def ggml_backend_buffer_get_size(buffer: ggml_backend_buffer_t) -> ctypes.c_size_t: + """ + Get ggml_backend_buffer size + """ + ... + + +# GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); +@ggml_base_function("ggml_backend_buffer_init_tensor", [ + ctypes.c_void_p, + ggml_tensor_p, +], ctypes.c_int32) +def ggml_backend_buffer_init_tensor( + buffer: ggml_backend_buffer_t, + tensor: ggml_tensor_p, # type: ignore +) -> ctypes.c_int32: + """ + Init tensor by ggml_backend_buffer + """ + ... + + +# GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_get_alignment", [ctypes.c_void_p], ctypes.c_size_t) +def ggml_backend_buffer_get_alignment(buffer: ggml_backend_buffer_t) -> ctypes.c_size_t: + """ + Get tensor alignment by ggml_backend_buffer + """ + ... + + +# GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_get_max_size", [ctypes.c_void_p], ctypes.c_size_t) +def ggml_backend_buffer_get_max_size(buffer: ggml_backend_buffer_t) -> ctypes.c_size_t: + """ + Get max buffer size that can be allocated (defaults to SIZE_MAX) + """ + ... + + +# GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor); +@ggml_base_function("ggml_backend_buffer_get_alloc_size", [ + ctypes.c_void_p, + ggml_tensor_p, +], ctypes.c_size_t) +def ggml_backend_buffer_get_alloc_size( + buffer: ggml_backend_buffer_t, + tensor: ggml_tensor_p, # type: ignore +) -> ctypes.c_size_t: + """ + Get alloc data size needed to allocate the tensor, including padding (defaults to ggml_nbytes) + """ + ... + + +# GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); +@ggml_base_function("ggml_backend_buffer_clear", [ctypes.c_void_p, ctypes.c_uint8], None) +def ggml_backend_buffer_clear(buffer: ggml_backend_buffer_t, value: ctypes.c_uint8): + """ + Clear ggml_backend_buffer + """ + ... + + +# GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_is_host", [ctypes.c_void_p], ctypes.c_bool) +def ggml_backend_buffer_is_host(buffer: ggml_backend_buffer_t) -> ctypes.c_bool: + """ + Check if ggml_backend_buffer is host + """ + ... + + +# GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); +@ggml_base_function("ggml_backend_buffer_set_usage", [ctypes.c_void_p, ctypes.c_int32], None) +def ggml_backend_buffer_set_usage(buffer: ggml_backend_buffer_t, usage: ctypes.c_int32): + """ + Set ggml_backend_buffer usage + """ + ... + + +# GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_get_usage", [ctypes.c_void_p], ctypes.c_int32) +def ggml_backend_buffer_get_usage(buffer: ggml_backend_buffer_t) -> ctypes.c_int32: + """ + Get ggml_backend_buffer usage + """ + ... + + +# GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_get_type", [ctypes.c_void_p], ctypes.c_void_p) +def ggml_backend_buffer_get_type(buffer: ggml_backend_buffer_t) -> ggml_backend_buffer_t: + """ + Get ggml_backend_buffer_type + """ + ... + + +# GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); +@ggml_base_function("ggml_backend_buffer_reset", [ctypes.c_void_p], None) +def ggml_backend_buffer_reset(buffer: ggml_backend_buffer_t): + """ + Reset ggml_backend_buffer + """ + ... + + +# // +# // Backend device +# // + +# enum ggml_backend_dev_type { +# // CPU device using system memory +# GGML_BACKEND_DEVICE_TYPE_CPU, +# // GPU device using dedicated memory +# GGML_BACKEND_DEVICE_TYPE_GPU, +# // integrated GPU device using host memory +# GGML_BACKEND_DEVICE_TYPE_IGPU, +# // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) +# GGML_BACKEND_DEVICE_TYPE_ACCEL, +# // "meta" device wrapping multiple other devices for tensor parallelism +# GGML_BACKEND_DEVICE_TYPE_META, +# }; +class GGMLBackendDevType(enum.IntEnum): + GGML_BACKEND_DEVICE_TYPE_CPU = 0 # CPU device using system memory + GGML_BACKEND_DEVICE_TYPE_GPU = 1 # GPU device using dedicated memory + GGML_BACKEND_DEVICE_TYPE_IGPU = 2 # integrated GPU device using host memory + GGML_BACKEND_DEVICE_TYPE_ACCEL = 3 # accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) + GGML_BACKEND_DEVICE_TYPE_META = 4 # "meta" device wrapping multiple other devices for tensor parallelism + +ggml_backend_dev_type_t = NewType( + "ggml_backend_dev_type_t", + ctypes.c_void_p, ) # // # // Backend registry # // +# GGML_API void ggml_backend_register(ggml_backend_reg_t reg); +@ggml_function("ggml_backend_register", [ctypes.c_void_p], None) +def ggml_backend_register(reg: ctypes.c_void_p): + """ + Register ggml backend + """ + ... + +# GGML_API void ggml_backend_device_register(ggml_backend_dev_t device); +@ggml_function("ggml_backend_device_register", [ctypes.c_void_p], None) +def ggml_backend_device_register(device: ctypes.c_void_p): + """ + Register ggml backend device + """ + ... + + +# // Backend (reg) enumeration + +# GGML_API size_t ggml_backend_reg_count(void); +@ggml_function("ggml_backend_reg_count", [], ctypes.c_size_t) +def ggml_backend_reg_count() -> ctypes.c_size_t: + """ + Get ggml_backend_reg count + """ + ... + +# GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index); +@ggml_function("ggml_backend_reg_get", [ctypes.c_size_t], ctypes.c_void_p) +def ggml_backend_reg_get(index: ctypes.c_size_t) -> ggml_backend_reg_t: + """ + Get ggml_backend_reg by index + """ + +# GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name); +@ggml_function("ggml_backend_reg_by_name", [ctypes.c_char_p], ctypes.c_void_p) +def ggml_backend_reg_by_name(name: ctypes.c_char_p) -> ggml_backend_reg_t: + """ + Get ggml_backend_reg by name + """ + ... + +# // Device enumeration + +# GGML_API size_t ggml_backend_dev_count(void); +@ggml_function("ggml_backend_dev_count", [], ctypes.c_size_t) +def ggml_backend_dev_count() -> ctypes.c_size_t: + """ + Get ggml_backend_dev count + """ + ... + +# GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index); +@ggml_function("ggml_backend_dev_get", [ctypes.c_size_t], ctypes.c_void_p) +def ggml_backend_dev_get(index: ctypes.c_size_t) -> ggml_backend_dev_t: + """ + Get ggml_backend_dev by index + """ + ... + +# GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name); +@ggml_function("ggml_backend_dev_by_name", [ctypes.c_char_p], ctypes.c_void_p) +def ggml_backend_dev_by_name(name: ctypes.c_char_p) -> ggml_backend_dev_t: + """ + Get ggml_backend_dev by name + """ + ... + +# GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type); +@ggml_function("ggml_backend_dev_by_type", [ctypes.c_int32], ctypes.c_void_p) +def ggml_backend_dev_by_type(type: ctypes.c_int32) -> ggml_backend_dev_t: + """ + Get ggml_backend_dev by type + """ + ... + +# // Direct backend (stream) initialization + +# // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params) +# GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params); +@ggml_function("ggml_backend_init_by_name", [ctypes.c_char_p, ctypes.c_char_p], ctypes.c_void_p) +def ggml_backend_init_by_name(name: ctypes.c_char_p, params: ctypes.c_char_p) -> ggml_backend_t: + """ + = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params) + """ + ... + + +# // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params) +# GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params); +@ggml_base_function("ggml_backend_dev_init", [ctypes.c_int32, ctypes.c_char_p], ctypes.c_void_p) +def ggml_backend_dev_init(type: ctypes.c_int32, params: ctypes.c_char_p) -> ggml_backend_t: + """ + = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params) + """ + ... + + +# // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL) +# GGML_API ggml_backend_t ggml_backend_init_best(void); +@ggml_function("ggml_backend_init_best", [], ctypes.c_void_p) +def ggml_backend_init_best() -> ggml_backend_t: + """ + = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL) + """ + ... + + +# // Load a backend from a dynamic library and register it +# GGML_API ggml_backend_reg_t ggml_backend_load(const char * path); +@ggml_function("ggml_backend_load", [ctypes.c_char_p], ctypes.c_void_p) +def ggml_backend_load(path: ctypes.c_char_p) -> ggml_backend_reg_t: + """ + Load a backend from a dynamic library and register it + """ + ... + + +# // Unload a backend if loaded dynamically and unregister it +# GGML_API void ggml_backend_unload(ggml_backend_reg_t reg); +@ggml_function("ggml_backend_load_all", [ctypes.c_void_p], None) +def ggml_backend_load_all(reg: ggml_backend_reg_t): + """ + Unload a backend if loaded dynamically and unregister it + """ + ... + + # // Load all known backends from dynamic libraries + # GGML_API void ggml_backend_load_all(void); @ggml_function("ggml_backend_load_all", [], None) def ggml_backend_load_all(): - """Load all known backends from dynamic libraries""" + """ + Load all known backends from dynamic libraries + """ ... + # GGML_API void ggml_backend_load_all_from_path(const char * dir_path); @ggml_function("ggml_backend_load_all_from_path", [ctypes.c_char_p], None) def ggml_backend_load_all_from_path(dir_path: ctypes.c_char_p): - """Load all known backends from path""" + """ + Load all known backends from path + """ ... + +# // CPU buffer types are always available + +# GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); +@ggml_base_function( + "ggml_backend_cpu_buffer_from_ptr", + [ctypes.c_void_p, ctypes.c_size_t], + ctypes.c_void_p, +) +def ggml_backend_cpu_buffer_from_ptr( + ptr: ctypes.c_void_p, + size: ctypes.c_size_t +) -> ggml_backend_buffer_t: + """ + Return the CPU backend buffer type from ptr. + """ + ... + + +# GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void); +@ggml_base_function( + "ggml_backend_cpu_buffer_type", + [], + ctypes.c_void_p, +) +def ggml_backend_cpu_buffer_type() -> ggml_backend_buffer_type_t: + """ + Return the CPU backend buffer type. + """ + ... + + +# // +# // Backend scheduler +# // + +# typedef struct ggml_backend_sched * ggml_backend_sched_t; +ggml_backend_sched_t = NewType( + "ggml_backend_sched_t", + ctypes.c_void_p, +) + + +# // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback) +# // when ask == true, the scheduler wants to know if the user wants to observe this node +# // this allows the scheduler to batch nodes together in order to evaluate them in a single call +# // +# // when ask == false, the scheduler is passing the node tensor to the user for observation +# // if the user returns false, the scheduler will cancel the graph compute +# // +# typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); +ggml_backend_sched_eval_callback = ctypes.CFUNCTYPE( + ctypes.c_bool, ctypes.c_void_p, ctypes.c_bool, ctypes.c_void_p +) + + # // # // GGML internal header from ggml-impl.h # // @@ -933,8 +1406,8 @@ class GGMLCgraphEvalOrder(enum.IntEnum): class ggml_hash_set(ctypes.Structure): if TYPE_CHECKING: size: int - used: ctypes.POINTER(ggml_bitset_t) - keys: "ctypes.POINTER(ggml_tensor_p)" + used: ctypes.POINTER(ggml_bitset_t) # type: ignore + keys: ctypes.POINTER(ggml_tensor_p) # type: ignore _fields_ = [ ("size", ctypes.c_size_t), @@ -963,11 +1436,11 @@ class ggml_cgraph(ctypes.Structure): size: int n_nodes: int n_leafs: int - nodes: "ctypes.POINTER(ggml_tensor_p)" - grads: "ctypes.POINTER(ggml_tensor_p)" - grad_accs: "ctypes.POINTER(ggml_tensor_p)" - leafs: "ctypes.POINTER(ggml_tensor_p)" - use_counts: ctypes.POINTER(ctypes.c_int32) + nodes: ctypes.POINTER(ggml_tensor_p) # type: ignore + grads: ctypes.POINTER(ggml_tensor_p) # type: ignore + grad_accs: ctypes.POINTER(ggml_tensor_p) # type: ignore + leafs: ctypes.POINTER(ggml_tensor_p) # type: ignore + use_counts: ctypes.POINTER(ctypes.c_int32) # type: ignore visited_hash_set: ggml_hash_set order: int From b7064d7fc27d33d28837e7885e8e744feb2be3b1 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 1 May 2026 07:44:49 +0800 Subject: [PATCH 049/304] feat(llama): add fine-grained MoE CPU offloading controls - Introduce `cpu_moe` (bool) and `n_cpu_moe` (int) parameters to `Llama.__init__` for precise Mixture of Experts (MoE) weight offloading. - `cpu_moe=True` forces all MoE expert weights to the CPU memory, regardless of `n_gpu_layers`. - `n_cpu_moe=N` offloads the expert weights of the first N layers to the CPU, while keeping attention and router weights on the GPU. - Enhance `n_gpu_layers` to accept string literals "auto" (equivalent to -1) and "all" (equivalent to -2) alongside exact integers, improving configuration readability. - Update internal module aliases (e.g., `llama_cpp` to `llama_cpp_lib`) to avoid naming conflicts with the underlying C library. - Integrate `ggml_backend_cpu_buffer_type` to map specific tensor overrides (via regex) directly to CPU buffers during model load. Signed-off-by: JamePeng --- llama_cpp/llama.py | 195 +++++++++++++++++++++++++++++++-------------- 1 file changed, 137 insertions(+), 58 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 5ad8364a9b..1241f81e26 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -43,7 +43,7 @@ HybridCheckpointCache, # type: ignore ) from .llama_tokenizer import BaseLlamaTokenizer, LlamaTokenizer -import llama_cpp.llama_cpp as llama_cpp +import llama_cpp.llama_cpp as llama_cpp_lib import llama_cpp.llama_chat_format as llama_chat_format from llama_cpp.llama_speculative import LlamaDraftModel @@ -55,6 +55,9 @@ CommonSamplerType, CustomSampler, ) +from ._ggml import ( + ggml_backend_cpu_buffer_type, +) from ._logger import set_verbose from ._utils import suppress_stdout_stderr @@ -77,13 +80,17 @@ class Llama: __backend_initialized = False + LLM_FFN_EXPS_REGEX = rb"\.ffn_(up|down|gate|gate_up)_(ch|)exps" + def __init__( self, model_path: str, *, # Model Params - n_gpu_layers: int = 0, - split_mode: int = llama_cpp.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, + n_gpu_layers: Union[int, Literal["auto", "all"]] = "auto", + cpu_moe: bool = False, + n_cpu_moe: int = 0, + split_mode: int = llama_cpp_lib.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, vocab_only: bool = False, @@ -95,7 +102,7 @@ def __init__( no_host: bool = False, kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, # Context Params - seed: int = llama_cpp.LLAMA_DEFAULT_SEED, + seed: int = llama_cpp_lib.LLAMA_DEFAULT_SEED, n_ctx: int = 512, n_keep: int = 256, n_batch: int = 2048, @@ -105,10 +112,10 @@ def __init__( n_threads_batch: Optional[int] = None, rope_scaling_type: Optional[ int - ] = llama_cpp.llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, - pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED, - attention_type: Optional[int] = llama_cpp.llama_attention_type.LLAMA_ATTENTION_TYPE_UNSPECIFIED, - flash_attn_type: Optional[int] = llama_cpp.llama_flash_attn_type.LLAMA_FLASH_ATTN_TYPE_AUTO, + ] = llama_cpp_lib.llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, + pooling_type: int = llama_cpp_lib.LLAMA_POOLING_TYPE_UNSPECIFIED, + attention_type: Optional[int] = llama_cpp_lib.llama_attention_type.LLAMA_ATTENTION_TYPE_UNSPECIFIED, + flash_attn_type: Optional[int] = llama_cpp_lib.llama_flash_attn_type.LLAMA_FLASH_ATTN_TYPE_AUTO, rope_freq_base: float = 0.0, rope_freq_scale: float = 0.0, yarn_ext_factor: float = -1.0, @@ -174,7 +181,14 @@ def __init__( Args: model_path: Path to the model. - n_gpu_layers: Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded. + n_gpu_layers: Max number of model layers to store in VRAM (-ngl). + Accepts an exact integer, "auto", or "all". + "auto" / -1 lets llama.cpp choose automatically. + "all" / -2 stores all possible layers in VRAM. + 0 disables model layer offload. + cpu_moe: Keep all Mixture of Experts (MoE) weights in the CPU + n_cpu_moe: Keep the MoE expert weights of the first N layers on CPU. + Useful when VRAM is insufficient for MoE models. split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. @@ -237,40 +251,38 @@ def __init__( if not Llama.__backend_initialized: with suppress_stdout_stderr(disable=verbose): - llama_cpp.llama_backend_init() + llama_cpp_lib.llama_backend_init() Llama.__backend_initialized = True if isinstance(numa, bool): self.numa = ( - llama_cpp.GGML_NUMA_STRATEGY_DISTRIBUTE + llama_cpp_lib.GGML_NUMA_STRATEGY_DISTRIBUTE if numa - else llama_cpp.GGML_NUMA_STRATEGY_DISABLED + else llama_cpp_lib.GGML_NUMA_STRATEGY_DISABLED ) else: self.numa = numa - if self.numa != llama_cpp.GGML_NUMA_STRATEGY_DISABLED: + if self.numa != llama_cpp_lib.GGML_NUMA_STRATEGY_DISABLED: with suppress_stdout_stderr(disable=verbose): - llama_cpp.llama_numa_init(self.numa) + llama_cpp_lib.llama_numa_init(self.numa) self.model_path = model_path # Model Params - self.model_params = llama_cpp.llama_model_default_params() - self.model_params.n_gpu_layers = ( - 0x7FFFFFFF if n_gpu_layers == -1 else n_gpu_layers - ) # 0x7FFFFFFF is INT32 max, will be auto set to all layers + self.model_params = llama_cpp_lib.llama_model_default_params() + self.model_params.n_gpu_layers = self._parse_n_gpu_layers(n_gpu_layers) self.model_params.split_mode = split_mode self.model_params.main_gpu = main_gpu self.tensor_split = tensor_split self._c_tensor_split = None if self.tensor_split is not None: - if len(self.tensor_split) > llama_cpp.LLAMA_MAX_DEVICES: + if len(self.tensor_split) > llama_cpp_lib.LLAMA_MAX_DEVICES: raise ValueError( - f"Attempt to split tensors that exceed maximum supported devices. Current LLAMA_MAX_DEVICES={llama_cpp.LLAMA_MAX_DEVICES}" + f"Attempt to split tensors that exceed maximum supported devices. Current LLAMA_MAX_DEVICES={llama_cpp_lib.LLAMA_MAX_DEVICES}" ) # Type conversion and expand the list to the length of LLAMA_MAX_DEVICES - FloatArray = ctypes.c_float * llama_cpp.LLAMA_MAX_DEVICES + FloatArray = ctypes.c_float * llama_cpp_lib.LLAMA_MAX_DEVICES self._c_tensor_split = FloatArray( *tensor_split # type: ignore ) # keep a reference to the array so it is not gc'd @@ -283,13 +295,61 @@ def __init__( self.model_params.use_extra_bufts = use_extra_bufts self.model_params.no_host = no_host + # Logic of cpu_moe, n_cpu_moe + # Reference from llama.cpp/tools/llama-bench/llama-bench.cpp + self.cpu_moe = cpu_moe + self.n_cpu_moe = n_cpu_moe + self._cpu_moe_patterns = None + self._cpu_moe_tensor_buft_overrides = None + + if self.n_cpu_moe < 0: + raise ValueError("n_cpu_moe must be >= 0") + + if self.cpu_moe and self.n_cpu_moe != 0 and self.verbose: + print( + "Llama.__init__: cpu_moe=True already keeps all MoE expert weights on CPU; " + "n_cpu_moe is redundant.", + file=sys.stderr, + ) + + if self.cpu_moe or self.n_cpu_moe > 0: + cpu_buft = ggml_backend_cpu_buffer_type() + + if self.cpu_moe: + patterns = [self.LLM_FFN_EXPS_REGEX] + else: + patterns = [ + self._make_cpu_moe_pattern(i) + for i in range(self.n_cpu_moe) + ] + + # keep pattern bytes alive + self._cpu_moe_patterns = patterns + + TensorBuftOverrideArray = ( + llama_cpp_lib.llama_model_tensor_buft_override + * (len(patterns) + 1) + ) + self._cpu_moe_tensor_buft_overrides = TensorBuftOverrideArray() + + for i, pattern in enumerate(self._cpu_moe_patterns): + self._cpu_moe_tensor_buft_overrides[i].pattern = pattern + self._cpu_moe_tensor_buft_overrides[i].buft = cpu_buft + + self._cpu_moe_tensor_buft_overrides[len(patterns)].pattern = None + self._cpu_moe_tensor_buft_overrides[len(patterns)].buft = None + + self.model_params.tensor_buft_overrides = ( + self._cpu_moe_tensor_buft_overrides + ) + # kv_overrides is the original python dict self.kv_overrides = kv_overrides if kv_overrides is not None: # _kv_overrides_array is a ctypes.Array of llama_model_kv_override Structs kvo_array_len = len(kv_overrides) + 1 # for sentinel element self._kv_overrides_array = ( - llama_cpp.llama_model_kv_override * kvo_array_len + llama_cpp_lib.llama_model_kv_override * kvo_array_len )() for i, (k, v) in enumerate(kv_overrides.items()): @@ -297,17 +357,17 @@ def __init__( if isinstance(v, bool): self._kv_overrides_array[ i - ].tag = llama_cpp.LlamaModelKVOverrideType.LLAMA_KV_OVERRIDE_TYPE_BOOL.value + ].tag = llama_cpp_lib.LlamaModelKVOverrideType.LLAMA_KV_OVERRIDE_TYPE_BOOL.value self._kv_overrides_array[i].value.val_bool = v elif isinstance(v, int): self._kv_overrides_array[ i - ].tag = llama_cpp.LlamaModelKVOverrideType.LLAMA_KV_OVERRIDE_TYPE_INT.value + ].tag = llama_cpp_lib.LlamaModelKVOverrideType.LLAMA_KV_OVERRIDE_TYPE_INT.value self._kv_overrides_array[i].value.val_i64 = v elif isinstance(v, float): self._kv_overrides_array[ i - ].tag = llama_cpp.LlamaModelKVOverrideType.LLAMA_KV_OVERRIDE_TYPE_FLOAT.value + ].tag = llama_cpp_lib.LlamaModelKVOverrideType.LLAMA_KV_OVERRIDE_TYPE_FLOAT.value self._kv_overrides_array[i].value.val_f64 = v elif isinstance(v, str): # type: ignore v_bytes = v.encode("utf-8") @@ -316,12 +376,12 @@ def __init__( v_bytes = v_bytes.ljust(128, b"\0") self._kv_overrides_array[ i - ].tag = llama_cpp.LlamaModelKVOverrideType.LLAMA_KV_OVERRIDE_TYPE_STR.value + ].tag = llama_cpp_lib.LlamaModelKVOverrideType.LLAMA_KV_OVERRIDE_TYPE_STR.value # copy min(v_bytes, 128) to str_value address = typing.cast( int, ctypes.addressof(self._kv_overrides_array[i].value) - + llama_cpp.llama_model_kv_override_value.val_str.offset, + + llama_cpp_lib.llama_model_kv_override_value.val_str.offset, ) buffer_start = ctypes.cast(address, ctypes.POINTER(ctypes.c_char)) ctypes.memmove( @@ -344,10 +404,10 @@ def __init__( self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count() # Used by the sampler - self._seed = seed or llama_cpp.LLAMA_DEFAULT_SEED + self._seed = seed or llama_cpp_lib.LLAMA_DEFAULT_SEED # Context Params - self.context_params = llama_cpp.llama_context_default_params() + self.context_params = llama_cpp_lib.llama_context_default_params() self.context_params.n_ctx = n_ctx self.context_params.n_batch = self.n_batch self.context_params.n_ubatch = min(self.n_batch, n_ubatch) @@ -357,22 +417,22 @@ def __init__( self.context_params.rope_scaling_type = ( rope_scaling_type if rope_scaling_type is not None - else llama_cpp.llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED + else llama_cpp_lib.llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED ) self.context_params.pooling_type = ( pooling_type if pooling_type is not None - else llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED + else llama_cpp_lib.LLAMA_POOLING_TYPE_UNSPECIFIED ) self.context_params.attention_type = ( attention_type if attention_type is not None - else llama_cpp.llama_attention_type.LLAMA_ATTENTION_TYPE_UNSPECIFIED + else llama_cpp_lib.llama_attention_type.LLAMA_ATTENTION_TYPE_UNSPECIFIED ) self.context_params.flash_attn_type = ( flash_attn_type if flash_attn_type is not None - else llama_cpp.llama_flash_attn_type.LLAMA_FLASH_ATTN_TYPE_AUTO + else llama_cpp_lib.llama_flash_attn_type.LLAMA_FLASH_ATTN_TYPE_AUTO ) self.context_params.rope_freq_base = ( rope_freq_base if rope_freq_base != 0.0 else 0 @@ -517,7 +577,7 @@ def __init__( ) if self.verbose: - print(llama_cpp.llama_print_system_info().decode("utf-8"), file=sys.stderr) + print(llama_cpp_lib.llama_print_system_info().decode("utf-8"), file=sys.stderr) self.chat_format = chat_format self.chat_handler = chat_handler @@ -539,11 +599,6 @@ def __init__( self.input_ids: npt.NDArray[np.intc] = np.ndarray((n_ctx,), dtype=np.intc) self.scores: npt.NDArray[np.single] = np.ndarray((n_ctx if self._logits_all else 1, self._n_vocab), dtype=np.single) - - self._mirostat_mu = ctypes.c_float( - 2.0 * 5.0 - ) # TODO: Move this to sampling context - try: self.metadata = self._model.metadata() except Exception as e: @@ -673,12 +728,34 @@ def close(self) -> None: def __del__(self) -> None: self.close() + @staticmethod + def _parse_n_gpu_layers(n_gpu_layers: Union[int, str]) -> int: + if isinstance(n_gpu_layers, str): + value = n_gpu_layers.strip().lower() + if value == "auto": + return -1 + if value == "all": + return -2 + try: + return int(value) + except ValueError as exc: + raise ValueError("n_gpu_layers must be an int, 'auto', or 'all'") from exc + + if isinstance(n_gpu_layers, int): + return n_gpu_layers + + raise TypeError("n_gpu_layers must be an int, 'auto', or 'all'") + + @staticmethod + def _make_cpu_moe_pattern(i: int) -> bytes: + return f"blk\\.{i}".encode("utf-8") + Llama.LLM_FFN_EXPS_REGEX + @property - def ctx(self) -> llama_cpp.llama_context_p: + def ctx(self) -> llama_cpp_lib.llama_context_p: return self._ctx.ctx @property - def model(self) -> llama_cpp.llama_model_p: + def model(self) -> llama_cpp_lib.llama_model_p: return self._model.model @property @@ -1016,12 +1093,12 @@ def eval( self.scores[0, :] = logits_view # Helper method: Convert dict logit_bias to List[llama_logit_bias] - def _convert_logit_bias(self, logit_bias: Optional[Dict[int, float]]) -> List[llama_cpp.llama_logit_bias]: + def _convert_logit_bias(self, logit_bias: Optional[Dict[int, float]]) -> List[llama_cpp_lib.llama_logit_bias]: if not logit_bias: return [] bias_list = [] for token, bias in logit_bias.items(): - lb = llama_cpp.llama_logit_bias() + lb = llama_cpp_lib.llama_logit_bias() lb.token = token lb.bias = bias bias_list.append(lb) @@ -1133,7 +1210,7 @@ def sample( # LogitsProcessor Adapter if logits_processor: - def adapter(token_data_array: llama_cpp.llama_token_data_array): + def adapter(token_data_array: llama_cpp_lib.llama_token_data_array): if self._logits_all: current_scores = self._scores[self.n_tokens - 1, :] else: @@ -1402,7 +1479,7 @@ def generate( # Register custom python-level logits processors if provided if logits_processor: - def adapter(token_data_array: llama_cpp.llama_token_data_array): + def adapter(token_data_array: llama_cpp_lib.llama_token_data_array): if self._logits_all: current_scores = self._scores[self.n_tokens - 1, :] else: @@ -1613,7 +1690,7 @@ def embed( # get pooling information pooling_type = self.pooling_type() - logits_all = pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE + logits_all = pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_NONE if self.context_params.embeddings is False: raise RuntimeError( @@ -1621,7 +1698,7 @@ def embed( ) if self.verbose: - llama_cpp.llama_perf_context_reset(self._ctx.ctx) + llama_cpp_lib.llama_perf_context_reset(self._ctx.ctx) if isinstance(input, str): inputs = [input] @@ -1635,15 +1712,15 @@ def embed( data: Union[List[List[float]], List[List[List[float]]]] = [] def decode_batch(seq_sizes: List[int]): - llama_cpp.llama_memory_clear(llama_cpp.llama_get_memory(self._ctx.ctx), True) + llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(self._ctx.ctx), True) self._ctx.decode(self._batch) self._batch.reset() # store embeddings - if pooling_type == llama_cpp.LLAMA_POOLING_TYPE_NONE: + if pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_NONE: pos: int = 0 for i, size in enumerate(seq_sizes): - ptr = llama_cpp.llama_get_embeddings(self._ctx.ctx) + ptr = llama_cpp_lib.llama_get_embeddings(self._ctx.ctx) embedding: List[List[float]] = [ ptr[pos + j * n_embd : pos + (j + 1) * n_embd] for j in range(size) @@ -1656,7 +1733,7 @@ def decode_batch(seq_sizes: List[int]): pos += size else: for i in range(len(seq_sizes)): - ptr = llama_cpp.llama_get_embeddings_seq(self._ctx.ctx, i) + ptr = llama_cpp_lib.llama_get_embeddings_seq(self._ctx.ctx, i) embedding: List[float] = ptr[:n_embd] if normalize: embedding = internals.normalize_embedding(embedding) @@ -1702,11 +1779,11 @@ def decode_batch(seq_sizes: List[int]): decode_batch(s_batch) if self.verbose: - llama_cpp.llama_perf_context_print(self._ctx.ctx) + llama_cpp_lib.llama_perf_context_print(self._ctx.ctx) output = data[0] if isinstance(input, str) else data - llama_cpp.llama_memory_clear(llama_cpp.llama_get_memory(self._ctx.ctx), True) + llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(self._ctx.ctx), True) self.reset() if return_count: @@ -1862,7 +1939,7 @@ def _create_completion( if len(prompt_tokens) >= self._n_ctx: raise ValueError( - f"Requested tokens ({len(prompt_tokens)}) exceed context window of {llama_cpp.llama_n_ctx(self.ctx)}" + f"Requested tokens ({len(prompt_tokens)}) exceed context window of {llama_cpp_lib.llama_n_ctx(self.ctx)}" ) if max_tokens is None or max_tokens <= 0: @@ -1947,7 +2024,7 @@ def _create_completion( active_loras=active_loras, control_vector=control_vector, ): - if llama_cpp.llama_token_is_eog(self._model.vocab, token): + if llama_cpp_lib.llama_token_is_eog(self._model.vocab, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) finish_reason = "stop" break @@ -2871,6 +2948,8 @@ def __getstate__(self): model_path=self.model_path, # Model Params n_gpu_layers=self.model_params.n_gpu_layers, + cpu_moe=self.cpu_moe, + n_cpu_moe=self.n_cpu_moe, split_mode=self.model_params.split_mode, main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, @@ -2932,7 +3011,7 @@ def save_state(self) -> LlamaState: print("Llama.save_state: saving llama state", file=sys.stderr) # Query the backend for the required buffer size to store the current state. - state_size = llama_cpp.llama_state_get_size(self._ctx.ctx) + state_size = llama_cpp_lib.llama_state_get_size(self._ctx.ctx) if self.verbose: print(f"Llama.save_state: got state size: {state_size}", file=sys.stderr) @@ -2943,7 +3022,7 @@ def save_state(self) -> LlamaState: # Copy the raw state data from the internal C context into our Python-managed buffer. # Returns the actual number of bytes written (n_bytes). - n_bytes = llama_cpp.llama_state_get_data(self._ctx.ctx, llama_state, state_size) + n_bytes = llama_cpp_lib.llama_state_get_data(self._ctx.ctx, llama_state, state_size) if self.verbose: print(f"Llama.save_state: copied llama state: {n_bytes}", file=sys.stderr) @@ -2995,7 +3074,7 @@ def load_state(self, state: LlamaState) -> None: # Copy the raw bytes from the Python object into a C-compatible buffer. llama_state = LLamaStateArrayType.from_buffer_copy(state.llama_state) - if llama_cpp.llama_state_set_data(self._ctx.ctx, llama_state, state_size) != state_size: + if llama_cpp_lib.llama_state_set_data(self._ctx.ctx, llama_state, state_size) != state_size: raise RuntimeError("Failed to set llama state data") def n_ctx(self) -> int: From cfa01586fb4ff56bf31a83a63dc8329c85826dfc Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 1 May 2026 08:03:29 +0800 Subject: [PATCH 050/304] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1d56558d23..c9aba7d42d 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ This package provides: - Low-level access to C API via `ctypes` interface. - [llama_cpp_lib](https://github.com/JamePeng/llama-cpp-python/blob/main/llama_cpp/llama_cpp.py) - [mtmd_cpp_lib](https://github.com/JamePeng/llama-cpp-python/blob/main/llama_cpp/mtmd_cpp.py) + - [ggml_cpp_lib](https://github.com/JamePeng/llama-cpp-python/blob/main/llama_cpp/_ggml.py) + - *Note: Synchronize ggml's ctypes calls as needed, but won't fully implement it, because most of it is called at the lower level in the upstream llama.cpp.* - High-level Python API for text completion - OpenAI-like API and Type([llama_types.py](https://github.com/JamePeng/llama-cpp-python/blob/main/llama_cpp/llama_types.py)) - [High-level API](https://github.com/JamePeng/llama-cpp-python#high-level-api) From 120c5e2d03fe77a961d9f26885d4d1c0a17edeb7 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 1 May 2026 09:39:32 +0800 Subject: [PATCH 051/304] Update /docs/wiki/Llama.md --- docs/wiki/core/Llama.md | 12 +++++++----- docs/wiki/core/LlamaEmbedding.md | 8 ++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index eacda189e3..8db9d57626 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -2,7 +2,7 @@ --- title: Llama Class class_name: Llama -last_updated: 2026-04-26 +last_updated: 2026-05-01 version_target: "latest" --- ``` @@ -17,9 +17,11 @@ Initialize the model and context. Note that model loading will immediately alloc ### Core Model & Hardware Parameters | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | -| `model_path` | `str` | **Required** | Path to the `.gguf` model file. | -| `n_gpu_layers` | `int` | `0` | Number of layers to offload to GPU. Set to `-1` for all layers. | -| `split_mode` | `int` | `LLAMA_SPLIT_MODE_LAYER` | How to split the model across GPUs (e.g., `LLAMA_SPLIT_MODE_ROW`). | +| `model_path` | `str` | **Required** | Model file path (GGUF format) | +| `n_gpu_layers` | `Union[int, Literal["auto", "all"]]` | `"auto"` | Number of model layers stored in VRAM:
ā€¢ `auto`/`-1`: auto-selected by llama.cpp
ā€¢ `all`/`-2`: all layers
ā€¢ integer N: first N layers
ā€¢ `0`: disable layer offload | +| `cpu_moe` | `bool` | `False` | Whether to keep all MoE weights on CPU | +| `n_cpu_moe` | `int` | `0` | Number of first N MoE layers to keep on CPU (compatible with `cpu_moe`) | +| `split_mode` | `int` | `LLAMA_SPLIT_MODE_LAYER` | Model GPU split mode:
ā€¢ `LLAMA_SPLIT_MODE_NONE`: single GPU
ā€¢ `LLAMA_SPLIT_MODE_ROW`: row-level split
ā€¢ `LLAMA_SPLIT_MODE_LAYER`: layer-level split | | `main_gpu` | `int` | `0` | The primary GPU to use for intermediate results or the entire model. | | `tensor_split` | `List[float]` | `None` | Proportional split of tensors across GPUs (max `LLAMA_MAX_DEVICES`). | | `use_mmap` | `bool` | `True` | Whether to use memory mapping (mmap) if possible. | @@ -309,6 +311,6 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn --- ## Related Links -* [[LlamaEmbedding]] - Dedicated class for text embeddings and reranking. +* [[LlamaEmbedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/LlamaEmbedding.md)] - Dedicated class for text embeddings and reranking. * [[ChatHandlers]] - Customizing `LlamaChatCompletionHandler` for function calling and vision/omni models (e.g., `[[Gemma4ChatHandler]]`, `[[Qwen35ChatHandler]]`). * [[LlamaCache]] - Implementing disk or RAM-based prompt caching (LlamaRAMCache, **TrieCache**, **HybridCheckpointCache**). \ No newline at end of file diff --git a/docs/wiki/core/LlamaEmbedding.md b/docs/wiki/core/LlamaEmbedding.md index b28772b45e..fd74862d99 100644 --- a/docs/wiki/core/LlamaEmbedding.md +++ b/docs/wiki/core/LlamaEmbedding.md @@ -1,7 +1,7 @@ --- title: LlamaEmbedding class_name: LlamaEmbedding -last_updated: 2026-04-27 +last_updated: 2026-05-01 version_target: "latest" --- @@ -260,4 +260,8 @@ embeddings_raw = llm.embed(["search query", "document text"], normalize=NORM_MOD - This class is in development; some features may be unstable, especially reranking model support. - Performance issues can be addressed by adjusting `n_batch`, `n_ubatch`, and `n_gpu_layers`. -- For custom models, manual `pooling_type` configuration may be required to match model behavior. \ No newline at end of file +- For custom models, manual `pooling_type` configuration may be required to match model behavior. + +## Related Links + +* [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] From b875c9a1c5b68284a5badf60d797855dc50cbc2c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 2 May 2026 09:17:18 +0800 Subject: [PATCH 052/304] Update Submodule vendor/llama.cpp 660b1b4..b97ebdc --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 660b1b4bdc..b97ebdc98f 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 660b1b4bdc6fedc18e8c3d87a945ffb51f91c547 +Subproject commit b97ebdc98f6053604a19d861c08d8087601b96e0 From ef618a696f4eb4dfbd1344953d3f603713165cf5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 2 May 2026 11:12:01 +0800 Subject: [PATCH 053/304] Update docs/wiki/modules/LlamaCache.md and Separate the modules folder --- docs/wiki/core/Llama.md | 6 +- docs/wiki/modules/LlamaCache.md | 1392 +++++++++++++++++ .../wiki/{core => modules}/LlamaChatFormat.md | 0 .../{core => modules}/LlamaCppBindings.md | 0 docs/wiki/{core => modules}/LlamaEmbedding.md | 6 +- .../wiki/{core => modules}/MTMDCppBindings.md | 0 6 files changed, 1400 insertions(+), 4 deletions(-) create mode 100644 docs/wiki/modules/LlamaCache.md rename docs/wiki/{core => modules}/LlamaChatFormat.md (100%) rename docs/wiki/{core => modules}/LlamaCppBindings.md (100%) rename docs/wiki/{core => modules}/LlamaEmbedding.md (98%) rename docs/wiki/{core => modules}/MTMDCppBindings.md (100%) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 8db9d57626..299d546e57 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -1,6 +1,8 @@ ```yaml --- title: Llama Class +module_name: llama_cpp.llama +source_file: llama_cpp/llama.py class_name: Llama last_updated: 2026-05-01 version_target: "latest" @@ -311,6 +313,6 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn --- ## Related Links -* [[LlamaEmbedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/LlamaEmbedding.md)] - Dedicated class for text embeddings and reranking. +* [[LlamaEmbedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] - Dedicated class for text embeddings and reranking. +* [[LlamaCache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] - Implementing disk or RAM-based prompt caching (LlamaRAMCache, **TrieCache**, **HybridCheckpointCache**). * [[ChatHandlers]] - Customizing `LlamaChatCompletionHandler` for function calling and vision/omni models (e.g., `[[Gemma4ChatHandler]]`, `[[Qwen35ChatHandler]]`). -* [[LlamaCache]] - Implementing disk or RAM-based prompt caching (LlamaRAMCache, **TrieCache**, **HybridCheckpointCache**). \ No newline at end of file diff --git a/docs/wiki/modules/LlamaCache.md b/docs/wiki/modules/LlamaCache.md new file mode 100644 index 0000000000..ef02fd40e4 --- /dev/null +++ b/docs/wiki/modules/LlamaCache.md @@ -0,0 +1,1392 @@ +--- +title: Llama Cache +module_name: llama_cpp.llama_cache +source_file: llama_cpp/llama_cache.py +last_updated: 2026-05-02 +version_target: "latest" +--- + +# Llama Cache + +## Overview + +`llama_cpp.llama_cache` provides cache implementations for storing and restoring `LlamaState` objects or recurrent model state checkpoints. + +The module is mainly used to speed up repeated inference workflows by reusing previously computed model state for matching token prefixes. + +It defines several cache classes: + +| Class | Purpose | +|---|---| +| `BaseLlamaCache` | Abstract base class for llama.cpp state caches. | +| `LlamaRAMCache` | In-memory LRU cache for `LlamaState` objects. | +| `LlamaDiskCache` | Disk-backed cache using the `diskcache` library. | +| `LlamaTrieCache` | Trie-based cache optimized for fast longest-prefix lookup. | +| `HybridCheckpointCache` | Checkpoint manager for RNN/Hybrid model hidden states. | +| `HybridCheckpoint` | Dataclass representing one saved hybrid model checkpoint. | +| `TrieNode` | Internal trie node used by `LlamaTrieCache`. | + +The public compatibility alias is: + +```python +LlamaCache = LlamaTrieCache +```` + +This means that code importing `LlamaCache` receives the trie-based cache implementation. + +Defined in: `llama_cpp/llama_cache.py` + +Related pages: [[Llama]], [[Caching]], [[State Save Load]], [[Hybrid Models]] + +--- + +## Role in the API + +The cache module provides reusable storage for model runtime state. + +There are two main caching strategies: + +1. **Token-prefix state caching** + + Used by: + + * `LlamaRAMCache` + * `LlamaDiskCache` + * `LlamaTrieCache` + * `LlamaCache` + + These caches map token sequences to `llama_core.LlamaState` objects. When queried, they do not require an exact match. Instead, they return the state associated with the longest cached token prefix. + +2. **Hybrid / recurrent checkpoint caching** + + Used by: + + * `HybridCheckpoint` + * `HybridCheckpointCache` + + This is designed for Hybrid or recurrent models where rollback requires saving and restoring hidden state snapshots through low-level llama.cpp state APIs. + +--- + +## Public API Summary + +| API | Type | Public | Description | +| ----------------------- | -------------- | -------: | ----------------------------------------------- | +| `BaseLlamaCache` | Abstract class | Yes | Base interface for cache implementations. | +| `LlamaRAMCache` | Class | Yes | In-memory LRU cache with linear prefix lookup. | +| `LlamaDiskCache` | Class | Yes | Disk-backed cache using `diskcache.Cache`. | +| `LlamaTrieCache` | Class | Yes | Trie-based cache with efficient prefix lookup. | +| `LlamaCache` | Alias | Yes | Backward-compatible alias for `LlamaTrieCache`. | +| `HybridCheckpoint` | Dataclass | Yes | Represents one saved Hybrid/RNN checkpoint. | +| `HybridCheckpointCache` | Class | Yes | Manages Hybrid/RNN state checkpoints. | +| `TrieNode` | Class | Internal | Trie node used by `LlamaTrieCache`. | + +--- + +# `BaseLlamaCache` + +## Overview + +`BaseLlamaCache` is the abstract base class for llama.cpp cache implementations. + +It defines a common dictionary-like interface for storing and retrieving `llama_core.LlamaState` objects by token sequence. + +Subclasses are expected to implement: + +* `cache_size` +* `__getitem__` +* `__contains__` +* `__setitem__` + +Defined in: `llama_cpp/llama_cache.py` + +--- + +## Role in the API + +`BaseLlamaCache` acts as the shared contract for cache implementations used by higher-level llama-cpp-python runtime code. + +It is not intended to be used directly. Users should instantiate one of the concrete cache classes instead: + +* `LlamaRAMCache` +* `LlamaDiskCache` +* `LlamaTrieCache` +* `LlamaCache` + +--- + +## Constructor: `__init__` + +```python +def __init__(self, capacity_bytes: int = (2 << 30)): + ... +``` + +| Parameter | Type | Default | Required | Description | +| ---------------- | ----- | --------: | -------: | -------------------------------------------------------------------- | +| `capacity_bytes` | `int` | `2 << 30` | No | Maximum cache capacity in bytes. The default is approximately 2 GiB. | + +--- + +## Instance Variables + +| Name | Type | Description | +| ---------------- | ----- | ------------------------------------------------------------------------------------------------------------ | +| `capacity_bytes` | `int` | Maximum allowed cache size in bytes. Concrete subclasses use this value to decide when eviction is required. | + +--- + +## Properties + +### `cache_size` + +```python +@property +@abstractmethod +def cache_size(self) -> int: + ... +``` + +Returns the current cache size in bytes. + +Concrete implementations define how this value is calculated. + +--- + +## Core Methods + +### `_find_longest_prefix_key` + +```python +def _find_longest_prefix_key( + self, + key: Tuple[int, ...], +) -> Optional[Tuple[int, ...]]: + ... +``` + +Finds the cached key with the longest token prefix matching the requested key. + +In `BaseLlamaCache`, this method is only a placeholder and does not implement behavior. + +Concrete subclasses may override it. + +--- + +### `__getitem__` + +```python +@abstractmethod +def __getitem__(self, key: Sequence[int]) -> "llama_core.LlamaState": + ... +``` + +Retrieves a cached `LlamaState`. + +The expected behavior is longest-prefix matching rather than strict exact-key lookup. + +--- + +### `__contains__` + +```python +@abstractmethod +def __contains__(self, key: Sequence[int]) -> bool: + ... +``` + +Returns whether the cache contains a matching token prefix for the given key. + +--- + +### `__setitem__` + +```python +@abstractmethod +def __setitem__( + self, + key: Sequence[int], + value: "llama_core.LlamaState" +) -> None: + ... +``` + +Stores a `LlamaState` under a token sequence. + +--- + +# `LlamaRAMCache` + +## Overview + +`LlamaRAMCache` is an in-memory cache for `llama_core.LlamaState` objects. + +It stores token sequences in an `OrderedDict` and maintains an LRU eviction policy. Lookup is based on the longest cached token prefix. + +Defined in: `llama_cpp/llama_cache.py` + +--- + +## Role in the API + +`LlamaRAMCache` is useful when users want fast in-process caching without writing state to disk. + +It keeps all cached states in Python memory. This makes retrieval simple, but memory usage can grow quickly depending on the size of saved `LlamaState` objects. + +--- + +## Constructor: `__init__` + +```python +def __init__(self, capacity_bytes: int = (2 << 30), verbose: bool = False): + ... +``` + +| Parameter | Type | Default | Required | Description | +| ---------------- | ------ | --------: | -------: | ----------------------------------------------------------------------------------------------------------------------------- | +| `capacity_bytes` | `int` | `2 << 30` | No | Maximum total size of cached states in bytes. | +| `verbose` | `bool` | `False` | No | Whether to enable verbose behavior when computing token-prefix matches. This value is passed to `Llama.longest_token_prefix`. | + +--- + +## Instance Variables + +| Name | Type | Description | +| ---------------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `capacity_bytes` | `int` | Maximum cache capacity in bytes. | +| `cache_state` | `OrderedDict[Tuple[int, ...], llama_core.LlamaState]` | Stores cached token sequences and their corresponding `LlamaState` objects. The order is used for LRU eviction. | +| `_current_size` | `int` | Current total size of cached states in bytes. | +| `verbose` | `bool` | Passed to `llama_core.Llama.longest_token_prefix` during prefix comparison. | + +--- + +## Properties + +### `cache_size` + +```python +@property +def cache_size(self): + return self._current_size +``` + +Returns the current tracked memory usage of the cache in bytes. + +--- + +## Core Methods + +### `_find_longest_prefix_key` + +```python +def _find_longest_prefix_key( + self, + key: Tuple[int, ...], +) -> Optional[Tuple[int, ...]]: + ... +``` + +Finds the cached token sequence with the longest prefix match against `key`. + +This implementation scans every key in `cache_state` and calls: + +```python +llama_core.Llama.longest_token_prefix(k, key, self.verbose) +``` + +### Complexity + +| Operation | Complexity | +| ------------- | ---------: | +| Prefix lookup | `O(N * K)` | +| LRU update | `O(1)` | +| Size tracking | `O(1)` | + +Where: + +* `N` is the number of cached entries. +* `K` is the token sequence length. + +--- + +### `__getitem__` + +```python +def __getitem__(self, key: Sequence[int]) -> "llama_core.LlamaState": + ... +``` + +Returns the cached `LlamaState` for the longest matching token prefix. + +Behavior: + +1. Raises `KeyError("Cache is empty")` if the cache has no entries. +2. Converts the input key to a tuple. +3. Finds the longest cached prefix. +4. Raises `KeyError("Key not found")` if no matching prefix exists. +5. Moves the matched key to the end of `cache_state` to mark it as recently used. +6. Returns the matched `LlamaState`. + +--- + +### `__contains__` + +```python +def __contains__(self, key: Sequence[int]) -> bool: + ... +``` + +Returns `True` if any cached key is a prefix match for the requested token sequence. + +Returns `False` if the cache is empty. + +--- + +### `__setitem__` + +```python +def __setitem__(self, key: Sequence[int], value: "llama_core.LlamaState"): + ... +``` + +Stores a `LlamaState` in memory. + +Behavior: + +1. Converts `key` to a tuple. +2. If the key already exists, deletes the old entry. +3. Inserts the new `LlamaState`. +4. Adds `value.llama_state_size` to `_current_size`. +5. Evicts least-recently-used entries while `_current_size > capacity_bytes`. +6. Resets `_current_size` to `0` if the cache becomes empty. + +> Note: The current implementation increments `_current_size` by the new value size when replacing an existing key, but it does not subtract the old value size before deletion. This may cause size tracking to overcount replaced entries. + +--- + +## Example + +```python +from llama_cpp import Llama +from llama_cpp.llama_cache import LlamaRAMCache + +llm = Llama( + model_path="./models/model.gguf", + cache=LlamaRAMCache(capacity_bytes=1 << 30), +) + +response = llm("Q: What is llama.cpp?\nA:", max_tokens=64) + +print(response["choices"][0]["text"]) +``` + +--- + +## Best Practices + +* Use `LlamaRAMCache` when cache speed is more important than persistence. +* Keep `capacity_bytes` below available system memory. +* Reuse the same cache instance across repeated prompts when prefix reuse is expected. +* Prefer `LlamaTrieCache` or `LlamaCache` when many cached entries are expected and prefix lookup cost matters. + +--- + +# `LlamaDiskCache` + +## Overview + +`LlamaDiskCache` is a disk-backed cache for `llama_core.LlamaState` objects. + +It delegates storage, size limits, and LRU-style eviction behavior to the external `diskcache` library. + +Defined in: `llama_cpp/llama_cache.py` + +--- + +## Role in the API + +`LlamaDiskCache` is useful when cached model states should persist beyond the current Python process or when RAM usage should be limited. + +Compared with `LlamaRAMCache`, it may reduce memory pressure but can be slower due to disk I/O. + +--- + +## Constructor: `__init__` + +```python +def __init__( + self, + cache_dir: str = ".cache/llama_cache", + capacity_bytes: int = (2 << 30), + verbose: bool = False +): + ... +``` + +| Parameter | Type | Default | Required | Description | +| ---------------- | ------ | ---------------------: | -------: | ---------------------------------------------------------------------------------------------- | +| `cache_dir` | `str` | `".cache/llama_cache"` | No | Directory used by `diskcache.Cache` to store cached state data. | +| `capacity_bytes` | `int` | `2 << 30` | No | Maximum disk cache size in bytes. Passed to `diskcache.Cache(..., size_limit=capacity_bytes)`. | +| `verbose` | `bool` | `False` | No | Passed to `Llama.longest_token_prefix` when searching for the best prefix match. | + +--- + +## Instance Variables + +| Name | Type | Description | +| ---------------- | ----------------- | ---------------------------------------------------------------------------- | +| `cache_dir` | `str` | Filesystem directory for the disk cache. | +| `cache` | `diskcache.Cache` | SQLite-backed disk cache object. | +| `verbose` | `bool` | Passed to token-prefix comparison logic. | +| `capacity_bytes` | `int` | Maximum configured cache capacity in bytes, inherited from `BaseLlamaCache`. | + +--- + +## Properties + +### `cache_size` + +```python +@property +def cache_size(self): + return self.cache.volume() +``` + +Returns the current disk cache volume in bytes using `diskcache.Cache.volume()`. + +--- + +## Core Methods + +### `_find_longest_prefix_key` + +```python +def _find_longest_prefix_key( + self, + key: Tuple[int, ...], +) -> Optional[Tuple[int, ...]]: + ... +``` + +Finds the cached key with the longest token-prefix match. + +Behavior: + +1. Returns `None` immediately if the disk cache is empty. +2. Iterates over `self.cache.iterkeys()`. +3. Uses `llama_core.Llama.longest_token_prefix(k, key, self.verbose)` to compare each cached key. +4. Stops early if a perfect match is found. + +### Complexity + +| Operation | Complexity | +| ---------------------- | ------------------------------------: | +| Prefix lookup | `O(N * K)` | +| Disk iteration | Depends on `diskcache` and filesystem | +| Exact-match early exit | Supported | + +--- + +### `__getitem__` + +```python +def __getitem__(self, key: Sequence[int]) -> "llama_core.LlamaState": + ... +``` + +Retrieves the cached state associated with the longest matching prefix. + +Behavior: + +1. Prints `"LlamaDiskCache.__getitem__: called"` to `stderr`. +2. Raises `KeyError("Cache is empty")` if no entries exist. +3. Converts `key` to a tuple. +4. Finds the longest prefix key. +5. Raises `KeyError("Key not found")` if no match exists. +6. Reads and returns the cached `LlamaState`. + +The implementation notes that this read is non-destructive and automatically updates access time for LRU behavior through `diskcache`. + +--- + +### `__contains__` + +```python +def __contains__(self, key: Sequence[int]) -> bool: + ... +``` + +Returns whether the cache has any longest-prefix match for the given token sequence. + +--- + +### `__setitem__` + +```python +def __setitem__(self, key: Sequence[int], value: "llama_core.LlamaState"): + ... +``` + +Stores a `LlamaState` in the disk cache. + +Behavior: + +1. Prints `"LlamaDiskCache.__setitem__: called"` to `stderr`. +2. Converts `key` to a tuple. +3. Assigns the value to `self.cache[tuple(key)]`. + +`diskcache` handles capacity checks and eviction. + +--- + +## Example + +```python +from llama_cpp import Llama +from llama_cpp.llama_cache import LlamaDiskCache + +cache = LlamaDiskCache( + cache_dir=".cache/llama_cache", + capacity_bytes=2 << 30, +) + +llm = Llama( + model_path="./models/model.gguf", + cache=cache, +) + +response = llm("Q: What is llama.cpp?\nA:", max_tokens=64) + +print(response["choices"][0]["text"]) +``` + +--- + +## Best Practices + +* Use `LlamaDiskCache` when cache persistence is useful. +* Place `cache_dir` on a fast local SSD when possible. +* Avoid using slow network filesystems for high-throughput inference. +* Consider `LlamaTrieCache` for workloads where many prefix lookups happen within a single process. + +--- + +## Common Pitfalls + +* Disk-backed caching can be slower than RAM caching. +* The cache depends on the third-party `diskcache` package. +* Prefix lookup still scans cached keys linearly, even though storage and eviction are handled by `diskcache`. +* The implementation prints debug messages to `stderr` on get and set operations. + +--- + +# `TrieNode` + +## Overview + +`TrieNode` is an internal helper class used by `LlamaTrieCache`. + +Each node represents one position in a token-prefix tree. + +Defined in: `llama_cpp/llama_cache.py` + +--- + +## Role in the API + +`TrieNode` is not intended to be used directly by users. + +It stores: + +* Child nodes keyed by token ID. +* An optional `LlamaState` when the node marks the end of a cached token sequence. + +--- + +## Constructor: `__init__` + +```python +def __init__(self): + ... +``` + +The constructor takes no parameters. + +--- + +## Instance Variables + +| Name | Type | Description | +| ---------- | --------------------------------- | ----------------------------------------------------------------------------------------- | +| `children` | `Dict[int, TrieNode]` | Child trie nodes keyed by token ID. | +| `state` | `Optional[llama_core.LlamaState]` | Cached state stored at this node if the node represents a complete cached token sequence. | + +--- + +# `LlamaTrieCache` + +## Overview + +`LlamaTrieCache` is a trie-based cache implementation for `llama_core.LlamaState` objects. + +It optimizes longest-prefix lookup by storing token sequences in a prefix tree rather than scanning all cached keys. + +Defined in: `llama_cpp/llama_cache.py` + +--- + +## Role in the API + +`LlamaTrieCache` is the preferred cache implementation for efficient prefix lookup. + +It combines: + +* A trie for `O(K)` longest-prefix lookup. +* An `OrderedDict` for `O(1)` LRU tracking. +* Explicit byte-size tracking through `_current_size`. + +The compatibility alias `LlamaCache` points to this class: + +```python +LlamaCache = LlamaTrieCache +``` + +--- + +## Constructor: `__init__` + +```python +def __init__(self, capacity_bytes: int = (2 << 30)): + ... +``` + +| Parameter | Type | Default | Required | Description | +| ---------------- | ----- | --------: | -------: | -------------------------------------------------------------------------------------------- | +| `capacity_bytes` | `int` | `2 << 30` | No | Maximum cache size in bytes. Entries are evicted when tracked state size exceeds this value. | + +--- + +## Instance Variables + +| Name | Type | Description | +| ---------------- | ---------------------------------------- | --------------------------------------------------------------------------------- | +| `root` | `TrieNode` | Root node of the token-prefix trie. | +| `_current_size` | `int` | Current total size of cached states in bytes. | +| `lru_tracker` | `OrderedDict[Tuple[int, ...], TrieNode]` | Tracks cached keys by recency. The value is the terminal `TrieNode` for that key. | +| `capacity_bytes` | `int` | Maximum cache capacity in bytes, inherited from `BaseLlamaCache`. | + +--- + +## Properties + +### `cache_size` + +```python +@property +def cache_size(self) -> int: + return self._current_size +``` + +Returns the current total size of cached states in bytes. + +This is an `O(1)` operation. + +--- + +## Core Methods + +### `_find_longest_prefix_node` + +```python +def _find_longest_prefix_node( + self, + key: Tuple[int, ...] +) -> Tuple[Optional[TrieNode], Optional[Tuple[int, ...]]]: + ... +``` + +Finds the trie node containing the longest cached prefix for the given token sequence. + +Returns: + +```python +Tuple[Optional[TrieNode], Optional[Tuple[int, ...]]] +``` + +The first item is the matching trie node. + +The second item is the matching cached key. + +### Behavior + +1. Starts at the root node. +2. Checks whether the empty prefix has a cached state. +3. Walks one token at a time through the trie. +4. Updates the best match each time it reaches a node with a stored state. +5. Stops when the token path no longer exists. + +### Complexity + +| Operation | Complexity | +| ------------- | ---------: | +| Prefix lookup | `O(K)` | + +Where `K` is the length of the requested token sequence. + +--- + +### `__getitem__` + +```python +def __getitem__(self, key: Sequence[int]) -> "llama_core.LlamaState": + ... +``` + +Retrieves the `LlamaState` for the longest matching cached prefix. + +Behavior: + +1. Converts `key` to a tuple. +2. Finds the longest matching trie node. +3. Raises `KeyError` if no prefix match exists. +4. Moves the matched key to the end of `lru_tracker`. +5. Returns the stored `LlamaState`. + +--- + +### `__contains__` + +```python +def __contains__(self, key: Sequence[int]) -> bool: + ... +``` + +Returns `True` if any prefix of `key` is cached. + +This lookup is `O(K)`. + +--- + +### `_prune` + +```python +def _prune(self, key: Tuple[int, ...]): + ... +``` + +Removes a cached key from the trie and prunes empty parent nodes. + +This is an internal helper used during LRU eviction. + +Behavior: + +1. Walks the trie path for the given key. +2. Returns immediately if the key does not exist. +3. Removes the stored state from the terminal node. +4. Subtracts the state size from `_current_size`. +5. Walks backward through the path and removes empty trie nodes. + +--- + +### `__setitem__` + +```python +def __setitem__(self, key: Sequence[int], value: "llama_core.LlamaState"): + ... +``` + +Stores a `LlamaState` in the trie cache. + +Behavior: + +1. Converts `key` to a tuple. +2. Creates trie nodes for each token if needed. +3. If the terminal node already has a state, subtracts the old state size. +4. Stores the new state. +5. Adds `value.llama_state_size` to `_current_size`. +6. Updates `lru_tracker`. +7. Evicts least-recently-used items while `_current_size > capacity_bytes`. + +--- + +## Example + +```python +from llama_cpp import Llama +from llama_cpp.llama_cache import LlamaCache + +llm = Llama( + model_path="./models/model.gguf", + cache=LlamaCache(capacity_bytes=2 << 30), +) + +response = llm("Q: What is llama.cpp?\nA:", max_tokens=64) + +print(response["choices"][0]["text"]) +``` + +Because `LlamaCache` is an alias for `LlamaTrieCache`, this example uses the trie-based cache. + +--- + +## Performance Characteristics + +| Cache | Prefix Lookup | LRU Tracking | Storage | +| ---------------- | ------------: | -----------------------: | ------- | +| `LlamaRAMCache` | `O(N * K)` | `O(1)` | RAM | +| `LlamaDiskCache` | `O(N * K)` | Delegated to `diskcache` | Disk | +| `LlamaTrieCache` | `O(K)` | `O(1)` | RAM | + +Where: + +* `N` is the number of cached entries. +* `K` is the token sequence length. + +--- + +## Best Practices + +* Prefer `LlamaCache` for general use, because it currently aliases `LlamaTrieCache`. +* Use `LlamaTrieCache` directly when you want explicit control over the cache implementation. +* Use a realistic `capacity_bytes` value based on available RAM. +* Use this cache when many prompts share prefixes. + +--- + +## Common Pitfalls + +* The cache still stores full `LlamaState` objects, which may be large. +* `capacity_bytes` is based on `value.llama_state_size`; this assumes each stored state reports its size accurately. +* `TrieNode` is internal and should not be manipulated directly. +* Eviction removes entries from both `lru_tracker` and the trie. + +--- + +# `LlamaCache` + +## Overview + +`LlamaCache` is a backward-compatible alias for `LlamaTrieCache`. + +```python +LlamaCache = LlamaTrieCache +``` + +This means users can import `LlamaCache` and receive the trie-based implementation. + +--- + +## Example + +```python +from llama_cpp import Llama +from llama_cpp.llama_cache import LlamaCache + +cache = LlamaCache(capacity_bytes=2 << 30) + +llm = Llama( + model_path="./models/model.gguf", + cache=cache, +) +``` + +--- + +## Migration Notes + +Older code may expect `LlamaCache` to refer to another cache implementation. + +In the current source, `LlamaCache` resolves to `LlamaTrieCache`. + +When documenting or debugging cache behavior, treat `LlamaCache` as equivalent to: + +```python +from llama_cpp.llama_cache import LlamaTrieCache as LlamaCache +``` + +--- + +# `HybridCheckpoint` + +## Overview + +`HybridCheckpoint` is a dataclass representing one saved snapshot of a Hybrid or recurrent model's hidden state. + +It is used by `HybridCheckpointCache`. + +Defined in: `llama_cpp/llama_cache.py` + +--- + +## Role in the API + +Hybrid or recurrent models may require hidden-state rollback rather than standard KV-cache truncation. + +`HybridCheckpoint` stores enough metadata to verify and restore a specific recurrent state snapshot. + +--- + +## Dataclass Definition + +```python +@dataclass +class HybridCheckpoint: + pos: int + data: bytes + hash_val: str + size: int + seq_id: int +``` + +--- + +## Fields + +| Field | Type | Description | +| ---------- | ------- | --------------------------------------------------------------- | +| `pos` | `int` | Token position where this checkpoint was taken. | +| `data` | `bytes` | Raw binary RNN or Hybrid model state data. | +| `hash_val` | `str` | SHA-256 hash prefix used to verify exact token-prefix matching. | +| `size` | `int` | Size of the state data in bytes. | +| `seq_id` | `int` | Sequence ID associated with this checkpoint. | + +--- + +## Notes + +`HybridCheckpoint` objects are normally created by `HybridCheckpointCache.save_checkpoint`. + +Users usually do not need to instantiate this dataclass manually. + +--- + +# `HybridCheckpointCache` + +## Overview + +`HybridCheckpointCache` manages RNN or Hybrid model hidden-state checkpoints. + +It is designed for models that cannot physically truncate KV cache in the same way as standard transformer-only models. + +Instead of implementing dictionary-style cache operations, it provides explicit checkpoint operations: + +* `save_checkpoint` +* `find_best_checkpoint` +* `restore_checkpoint` +* `clear` +* `close` + +Defined in: `llama_cpp/llama_cache.py` + +--- + +## Role in the API + +`HybridCheckpointCache` is a specialized cache manager for Hybrid/Recurrent model rollback. + +It stores raw state snapshots extracted from the llama.cpp backend through low-level C API functions: + +* `llama_state_seq_get_size_ext` +* `llama_state_seq_get_data_ext` +* `llama_state_seq_set_data_ext` + +It is not a drop-in replacement for `LlamaRAMCache`, `LlamaDiskCache`, or `LlamaTrieCache`. + +--- + +## Constructor: `__init__` + +```python +def __init__( + self, + ctx: llama_cpp_lib.llama_context_p, + max_checkpoints: int = 16, + verbose: bool = False +): + ... +``` + +| Parameter | Type | Default | Required | Description | +| ----------------- | ------------------------------- | ------: | -------: | ------------------------------------------------------------------------------------------- | +| `ctx` | `llama_cpp_lib.llama_context_p` | ā€” | Yes | Low-level llama.cpp context pointer. Required for extracting and restoring sequence state. | +| `max_checkpoints` | `int` | `16` | No | Maximum number of checkpoints to retain. If set to `0` or below, checkpointing is disabled. | +| `verbose` | `bool` | `False` | No | Enables diagnostic messages printed to `stderr`. | + +--- + +## Constructor Behavior + +The constructor raises `ValueError` if `ctx` is `None`. + +```python +if ctx is None: + raise ValueError( + "HybridCheckpointCache(__init__): Failed to create HybridCheckpointCache with model context" + ) +``` + +If `max_checkpoints <= 0`, checkpointing is disabled. In verbose mode, the cache reports that rollback capabilities are turned off. + +This mode is intended to avoid expensive state extraction for single-turn workflows. + +--- + +## Instance Variables + +| Name | Type | Description | +| ----------------- | ------------------------------- | ------------------------------------------------------------------------------------------------ | +| `_ctx` | `llama_cpp_lib.llama_context_p` | Low-level llama.cpp context pointer used for state extraction and restoration. | +| `max_checkpoints` | `int` | Maximum number of checkpoints retained. Values less than or equal to zero disable checkpointing. | +| `checkpoints` | `list[HybridCheckpoint]` | Stored checkpoint objects. | +| `_current_size` | `int` | Total memory used by all stored checkpoints in bytes. | +| `_get_size_ext` | Callable | Cached reference to `llama_state_seq_get_size_ext`. | +| `_get_data_ext` | Callable | Cached reference to `llama_state_seq_get_data_ext`. | +| `_set_data_ext` | Callable | Cached reference to `llama_state_seq_set_data_ext`. | +| `_flag_partial` | int | Cached value of `LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY`. | +| `verbose` | `bool` | Enables debug output. | + +--- + +## Properties + +### `cache_size` + +```python +@property +def cache_size(self) -> int: + return self._current_size +``` + +Returns the total memory used by stored checkpoints in bytes. + +--- + +## Core Methods + +### `clear` + +```python +def clear(self): + ... +``` + +Clears all stored checkpoints and resets `_current_size` to `0`. + +If the checkpoint list is already empty, it returns immediately. + +In verbose mode, it prints: + +```text +HybridCheckpointCache: cleared +``` + +--- + +### `close` + +```python +def close(self): + ... +``` + +Releases references held by the cache. + +Behavior: + +* Sets `checkpoints` to `None`. +* Sets `_ctx` to `None`. +* Sets cached C API function references to `None`. + +This method is also called by `__del__`. + +--- + +### `__del__` + +```python +def __del__(self) -> None: + self.close() +``` + +Finalizer that calls `close`. + +--- + +### `_hash_prefix` + +```python +def _hash_prefix(self, tokens: List[int], length: int) -> str: + ... +``` + +Computes a SHA-256 hash for the token prefix up to `length`. + +Behavior: + +1. Returns `"empty"` if `length <= 0`. +2. Clamps `length` to the actual token list length. +3. Converts the selected token prefix into an `array.array('i')`. +4. Hashes the bytes with SHA-256. +5. Returns the first 32 hex characters. + +This hash is used to ensure checkpoints are restored only when the token prefix exactly matches. + +--- + +### `find_best_checkpoint` + +```python +def find_best_checkpoint( + self, + tokens: List[int], + seq_id: int = 0 +) -> Optional[HybridCheckpoint]: + ... +``` + +Finds the longest valid checkpoint matching the given token prefix and sequence ID. + +Returns `None` if: + +* Checkpointing is disabled. +* There are no checkpoints. +* No checkpoint matches the requested sequence ID and token prefix. + +Behavior: + +1. Skips checkpoints whose `seq_id` differs. +2. Skips checkpoints whose `pos` is greater than the current token length. +3. Verifies token-prefix integrity using `_hash_prefix`. +4. Returns the checkpoint with the largest matching `pos`. + +--- + +### `save_checkpoint` + +```python +def save_checkpoint( + self, + current_pos: int, + tokens: List[int], + seq_id: int = 0 +) -> bool: + ... +``` + +Extracts the current recurrent model state from the C++ backend and stores it as a `HybridCheckpoint`. + +Returns `True` if the checkpoint was saved successfully. + +Returns `False` if: + +* Checkpointing is disabled. +* The backend reports state size `0`. +* State extraction writes an unexpected number of bytes. + +### Behavior + +1. Returns immediately if `max_checkpoints <= 0`. +2. Calls `_get_size_ext` to query the required state buffer size. +3. Allocates a `ctypes.c_uint8` buffer. +4. Calls `_get_data_ext` to extract state data. +5. Copies the state bytes into a Python `bytes` object. +6. Computes a hash of the token prefix. +7. Appends a new `HybridCheckpoint`. +8. Increments `_current_size`. +9. Evicts old checkpoints using FIFO order if the number of checkpoints exceeds `max_checkpoints`. + +### Important Performance Note + +The implementation intentionally bypasses checkpoint extraction when `max_checkpoints <= 0`. + +This avoids potentially large synchronous VRAM-to-RAM transfers for single-turn workflows. + +--- + +### `restore_checkpoint` + +```python +def restore_checkpoint( + self, + cp: HybridCheckpoint, + seq_id: int = 0 +) -> bool: + ... +``` + +Restores a previously saved checkpoint into the C++ backend. + +Returns `True` if restoration succeeds. + +Returns `False` if: + +* The checkpoint sequence ID does not match the requested `seq_id`. +* The current backend state size differs from the checkpoint size. +* The backend does not report the expected number of restored bytes. + +### Behavior + +1. Verifies `cp.seq_id == seq_id`. +2. Queries current expected state size from the backend. +3. Verifies it matches `cp.size`. +4. Copies checkpoint bytes into a ctypes buffer. +5. Calls `_set_data_ext` to restore the state. +6. Returns whether the number of restored bytes equals `cp.size`. + +--- + +## Disabled Dictionary Interface + +`HybridCheckpointCache` inherits from `BaseLlamaCache`, but it intentionally disables the dictionary-style methods. + +### `__getitem__` + +```python +def __getitem__(self, key): + raise NotImplementedError( + "HybridCheckpointCache: pls use save_checkpoint or restore_checkpoint method" + ) +``` + +### `__setitem__` + +```python +def __setitem__(self, key, value): + raise NotImplementedError( + "HybridCheckpointCache: pls use save_checkpoint or restore_checkpoint method" + ) +``` + +### `__contains__` + +```python +def __contains__(self, key): + raise NotImplementedError( + "HybridCheckpointCache: pls use save_checkpoint or restore_checkpoint method" + ) +``` + +Users should use checkpoint-specific methods instead. + +--- + +## Example + +```python +from llama_cpp.llama_cache import HybridCheckpointCache + +# `ctx` must be a valid llama.cpp context pointer. +checkpoint_cache = HybridCheckpointCache( + ctx=ctx, + max_checkpoints=16, + verbose=True, +) + +tokens = [1, 2, 3, 4] +current_pos = len(tokens) + +saved = checkpoint_cache.save_checkpoint( + current_pos=current_pos, + tokens=tokens, + seq_id=0, +) + +if saved: + checkpoint = checkpoint_cache.find_best_checkpoint(tokens, seq_id=0) + + if checkpoint is not None: + restored = checkpoint_cache.restore_checkpoint(checkpoint, seq_id=0) + print("Restored:", restored) +``` + +> Note: This example assumes `ctx` is already available from lower-level llama.cpp runtime code. Most high-level users do not manually create this cache. + +--- + +## Best Practices + +* Use `HybridCheckpointCache` only for Hybrid or recurrent model workflows that require hidden-state rollback. +* Set `max_checkpoints=0` for single-turn workflows where rollback is not needed. +* Keep `max_checkpoints` small if checkpoint states are large. +* Use `find_best_checkpoint` before calling `restore_checkpoint`. +* Do not use dictionary-style cache access with this class. + +--- + +## Common Pitfalls + +* Passing `ctx=None` raises `ValueError`. +* `max_checkpoints <= 0` disables checkpointing. +* Restoring a checkpoint with the wrong `seq_id` fails. +* Restore fails if the current backend state size no longer matches the checkpoint size. +* `close()` sets internal references to `None`; the object should not be reused afterward. +* This class is not equivalent to `LlamaCache`. + +--- + +# Module Variables and Constants + +## `LlamaCache` + +```python +LlamaCache = LlamaTrieCache +``` + +Backward-compatible alias for `LlamaTrieCache`. + +Users can import either: + +```python +from llama_cpp.llama_cache import LlamaCache +``` + +or: + +```python +from llama_cpp.llama_cache import LlamaTrieCache +``` + +Both refer to the trie-based cache implementation in the current source. + +--- + +# How the Cache Implementations Compare + +| Class | Storage | Prefix Lookup | Eviction | Persistence | Best For | +| ----------------------- | ------- | ------------------------------: | ------------------------ | ----------: | -------------------------------------------- | +| `LlamaRAMCache` | RAM | `O(N * K)` | LRU | No | Small in-memory caches. | +| `LlamaDiskCache` | Disk | `O(N * K)` | Delegated to `diskcache` | Yes | Persistent cache across runs. | +| `LlamaTrieCache` | RAM | `O(K)` | LRU | No | Fast prefix lookup with many cached entries. | +| `HybridCheckpointCache` | RAM | Hash-verified checkpoint search | FIFO by checkpoint count | No | Hybrid/Recurrent model rollback. | + +--- + +# Recommended Entry Points + +For most users: + +```python +from llama_cpp.llama_cache import LlamaCache +``` + +This currently gives the trie-based implementation. + +For explicit cache selection: + +```python +from llama_cpp.llama_cache import LlamaRAMCache +from llama_cpp.llama_cache import LlamaDiskCache +from llama_cpp.llama_cache import LlamaTrieCache +``` + +For Hybrid/Recurrent models: + +```python +from llama_cpp.llama_cache import HybridCheckpointCache +``` + +--- + +# Related Links + +* [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] diff --git a/docs/wiki/core/LlamaChatFormat.md b/docs/wiki/modules/LlamaChatFormat.md similarity index 100% rename from docs/wiki/core/LlamaChatFormat.md rename to docs/wiki/modules/LlamaChatFormat.md diff --git a/docs/wiki/core/LlamaCppBindings.md b/docs/wiki/modules/LlamaCppBindings.md similarity index 100% rename from docs/wiki/core/LlamaCppBindings.md rename to docs/wiki/modules/LlamaCppBindings.md diff --git a/docs/wiki/core/LlamaEmbedding.md b/docs/wiki/modules/LlamaEmbedding.md similarity index 98% rename from docs/wiki/core/LlamaEmbedding.md rename to docs/wiki/modules/LlamaEmbedding.md index fd74862d99..57c0788e3f 100644 --- a/docs/wiki/core/LlamaEmbedding.md +++ b/docs/wiki/modules/LlamaEmbedding.md @@ -1,11 +1,13 @@ --- -title: LlamaEmbedding +title: Llama Embedding +module_name: llama_cpp.llama_embedding +source_file: llama_cpp/llama_embedding.py class_name: LlamaEmbedding last_updated: 2026-05-01 version_target: "latest" --- -# LlamaEmbedding +# Llama Embedding ## Overview diff --git a/docs/wiki/core/MTMDCppBindings.md b/docs/wiki/modules/MTMDCppBindings.md similarity index 100% rename from docs/wiki/core/MTMDCppBindings.md rename to docs/wiki/modules/MTMDCppBindings.md From 4e58a6363d7e233c2992560aa3abd866ca6305f4 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 2 May 2026 11:25:14 +0800 Subject: [PATCH 054/304] docs: update LLM wiki schema to v0.3 - Add schema metadata, documentation language rules, expanded page templates, attribute/state documentation guidance, and clearer update rules for LLM-maintained llama-cpp-python wiki pages. Signed-off-by: JamePeng --- docs/wiki/SCHEMA.md | 72 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/docs/wiki/SCHEMA.md b/docs/wiki/SCHEMA.md index f5676442cb..b96ec964c7 100644 --- a/docs/wiki/SCHEMA.md +++ b/docs/wiki/SCHEMA.md @@ -1,45 +1,103 @@ # LLM Wiki Schema ā€“ llama-cpp-python -**Purpose**: Maintain a living, always-up-to-date, structured documentation wiki for the llama-cpp-python library using LLMs as the primary maintainer. +**Schema Metadata**: +- **Author**: JamePeng +- **Maintainer**: LLM-assisted documentation workflow +- **Project**: [llama-cpp-python](https://github.com/JamePeng/llama-cpp-python) wiki +- **Last Modified**: 2026-05-02 +- **Version Target**: latest source code +- **Schema Version**: 0.3 + +**Purpose**: +- Maintain a living, always-up-to-date, structured documentation wiki for the `llama-cpp-python` library, with LLMs acting as the primary documentation maintainer. +- The wiki must help users understand the latest public API, core classes, modules, configuration options, examples, and migration paths based on the current source code. +- The wiki should explain not only *how to call an API*, but also *what role the class/module plays in the library*, *how its state is configured*, and *how users should choose between related APIs*. **Core Principles**: -- The source of truth is the latest code in `llama_cpp/` (especially `llama.py`, `llama_chat_format.py`, `llama_cpp.py`, `llama_types.py`, `llama_embedding.py`, `mtmd_cpp.py`, `_internals.py`, `_ggml.py`). +- The source of truth is the latest code in `llama_cpp/`, especially: + - `llama.py` + - `_internals.py` + - `llama_chat_format.py` + - `llama_cache.py` + - `llama_embedding.py` + - `llama_types.py` + - `llama_cpp.py` + - `mtmd_cpp.py` + - `_ggml.py` - Never invent parameters or behavior. Always read the current source code before writing/updating a page. +- Prefer documenting public and user-facing APIs first. Internal implementation details may be documented only when they help users understand behavior, extension points, debugging, or advanced usage. - All examples must be complete, runnable with the latest API, and include necessary imports. -- Clearly mark any deprecated/old usage with a warning and show the modern replacement. +- Clearly mark deprecated, legacy, or changed usage with a warning and show the modern replacement. - Use internal wiki links (e.g. [[Llama]], [[Qwen35ChatHandler]]) for cross-referencing. - Keep pages concise, professional, and user-friendly. +**Documentation Language**: +- The default documentation language is **English**. +- All generated wiki pages, examples, explanations, titles, tables, and warnings should be written in English unless the user explicitly requests another language. +- Code comments inside examples should also be in English by default. +- If the source code contains Chinese comments or non-English notes, translate them into clear English while preserving the original meaning. + **Page Types and Templates**: -1. **Class / Module Page** (e.g. core/Llama.md) +1. **Class / Module Page** (e.g. core/Llama.md, modules/LlamaEmbedding.md) - Frontmatter (YAML): ```yaml --- title: Llama Class class_name: Llama + source_file: llama_cpp/llama.py last_updated: YYYY-MM-DD version_target: "latest" --- ``` - Sections (in order): - Overview + - Role in the Library - Constructor (`__init__`) ā€“ full parameter table with types, defaults, and explanations - - Core Methods (with signatures and examples) + - Important Attributes / State + - Core Methods (with signatures and usage examples) - Best Practices & Common Patterns - Deprecated / Changed APIs (with migration notes) - Related Links + - The **Overview** should briefly explain: + - What the class or module is. + - What problem it solves. + - Whether it is a high-level public API, extension point, helper, or internal implementation detail. + - When users should use it. + + - The **Role in the Library** should explain how the class or module relates to nearby APIs. For example, whether it wraps low-level bindings, handles chat formatting, manages cache state, provides embeddings, or connects to multimodal behavior. + + - Constructor parameter tables should use: + + | Parameter | Type | Default | Description | + |---|---|---|---| + + - Important attributes or state should use: + + | Attribute | Type | Source | Description | + |---|---|---|---| + + - Only document attributes that affect user understanding, configuration, lifecycle, inference behavior, caching, chat formatting, embeddings, or debugging. Do not document every trivial private variable. + 2. **Feature Page** (features/xxx.md) - - Overview, When to use, Code examples, Limitations, Related features + - Overview, When to use, Related APIs, Code examples, Configuration Notes, Limitations, Related features + - Feature pages should explain workflows across multiple classes or modules. 3. **Example Page** (examples/xxx.md) - Goal, Prerequisites, Complete runnable code block, Expected output, Tips + - Rules: + * Use the latest API. + * Include all imports as need. + * Avoid pseudo-code. + * Keep examples focused. + * Mention required model assumptions when needed, such as GGUF file path or chat format. **Update Rules**: - Before updating any page, the LLM must read the relevant source files. - Update the `last_updated` date. -- If a new feature (e.g. new ChatHandler, new sampler) appears in code, create or expand the corresponding page. +- If a new feature appears, such as a new chat handler, sampler, cache type, embedding API, multimodal API, or backend option, create or expand the corresponding page. +- If behavior is inferred from implementation rather than explicitly documented in code, mark the explanation as implementation-based. - Maintain a high standard of readability and accuracy. This schema is the contract. All generated content must follow it. \ No newline at end of file From 50aafd47383656acda8edd514298205281d85ba9 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 2 May 2026 11:51:47 +0800 Subject: [PATCH 055/304] Upload /docs/wiki/modules/LlamaSpeculative.md for llama_speculative.py Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 5 +- docs/wiki/modules/LlamaSpeculative.md | 237 ++++++++++++++++++++++++++ 2 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 docs/wiki/modules/LlamaSpeculative.md diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 299d546e57..f010dd0c67 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -313,6 +313,7 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn --- ## Related Links -* [[LlamaEmbedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] - Dedicated class for text embeddings and reranking. -* [[LlamaCache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] - Implementing disk or RAM-based prompt caching (LlamaRAMCache, **TrieCache**, **HybridCheckpointCache**). +* [[Llama Cache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] - Implementing disk or RAM-based prompt caching (LlamaRAMCache, **TrieCache**, **HybridCheckpointCache**). +* [[Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] - Dedicated class for text embeddings and reranking. +* [[Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] - Provides draft model interfaces and prompt-based speculative decoding helpers. * [[ChatHandlers]] - Customizing `LlamaChatCompletionHandler` for function calling and vision/omni models (e.g., `[[Gemma4ChatHandler]]`, `[[Qwen35ChatHandler]]`). diff --git a/docs/wiki/modules/LlamaSpeculative.md b/docs/wiki/modules/LlamaSpeculative.md new file mode 100644 index 0000000000..d1463883b9 --- /dev/null +++ b/docs/wiki/modules/LlamaSpeculative.md @@ -0,0 +1,237 @@ +--- +title: Llama Speculative Decoding +module_name: llama_cpp.llama_speculative +source_file: llama_cpp/llama_speculative.py +last_updated: 2026-05-02 +version_target: "latest" +--- + +# Llama Speculative Decoding + +## Overview + +`llama_speculative.py` provides draft model interfaces and prompt-based speculative decoding helpers for `llama-cpp-python`. + +Speculative decoding uses a lightweight draft model to propose candidate tokens before the main model verifies them. In this module, the draft model does not need to be a neural model. It can also be a prompt lookup decoder that predicts future tokens by finding repeated token patterns in the existing context. + +This module currently defines: + +| Class | Status | Description | +|---|---|---| +| `LlamaDraftModel` | public interface | Abstract base class for draft models used by speculative decoding. | +| `LlamaNGramMapDecoding` | public | Fast stateful n-gram map based speculative decoder. | +| `LlamaPromptLookupDecoding` | legacy public | Stateless NumPy sliding-window prompt lookup decoder. | + +## Role in the Library + +This module defines the draft-model side of speculative decoding. + +A draft model receives the current token sequence and returns predicted draft tokens. These draft tokens can then be verified by the main `Llama` model during generation. + +The module provides two prompt-based implementations: + +- `LlamaNGramMapDecoding`: optimized, stateful, hash-map based lookup. +- `LlamaPromptLookupDecoding`: older stateless NumPy sliding-window implementation. + +For new usage, prefer `LlamaNGramMapDecoding` because it incrementally maintains an n-gram index instead of scanning the full token history on every call. + +## Classes + +## `LlamaDraftModel` + +```python +class LlamaDraftModel(abc.ABC) +```` + +Abstract base class for speculative draft models. + +A draft model must implement `__call__` and return an array of predicted token IDs. + +### Method + +```python +def __call__( + self, + input_ids: npt.NDArray[np.intc], + /, + **kwargs: Any, +) -> npt.NDArray[np.intc] +``` + +| Parameter | Type | Description | +| ----------- | ---------------------- | ----------------------------------------------------------------- | +| `input_ids` | `npt.NDArray[np.intc]` | Current token sequence. | +| `**kwargs` | `Any` | Additional generation arguments. Implementations may ignore them. | + +Returns: + +| Type | Description | +| ---------------------- | -------------------------------------------- | +| `npt.NDArray[np.intc]` | Draft token IDs proposed by the draft model. | + +## `LlamaNGramMapDecoding` + +```python +class LlamaNGramMapDecoding(LlamaDraftModel) +``` + +Fast speculative decoder based on an n-gram hash map. + +This decoder maintains an internal inverted index from historical n-grams to their positions. When called with the current token sequence, it looks up the final n-gram in the history and returns the following tokens from the most recent matching context. + +### Constructor + +```python +def __init__( + self, + ngram_size: int = 3, + num_pred_tokens: int = 10, +) +``` + +| Parameter | Type | Default | Description | +| ----------------- | ----- | ------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `ngram_size` | `int` | `3` | Length of the token sequence used as the lookup key. Larger values require stricter context matches but may produce fewer hits. | +| `num_pred_tokens` | `int` | `10` | Maximum number of draft tokens to return after a matching n-gram is found. | + +### Important Attributes / State + +| Attribute | Type | Source | Description | +| ----------------- | ---------------------------------- | -------------- | -------------------------------------------------------------------------------- | +| `ngram_size` | `int` | constructor | Number of tokens used as the n-gram lookup key. | +| `num_pred_tokens` | `int` | constructor | Maximum number of predicted draft tokens to return. | +| `_ngram_map` | `Dict[Tuple[int, ...], List[int]]` | internal cache | Internal inverted index mapping n-gram tuples to positions in the token history. | +| `_history` | `List[int]` | internal cache | Internal token history used to maintain the n-gram map. | + +`_ngram_map` and `_history` are internal state and should not be modified directly. + +### Behavior + +When called, `LlamaNGramMapDecoding`: + +1. Synchronizes its internal history with the provided `input_ids`. +2. Incrementally updates the n-gram map when tokens are appended. +3. Rebuilds the map if the input sequence is no longer a simple continuation, such as after rollback or a new prompt. +4. Uses the last `ngram_size` tokens as the search key. +5. Returns up to `num_pred_tokens` tokens following the most recent historical match. +6. Returns an empty NumPy array if no match is found. + +### Example + +```python +import numpy as np +from llama_cpp.llama_speculative import LlamaNGramMapDecoding + +draft_model = LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=5, +) + +input_ids = np.array([1, 2, 3, 4, 1, 2, 3], dtype=np.intc) + +draft_tokens = draft_model(input_ids) + +print(draft_tokens) +``` + +## `LlamaPromptLookupDecoding` + +```python +class LlamaPromptLookupDecoding(LlamaDraftModel) +``` + +Legacy speculative decoder based on NumPy sliding-window lookup. + +This implementation is stateless. Each call scans the input token sequence to find previous occurrences of the current n-gram and returns the following tokens as draft predictions. + +> Warning: This implementation may have high computational overhead for long contexts. Prefer `LlamaNGramMapDecoding` for new usage. + +### Constructor + +```python +def __init__( + self, + max_ngram_size: int = 3, + num_pred_tokens: int = 10, +) +``` + +| Parameter | Type | Default | Description | +| ----------------- | ----- | ------- | -------------------------------------------------------------------------- | +| `max_ngram_size` | `int` | `3` | Maximum n-gram size to search for. The decoder tries larger n-grams first. | +| `num_pred_tokens` | `int` | `10` | Maximum number of draft tokens to return. | + +### Important Attributes / State + +| Attribute | Type | Source | Description | +| ----------------- | ----- | ----------- | --------------------------------------------------- | +| `max_ngram_size` | `int` | constructor | Maximum n-gram window size used during lookup. | +| `num_pred_tokens` | `int` | constructor | Maximum number of predicted draft tokens to return. | + +### Static Method + +```python +@staticmethod +def find_candidate_pred_tokens( + input_ids: npt.NDArray[np.intc], + max_ngram_size: int, + num_pred_tokens: int, +) +``` + +Linearly scans `input_ids` using NumPy sliding windows to find matching n-grams. + +| Parameter | Type | Description | +| ----------------- | ---------------------- | ----------------------------------------- | +| `input_ids` | `npt.NDArray[np.intc]` | Complete token sequence. | +| `max_ngram_size` | `int` | Maximum n-gram size to search for. | +| `num_pred_tokens` | `int` | Maximum number of draft tokens to return. | + +Returns: + +| Type | Description | +| ---------------------- | --------------------------------------------------------------- | +| `npt.NDArray[np.intc]` | Candidate draft tokens, or an empty array if no match is found. | + +### Example + +```python +from llama_cpp import Llama +from llama_cpp.llama_speculative import LlamaNGramMapDecoding + +llama = Llama( + model_path="path/to/qwen-3.6-27b.gguf", + n_ctx=4096, + n_gpu_layers=-1, + draft_model=LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10 + ) +) + +response = llama.create_chat_completion( + messages=[{"role": "user", "content": """ + Write a Python script using `sqlite3` to define CRUD (Create, Read, Update, Delete) operations for an e-commerce database. +You need to create 5 separate classes for the following entities: `User`, `Product`, `Order`, `Review`, and `Category`. +Each class MUST have exactly the same internal structure and method names (create, get, update, delete). Do not add extra logic, just the standard boilerplate. + """}] +) +``` + +## Best Practices & Common Patterns + +* Prefer `LlamaNGramMapDecoding` for new usage. +* Use `LlamaPromptLookupDecoding` only when compatibility with the older stateless prompt lookup behavior is needed. +* Increase `ngram_size` or `max_ngram_size` for stricter context matching. +* Increase `num_pred_tokens` when you want longer draft proposals, but keep in mind that speculative decoding still depends on later verification by the main model. +* Do not mutate `_ngram_map` or `_history` directly. +* If input token history rolls back or changes unexpectedly, `LlamaNGramMapDecoding` automatically rebuilds its internal cache. + +## Deprecated / Changed APIs + +`LlamaPromptLookupDecoding` is marked as a legacy NumPy sliding-window implementation in the source code. It is still available, but `LlamaNGramMapDecoding` is the preferred implementation for faster repeated calls over long contexts. + +## Related Links + +* [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] + From cbf15da62226ef86b3bc9cc4309f5189298379da Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 2 May 2026 12:09:59 +0800 Subject: [PATCH 056/304] Update /docs/wiki/contributing-to-wiki.md --- docs/wiki/contributing-to-wiki.md | 196 ++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/docs/wiki/contributing-to-wiki.md b/docs/wiki/contributing-to-wiki.md index e69de29bb2..ec8d6e0a27 100644 --- a/docs/wiki/contributing-to-wiki.md +++ b/docs/wiki/contributing-to-wiki.md @@ -0,0 +1,196 @@ +# Contributing to the LLM Wiki + +Thank you for helping improve the `llama-cpp-python` LLM Wiki. + +This wiki is maintained with the help of LLMs, but all documentation must stay grounded in the latest source code. The goal is to keep the wiki accurate, practical, and easy to use for both humans and LLM-based documentation workflows. + +## Documentation Source of Truth + +The source of truth is always the current code in `llama_cpp/`. + +Before creating or updating a wiki page, read the relevant source files first. Do not rely only on memory, old examples, outdated documentation, or external summaries. + +Important source files include: + +- `llama.py` +- `_internals.py` +- `llama_chat_format.py` +- `llama_cache.py` +- `llama_embedding.py` +- `llama_types.py` +- `llama_cpp.py` +- `llama_speculative.py` +- `mtmd_cpp.py` +- `_ggml.py` + +## General Rules + +When contributing documentation: + +- Use English by default. +- Keep pages concise, clear, and practical. +- Do not invent parameters, defaults, return values, or behavior. +- Include complete runnable examples when adding code samples. +- Prefer modern APIs over deprecated or legacy usage. +- Clearly mark deprecated or changed APIs with migration notes. +- Use internal wiki links such as `[[Llama]]`, `[[Chat Completion]]`, or `[[LlamaNGramMapDecoding]]`. +- Update the `last_updated` field in page frontmatter. + +## Page Structure + +Follow the project wiki schema defined in [`SCHEMA.md`](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/SCHEMA.md). + +Most pages should be one of the following types: + +### Class / Module Page + +Use this for important classes or source modules. + +Typical sections: + +- Overview +- Role in the Library +- Constructor (`__init__`) +- Important Attributes / State +- Core Methods +- Best Practices & Common Patterns +- Deprecated / Changed APIs +- Related Links + +### Feature Page + +Use this for workflows that involve multiple APIs. + +Examples: + +- Chat completion +- Text completion +- Embeddings +- Caching +- Speculative decoding +- Multimodal usage + +Typical sections: + +- Overview +- When to use +- Related APIs +- Code examples +- Configuration notes +- Limitations +- Related features + +### Example Page + +Use this for runnable examples. + +Typical sections: + +- Goal +- Prerequisites +- Complete code +- Expected output +- Tips +- Related links + +## Handling Files with Multiple Classes + +If a source file contains multiple related classes, create one module overview page first. + +Create separate class pages only when a class is: + +- Public or commonly imported by users +- Configuration-heavy +- Behavior-heavy +- A major extension point +- Likely to be searched by name + +Small helper classes, internal classes, and simple data containers can usually stay documented only on the module page. + +Avoid duplicating full documentation across module pages and class pages. + +## Documenting Parameters and Attributes + +Constructor parameters should use this format: + +| Parameter | Type | Default | Description | +|---|---|---|---| + +Important attributes or state should use this format: + +| Attribute | Type | Source | Description | +|---|---|---|---| + +Only document attributes that help users understand configuration, lifecycle, inference behavior, caching, chat formatting, embeddings, debugging, or extension points. + +Do not document every private variable. Private attributes may be mentioned only when they are useful for explaining behavior or debugging, and they should be marked as internal. + +## Code Examples + +All code examples should: + +- Include required imports. +- Be runnable with the latest API. +- Avoid pseudo-code unless explicitly marked as conceptual. +- Use clear model path placeholders such as `./model.gguf`. +- Mention assumptions such as chat format, embedding mode, GPU configuration, or required model type when relevant. + +Example: + +```python +from llama_cpp import Llama + +llm = Llama(model_path="./model.gguf") + +output = llm.create_completion("Hello,") +print(output["choices"][0]["text"]) +```` + +## Accuracy Requirements + +Do not guess. + +If behavior is not clearly documented but can be inferred from implementation, say: + +> Based on the current implementation, ... + +If an API appears internal, say: + +> This appears to be an internal implementation detail and should not be treated as a stable public API. + +If you cannot verify something from the current source code, do not include it as fact. + +## Pull Request Checklist + +Before submitting a documentation change, check that: + +* [ ] The relevant source files were reviewed. +* [ ] The page follows `SCHEMA.md`. +* [ ] Frontmatter is present and `last_updated` is updated. +* [ ] Parameters, defaults, and signatures match the source code. +* [ ] Examples are complete and runnable. +* [ ] Deprecated or legacy APIs are clearly marked. +* [ ] Internal APIs are not presented as stable public APIs. +* [ ] Related pages are linked with internal wiki links. +* [ ] The page is concise and avoids unnecessary duplication. + +## Commit Message Style + +Use simple documentation-focused commit messages. + +Examples: + +```bash +docs: add speculative decoding wiki page +docs: update Llama constructor parameters +docs: expand chat handler documentation +docs: clarify cache API usage +docs: update wiki schema to v0.3 +``` + +## Final Note + +The wiki should help users understand the latest `llama-cpp-python` API from the source code itself. + +Accuracy is more important than completeness. When in doubt, verify the code first. + From 28f842c3c0ec66eaaa30a905eebe8f291a8d92a3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 2 May 2026 12:49:47 +0800 Subject: [PATCH 057/304] docs: Update /docs/wiki/index.md Signed-off-by: JamePeng --- docs/wiki/index.md | 111 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/docs/wiki/index.md b/docs/wiki/index.md index e69de29bb2..aadfd249e5 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -0,0 +1,111 @@ +# llama-cpp-python Wiki + +Welcome to the `llama-cpp-python` wiki :) + +This wiki provides structured, source-code-aligned documentation for the public APIs, core classes, modules, examples, and development notes of `llama-cpp-python`. + +The documentation is maintained with the help of LLMs, but the source of truth is always the latest code in `llama_cpp/`. + +--- + +## Quick Navigation + +### Core API + +Start here if you are using `llama-cpp-python` directly. + +| Page | Description | +|---|---| +| [core/Llama\|Llama] | Main high-level interface for loading GGUF models, running completions, chat completions, tokenization, embeddings, and model configuration. | + +--- + +### Modules + +These pages document major source modules and related classes. + +| Page | Description | +|---|---| +| [modules/LlamaCache\|Llama Cache] | Cache interfaces and implementations for reusing model state across repeated prompts. | +| [modules/LlamaEmbedding\|Llama Embedding] | Embedding-related APIs and usage patterns. | +| [modules/LlamaSpeculative\|Llama Speculative Decoding] | Draft model interfaces and prompt-based speculative decoding helpers. | + +--- + +### Wiki Maintenance + +These pages define how the wiki should be written, updated, and reviewed. + +| Page | Description | +|---|---| +| [SCHEMA\|Wiki Schema] | Documentation schema and rules for LLM-maintained wiki pages. | +| [contributing-to-wiki\|Contributing to the Wiki] | Contribution guide for writing and updating wiki documentation. | + +--- + +## Recommended Reading Order + +If you are new to this wiki, read the pages in this order: + +1. [[core/Llama|Llama](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] +2. [[modules/LlamaEmbedding|Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] +3. [[modules/LlamaCache|Llama Cache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] +4. [[modules/LlamaSpeculative|Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] + +If you are contributing documentation, start with: + +1. [[SCHEMA|Wiki Schema](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/SCHEMA.md)] +2. [[contributing-to-wiki|Contributing to the Wiki](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/contributing-to-wiki.md)] + +--- + +## Documentation Status + +The wiki is still being expanded. + +Currently available pages: + +- `core/Llama.md` +- `modules/LlamaCache.md` +- `modules/LlamaEmbedding.md` +- `modules/LlamaSpeculative.md` +- `SCHEMA.md` +- `contributing-to-wiki.md` + +Some planned pages may already exist as empty placeholder files. Empty pages are intentionally not linked from this index until they are completed. + +--- + +## Planned Areas + +Future documentation may cover: + +- Installation and build options +- Chat formats and chat handlers +- Low-level ctypes bindings +- Multimodal APIs +- Type definitions and structured return values +- Troubleshooting +- Runnable examples +- Development notes + +--- + +## Documentation Principles + +This wiki follows a few core rules: + +- Source code is the source of truth. +- Parameters, defaults, and behavior must match the latest implementation. +- Examples should be complete and runnable. +- Deprecated or legacy APIs should be clearly marked. +- Internal implementation details should not be presented as stable public APIs. +- Pages should be concise, practical, and easy to navigate. + +--- + +## Project Links + +- GitHub: [llama-cpp-python](https://github.com/JamePeng/llama-cpp-python) +- Wiki schema: [SCHEMA](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/SCHEMA.md) +- Contribution guide: [contributing-to-wiki](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/contributing-to-wiki.md) \ No newline at end of file From 374c0d00aab924f03c18a2a53ab65c2fa20ce66c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 2 May 2026 14:18:54 +0800 Subject: [PATCH 058/304] Update Submodule vendor/llama.cpp b97ebdc..63d93d1 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b97ebdc98f..63d93d1733 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b97ebdc98f6053604a19d861c08d8087601b96e0 +Subproject commit 63d93d17336e41e4cc73a64451e5b1d2477abdb1 From fe38cbfac424973ccd9cfaa199a64a02502af4d1 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 2 May 2026 14:50:46 +0800 Subject: [PATCH 059/304] Bump version to 0.3.37 Signed-off-by: JamePeng --- CHANGELOG.md | 76 +++++++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 480b3c24cc..d144fe2267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,82 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.37] MoE CPU Offloading, O(1) Speculative Decoding, Thread-Safe Abort & New LLM Wiki + +- docs: A basic new documentation system for the LLM-Wiki has been initially established. + - Based on the continuously optimized `SCHEMA.md`, I am attempting to enable AI to automatically learn code files and write and update corresponding Markdown documents. + - Currently, the documentation under the path `/docs/wiki/` is complete: + - `core/Llama.md` + - `modules/LlamaCache.md` + - `modules/LlamaEmbedding.md` + - `modules/LlamaSpeculative.md` + - `SCHEMA.md` + - `contributing-to-wiki.md` + - `index.md` + - The Github Wiki is now also synchronized with `/docs/wiki/index.md` + - Note: The LLM-wiki is still being expanded. + +- docs: update LLM wiki schema to v0.3 + - Add schema metadata, documentation language rules, expanded page templates, attribute/state documentation guidance, and clearer update rules for LLM-maintained llama-cpp-python wiki pages. + +- feat(llama): add fine-grained MoE CPU offloading controls + - Introduce `cpu_moe` (bool) and `n_cpu_moe` (int) parameters to `Llama.__init__` for precise Mixture of Experts (MoE) weight offloading. + - `cpu_moe=True` forces all MoE expert weights to the CPU memory, regardless of `n_gpu_layers`. + - `n_cpu_moe=N` offloads the expert weights of the first N layers to the CPU, while keeping attention and router weights on the GPU. + - Enhance `n_gpu_layers` to accept string literals "auto" (equivalent to -1) and "all" (equivalent to -2) alongside exact integers, improving configuration readability. + - Update internal module aliases (e.g., `llama_cpp` to `llama_cpp_lib`) to avoid naming conflicts with the underlying C library. + - Integrate `ggml_backend_cpu_buffer_type` to map specific tensor overrides (via regex) directly to CPU buffers during model load. + +- feat(_ggml): implement ggml-backend API bindings and fix type hints + - Introduces extensive ctypes bindings for `ggml-backend.h` (devices, buffers, registries, and CPU buffer types) to support advanced memory routing like MoE CPU offloading. Also fixes various static typing warnings by adding `# type: ignore` to pointer annotations. + - *Note: Synchronize ggml's ctypes calls as needed, but won't fully implement it, because most of it is called at the lower level in the upstream llama.cpp.* + +- feat(handler): Support `add_generation_prompt` parameter pass to `MTMDChatHandler` + - supports disabling assistant part injection, used to support the multimodal `assistant_prefill` functionality. + +- feat(core): implement thread-safe generation abort mechanism + - Add `AbortCriteria` class and a thread-safe `Llama.abort()` method to allow graceful interruption of ongoing text generation from external threads (e.g., UI or async environments). + - Automatically inject `AbortCriteria` into the stopping criteria sequence at the start of `_create_completion`. + - Ensure that when an abort is triggered, the partially generated `completion_tokens` are correctly detokenized and preserved. + - Set `finish_reason` to `"abort"` when generation is interrupted, allowing downstream streaming clients to correctly identify manual cancellations. + - Simplify and optimize the stopping criteria evaluation logic within the core `generate` loop. + - Reorganize and sort module imports for better readability. + - Update /docs/wiki/core/Llama.md for `abort()` and example code + +- feat(speculative): introduce O(1) hash-based N-Gram speculative decoding + - Add `LlamaNGramMapDecoding` to `llama_speculative.py`, implementing an ultra-fast speculative decoder based on a hash inverted index and incremental updates. + - Achieve O(1) time complexity for draft token generation, completely eliminating the CPU bottleneck present in the legacy Numpy sliding window approach. + - Update `README.md` and `docs/wiki/core/Llama.md` to recommend `LlamaNGramMapDecoding` as the default and fastest speculative decoding method, along with updated initialization examples. + - Add docs comment to the speculative decoding classes for better developer experience. + - Add warnings to the legacy `LlamaPromptLookupDecoding` class regarding its high computational overhead for long contexts. + +- docs: Update README.md + +- feat(types): introduce MCP definitions and align with latest OpenAI spec + - Add comprehensive Model Context Protocol (MCP) type definitions, including `MCPTool`, `MCPToolCall`, `MCPListTools`, connector IDs, and approval filters to support remote server tool calling. + - Add `ServiceTier` literal ("auto", "default", etc.) and include the `service_tier` field in `CreateChatCompletionResponse`. + - Restrict `finish_reason` in completion responses to strict standard literals (`stop`, `length`, `tool_calls`, `content_filter`, `function_call`). + - Introduce `ChatCompletionMessageCustomToolCall` to support custom tool calls generated by the model. + - Update `ChatCompletionRequestAssistantMessage` to include the `name` field and add descriptive docstrings to message types. + +- docs: initialize LLM Wiki structure for better documentation maintenance + - Create docs/wiki/ directory with full folder structure + - Add SCHEMA.md, index.md and contributing guidelines + - Set up core/, features/, modules/, examples/, types/ and subdirectories + - Prepare for LLM-powered living documentation (Llama class, multi-modal chat handlers, vision/audio examples, etc.) + - Include .gitkeep files to preserve empty directories + + This lays the foundation for a modern, maintainable wiki that will replace outdated static docs. + Future commits will populate pages with up-to-date content generated from latest source code. + +- chore(ci): upgrade astral-sh/setup-uv@v7 and Jimver/cuda-toolkit@v0.2.35 (Node 24 runtime) + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/63d93d17336e41e4cc73a64451e5b1d2477abdb1](https://github.com/ggml-org/llama.cpp/commit/63d93d17336e41e4cc73a64451e5b1d2477abdb1) + +- feat: Sync llama.cpp llama/mtmd API Binding 20260421 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/b97cb637cd6124fc47f569721b1716014bd856a8...374c0d00aab924f03c18a2a53ab65c2fa20ce66c + ## [0.3.36] Gemma-4 Omni-Multimodal and ToolCall Improved, Qwen3.6 / Step3-VL Support, Compilation workflow optimization - feat: enhance `Qwen35ChatHandler` with preserve_thinking and `Qwen3.6` Support diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index a02ec5af51..7fd2b4c492 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.36" +__version__ = "0.3.37" From 5336947364f8d2b068c46a77609e0406541067b7 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 3 May 2026 15:26:48 +0800 Subject: [PATCH 060/304] feat(grammar): sync JSON schema to GBNF converter with upstream - Allow `LlamaGrammar.from_json_schema` and `json_schema_to_gbnf` to accept both string and dict schema inputs. - Expose `allow_fetch`, `dotall`, and `raw_pattern` arguments to the public API to match the upstream script. - Fix missing handling for empty/unconstrained schema objects (e.g. `{"description": "..."}`) which now correctly default to accepting any value. - Fix bug where `has_min and has_max` evaluated incorrectly when variables were zero. Replaced `!= None` with `is not None` in `_generate_min_max_int`. - Update internal constants and regex patterns (`INVALID_RULE_CHARS_RE`, `GRAMMAR_LITERAL_ESCAPE_RE`, `GRAMMAR_RANGE_LITERAL_ESCAPE_RE`) to resolve character escaping issues. - Add type hint `| None` for dependencies in `BuiltinRule`. - Update reference link to point to the new `ggml-org` organization. Signed-off-by: JamePeng --- llama_cpp/llama_grammar.py | 66 ++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/llama_cpp/llama_grammar.py b/llama_cpp/llama_grammar.py index 21bb688dee..3c431fc3d8 100644 --- a/llama_cpp/llama_grammar.py +++ b/llama_cpp/llama_grammar.py @@ -1,4 +1,4 @@ -"""Python implementation of llama grammar parser directly translated from C++ source file in vendor/llama.cpp/common/grammar-parser.cpp.""" +"""Python implementation of llama grammar parser. Reference: vendor/llama.cpp/examples/json_schema_to_grammar.py""" # flake8: noqa from pathlib import Path @@ -51,8 +51,11 @@ def from_file(cls, file: Union[str, Path], verbose: bool = True) -> "LlamaGramma @classmethod def from_json_schema( cls, - json_schema: str, + json_schema: Union[str, dict], prop_order: Optional[List[str]] = None, + allow_fetch: bool = False, + dotall: bool = False, + raw_pattern: bool = False, verbose: bool = True ) -> "LlamaGrammar": """ @@ -63,7 +66,13 @@ def from_json_schema( verbose: Whether to log. """ try: - gbnf_grammar_str = json_schema_to_gbnf(json_schema, prop_order=prop_order) + gbnf_grammar_str = json_schema_to_gbnf( + json_schema, + prop_order=prop_order, + allow_fetch=allow_fetch, + dotall=dotall, + raw_pattern=raw_pattern, + ) return cls.from_string(gbnf_grammar_str, verbose=verbose) except Exception as e: raise ValueError(f"{cls.__name__}.from_json_schema: conversion failed: {e}") @@ -285,9 +294,6 @@ def _build_repetition(item_rule, min_items, max_items, separator_rule=None): return f'({result})?' if min_items == 0 else result def _generate_min_max_int(min_value: Optional[int], max_value: Optional[int], out: list, decimals_left: int = 16, top_level: bool = True): - has_min = min_value != None - has_max = max_value != None - def digit_range(from_char: str, to_char: str): out.append("[") if from_char == to_char: @@ -363,7 +369,7 @@ def uniform_range(from_str: str, to_str: str): out.append(to_str[i]) out.append("]") - if has_min and has_max: + if min_value is not None and max_value is not None: if min_value < 0 and max_value < 0: out.append("\"-\" (") _generate_min_max_int(-max_value, -min_value, out, decimals_left, top_level=True) @@ -390,7 +396,7 @@ def uniform_range(from_str: str, to_str: str): less_decimals = max(decimals_left - 1, 1) - if has_min: + if min_value is not None: if min_value < 0: out.append("\"-\" (") _generate_min_max_int(None, -min_value, out, decimals_left, top_level=False) @@ -434,7 +440,7 @@ def uniform_range(from_str: str, to_str: str): more_digits(length - 1, less_decimals) return - if has_max: + if max_value is not None: if max_value >= 0: if top_level: out.append("\"-\" [1-9] ") @@ -488,13 +494,13 @@ def __init__(self, content: str, deps: list = None): RESERVED_NAMES = set(["root", "dot", *PRIMITIVE_RULES.keys(), *STRING_FORMAT_RULES.keys()]) -INVALID_RULE_CHARS_RE = re.compile(r"[^a-zA-Z0-9-]+") -GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n\"\\\\]') -GRAMMAR_RANGE_LITERAL_ESCAPE_RE = re.compile(r'[\r\n\"\\]\\-\\\\]') -GRAMMAR_LITERAL_ESCAPES = {"\r": "\\r", "\n": "\\n", '"': '\\"', "-": "\\-", "]": "\\]", "\\": "\\\\"} +INVALID_RULE_CHARS_RE = re.compile(r'[^a-zA-Z0-9-]+') +GRAMMAR_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\\]') +GRAMMAR_RANGE_LITERAL_ESCAPE_RE = re.compile(r'[\r\n"\]\-\\]') +GRAMMAR_LITERAL_ESCAPES = {'\r': '\\r', '\n': '\\n', '"': '\\"', '-': '\\-', ']': '\\]', '\\': '\\\\'} -NON_LITERAL_SET = set("|.()[]{}*+?") -ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set("^$.[]()|{}*+?") +NON_LITERAL_SET = set('|.()[]{}*+?') +ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = set('^$.[]()|{}*+?') class SchemaConverter: def __init__(self, *, prop_order, allow_fetch, dotall, raw_pattern): @@ -659,7 +665,7 @@ def _visit_pattern(self, pattern, name): Transforms a regular expression pattern into a GBNF rule. Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions - Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md + Output: https://github.com/ggml-org/llama.cpp/blob/master/grammars/README.md Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers. @@ -946,6 +952,11 @@ def add_component(comp_schema, is_required): elif (schema_type == 'object') or (len(schema) == 0): return self._add_rule(rule_name, self._add_primitive('object', PRIMITIVE_RULES['object'])) + elif schema_type is None and isinstance(schema, dict): + # No type constraint and no recognized structural keywords (e.g. {"description": "..."}). + # Per JSON Schema semantics this is equivalent to {} and accepts any value. + return self._add_rule(rule_name, self._add_primitive('value', PRIMITIVE_RULES['value'])) + else: assert schema_type in PRIMITIVE_RULES, f'Unrecognized schema: {schema}' # TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero @@ -1031,12 +1042,27 @@ def format_grammar(self): ) -def json_schema_to_gbnf(schema: str, prop_order: Optional[List[str]] = None): +def json_schema_to_gbnf( + schema: Union[str, dict], + prop_order: Optional[List[str]] = None, + allow_fetch: bool = False, + dotall: bool = False, + raw_pattern: bool = False, +): prop_order = prop_order or [] - schema = json.loads(schema) - prop_order = {name: idx for idx, name in enumerate(prop_order)} + + if isinstance(schema, str): + schema = json.loads(schema) + elif isinstance(schema, dict): + schema = dict(schema) + else: + raise TypeError("schema must be a JSON string or dictionary") + converter = SchemaConverter( - prop_order=prop_order, allow_fetch=False, dotall=False, raw_pattern=False + prop_order={name: idx for idx, name in enumerate(prop_order)}, + allow_fetch=allow_fetch, + dotall=dotall, + raw_pattern=raw_pattern, ) schema = converter.resolve_refs(schema, "stdin") converter.visit(schema, "") From d7ed1895baa1149cd376bc9ed494d67acb18b898 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 3 May 2026 16:07:59 +0800 Subject: [PATCH 061/304] docs: add `LlamaGrammar` wiki page Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 2 + docs/wiki/modules/LlamaCache.md | 1 + docs/wiki/modules/LlamaChatFormat.md | 0 docs/wiki/modules/LlamaEmbedding.md | 1 + docs/wiki/modules/LlamaGrammar.md | 461 ++++++++++++++++++++++++++ docs/wiki/modules/LlamaSpeculative.md | 1 + 6 files changed, 466 insertions(+) delete mode 100644 docs/wiki/modules/LlamaChatFormat.md create mode 100644 docs/wiki/modules/LlamaGrammar.md diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index f010dd0c67..7a9b7bd6ad 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -313,6 +313,8 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn --- ## Related Links + +* [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] * [[Llama Cache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] - Implementing disk or RAM-based prompt caching (LlamaRAMCache, **TrieCache**, **HybridCheckpointCache**). * [[Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] - Dedicated class for text embeddings and reranking. * [[Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] - Provides draft model interfaces and prompt-based speculative decoding helpers. diff --git a/docs/wiki/modules/LlamaCache.md b/docs/wiki/modules/LlamaCache.md index ef02fd40e4..64e6bbb5f8 100644 --- a/docs/wiki/modules/LlamaCache.md +++ b/docs/wiki/modules/LlamaCache.md @@ -1389,4 +1389,5 @@ from llama_cpp.llama_cache import HybridCheckpointCache # Related Links +* [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] * [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] diff --git a/docs/wiki/modules/LlamaChatFormat.md b/docs/wiki/modules/LlamaChatFormat.md deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/docs/wiki/modules/LlamaEmbedding.md b/docs/wiki/modules/LlamaEmbedding.md index 57c0788e3f..1279db5cab 100644 --- a/docs/wiki/modules/LlamaEmbedding.md +++ b/docs/wiki/modules/LlamaEmbedding.md @@ -266,4 +266,5 @@ embeddings_raw = llm.embed(["search query", "document text"], normalize=NORM_MOD ## Related Links +* [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] * [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] diff --git a/docs/wiki/modules/LlamaGrammar.md b/docs/wiki/modules/LlamaGrammar.md new file mode 100644 index 0000000000..8c67633baa --- /dev/null +++ b/docs/wiki/modules/LlamaGrammar.md @@ -0,0 +1,461 @@ +--- +title: Llama Grammar +module_name: llama_cpp.llama_grammar +source_file: llama_cpp/llama_grammar.py +last_updated: 2026-05-03 +version_target: "latest" +--- + +# Llama Grammar + +## Overview + +`llama_grammar.py` provides grammar utilities for constrained generation in `llama-cpp-python`. + +The module defines the `LlamaGrammar` class, a collection of built-in GBNF grammar strings, and a JSON Schema to GBNF converter based on the upstream `llama.cpp` grammar tooling. + +Use this module when you need to guide model output toward a specific grammar, such as JSON, JSON arrays, lists, arithmetic expressions, or custom GBNF rules. + +## Role in the Library + +`LlamaGrammar` acts as a lightweight wrapper around a GBNF grammar string. + +The module also includes helper logic for converting JSON Schema definitions into GBNF grammar text. This allows users to define structured output constraints using JSON Schema-like input and convert it into a grammar format usable by llama.cpp-style constrained generation. + +## Important Classes + +| Class | Status | Description | +|---|---|---| +| `LlamaGrammar` | public | Main wrapper class for grammar strings. Supports creation from raw strings, files, and JSON Schema. | +| `BuiltinRule` | internal helper | Small container used by the JSON Schema converter to store built-in grammar rule content and dependencies. | +| `SchemaConverter` | internal implementation | Converts JSON Schema structures into GBNF grammar rules. Used by `json_schema_to_gbnf`. | + +## Constants + +### Default Root + +| Constant | Type | Value | Description | +|---|---|---|---| +| `LLAMA_GRAMMAR_DEFAULT_ROOT` | `str` | `"root"` | Default root rule name used by `LlamaGrammar`. | + +### Built-in GBNF Grammars + +The module includes several built-in GBNF grammar strings. + +| Constant | Description | +|---|---| +| `ARITHMETIC_GBNF` | Grammar for simple arithmetic-like expressions. | +| `C_GBNF` | Example grammar for a subset of C-like declarations and statements. | +| `CHESS_GBNF` | JSON-like grammar currently defined similarly to object/array/value grammar. | +| `ENGLISH_GBNF` | Simple English-character grammar. The source notes that it may be incomplete and mostly serves as an example. | +| `JAPANESE_GBNF` | JSON-like grammar currently defined similarly to object/array/value grammar. | +| `JSON_ARR_GBNF` | Grammar for generating JSON arrays. | +| `JSON_GBNF` | Grammar for JSON objects and values. | +| `LIST_GBNF` | Grammar for newline-separated Markdown-style list items. | + +### JSON Schema Conversion Rules + +The module also defines internal constants used by `SchemaConverter`. + +| Constant | Description | +|---|---| +| `SPACE_RULE` | Shared grammar rule for constrained whitespace. | +| `PRIMITIVE_RULES` | Built-in grammar rules for primitive schema types such as boolean, number, integer, object, array, string, and null. | +| `STRING_FORMAT_RULES` | Built-in grammar rules for selected string formats such as date, time, and date-time. | +| `RESERVED_NAMES` | Rule names reserved by the converter. | +| `DOTALL` | Pattern rule matching any Unicode code point. | +| `DOT` | Pattern rule matching any character except line breaks. | + +## `LlamaGrammar` + +```python +class LlamaGrammar +```` + +Main wrapper for GBNF grammar text. + +### Constructor + +```python +def __init__(self, *args, _grammar: str, **kwargs) +``` + +| Parameter | Type | Default | Description | +| ---------- | -------- | -------- | -------------------------------------------------------------------------------- | +| `*args` | variadic | none | Accepted by the constructor but not used directly in the current implementation. | +| `_grammar` | `str` | required | Grammar string stored by the instance. | +| `**kwargs` | variadic | none | Accepted by the constructor but not used directly in the current implementation. | + +### Important Attributes / State + +| Attribute | Type | Source | Description | +| ---------- | -------------- | ------------------------------- | ------------------------------------------------------- | +| `_grammar` | `str` | `_grammar` constructor argument | Internal grammar string stored by the instance. | +| `_root` | `str` | `LLAMA_GRAMMAR_DEFAULT_ROOT` | Internal root rule name. Defaults to `"root"`. | +| `grammar` | `str` property | `_grammar` | Read-only property returning the stored grammar string. | + +## Class Methods + +### `from_string` + +```python +@classmethod +def from_string( + cls, + grammar: str, + verbose: bool = True, +) -> "LlamaGrammar" +``` + +Creates a `LlamaGrammar` instance from a raw GBNF grammar string. + +| Parameter | Type | Default | Description | +| --------- | ------ | -------- | ------------------------------------------------------------------------------------------------- | +| `grammar` | `str` | required | Raw GBNF grammar string. | +| `verbose` | `bool` | `True` | Accepted by the method. The current implementation forwards no logging behavior from this method. | + +Returns: + +| Type | Description | +| -------------- | -------------------------------------------------------- | +| `LlamaGrammar` | Grammar instance containing the provided grammar string. | + +#### Example + +```python +from llama_cpp.llama_grammar import LlamaGrammar, JSON_GBNF + +grammar = LlamaGrammar.from_string(JSON_GBNF) + +print(grammar.grammar) +``` + +### `from_file` + +```python +@classmethod +def from_file( + cls, + file: Union[str, Path], + verbose: bool = True, +) -> "LlamaGrammar" +``` + +Creates a `LlamaGrammar` instance from a UTF-8 grammar file. + +| Parameter | Type | Default | Description | +| --------- | ------------------ | -------- | ------------------------ | +| `file` | `Union[str, Path]` | required | Path to a grammar file. | +| `verbose` | `bool` | `True` | Passed to `from_string`. | + +Behavior based on the current implementation: + +* Raises `FileNotFoundError` if the file does not exist. +* Raises `IOError` if reading the file fails. +* Raises `ValueError` if the grammar file is empty. +* Reads the file using UTF-8 encoding. + +#### Example + +```python +from llama_cpp.llama_grammar import LlamaGrammar + +grammar = LlamaGrammar.from_file("./json.gbnf") + +print(grammar.grammar) +``` + +### `from_json_schema` + +```python +@classmethod +def from_json_schema( + cls, + json_schema: Union[str, dict], + prop_order: Optional[List[str]] = None, + allow_fetch: bool = False, + dotall: bool = False, + raw_pattern: bool = False, + verbose: bool = True, +) -> "LlamaGrammar" +``` + +Creates a `LlamaGrammar` instance by converting a JSON Schema string or dictionary into GBNF grammar. + +| Parameter | Type | Default | Description | +| ------------- | --------------------- | -------- | -------------------------------------------------------------------------------------------------------- | +| `json_schema` | `Union[str, dict]` | required | JSON Schema input as a JSON string or Python dictionary. | +| `prop_order` | `Optional[List[str]]` | `None` | Optional property order. The source comment notes this can help improve stability for small models. | +| `allow_fetch` | `bool` | `False` | Allows remote schema fetching for HTTPS `$ref` values when enabled. | +| `dotall` | `bool` | `False` | Controls whether pattern `.` should match all Unicode code points during regex-to-grammar conversion. | +| `raw_pattern` | `bool` | `False` | Controls whether regex patterns are converted as raw grammar patterns instead of quoted string patterns. | +| `verbose` | `bool` | `True` | Passed to `from_string`. | + +Returns: + +| Type | Description | +| -------------- | -------------------------------------------------------------- | +| `LlamaGrammar` | Grammar instance containing the generated GBNF grammar string. | + +If conversion fails, the method raises `ValueError`. + +#### Example + +```python +from llama_cpp.llama_grammar import LlamaGrammar + +schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + "required": ["name"], +} + +grammar = LlamaGrammar.from_json_schema(schema) + +print(grammar.grammar) +``` + +## `json_schema_to_gbnf` + +```python +def json_schema_to_gbnf( + schema: Union[str, dict], + prop_order: Optional[List[str]] = None, + allow_fetch: bool = False, + dotall: bool = False, + raw_pattern: bool = False, +) +``` + +Converts a JSON Schema string or dictionary into a GBNF grammar string. + +| Parameter | Type | Default | Description | +| ------------- | --------------------- | -------- | --------------------------------------------------------------------------------------------------- | +| `schema` | `Union[str, dict]` | required | JSON Schema input. Strings are parsed with `json.loads`; dictionaries are copied before conversion. | +| `prop_order` | `Optional[List[str]]` | `None` | Optional property ordering used by object rule generation. | +| `allow_fetch` | `bool` | `False` | Allows remote HTTPS `$ref` fetching when enabled. | +| `dotall` | `bool` | `False` | Controls regex dot behavior during pattern conversion. | +| `raw_pattern` | `bool` | `False` | Controls how regex pattern rules are emitted. | + +Returns: + +| Type | Description | +| ----- | ------------------------------ | +| `str` | Generated GBNF grammar string. | + +The function raises `TypeError` if `schema` is neither a JSON string nor a dictionary. + +### Example + +```python +from llama_cpp.llama_grammar import json_schema_to_gbnf + +schema = { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "maxItems": 3, +} + +gbnf = json_schema_to_gbnf(schema) + +print(gbnf) +``` + +## `SchemaConverter` + +```python +class SchemaConverter +``` + +Internal implementation class used by `json_schema_to_gbnf`. + +`SchemaConverter` walks a JSON Schema dictionary, resolves references, builds grammar rules, and formats them into GBNF. + +This class is useful for understanding how conversion works, but most users should use `LlamaGrammar.from_json_schema` or `json_schema_to_gbnf` instead. + +> Warning: `SchemaConverter` appears to be an implementation detail. It should not be treated as the primary public API unless the project explicitly documents it as stable. + +### Constructor + +```python +def __init__( + self, + *, + prop_order, + allow_fetch, + dotall, + raw_pattern, +) +``` + +| Parameter | Type | Description | +| ------------- | ------------ | ----------------------------------------------------------------------- | +| `prop_order` | mapping-like | Property ordering map used when generating object rules. | +| `allow_fetch` | `bool` | Enables or disables remote schema fetching for supported `$ref` values. | +| `dotall` | `bool` | Controls regex dot behavior. | +| `raw_pattern` | `bool` | Controls raw pattern handling. | + +### Important Internal State + +| Attribute | Type | Description | +| ---------------------- | ------------ | ------------------------------------------------ | +| `_prop_order` | mapping-like | Stores property ordering preferences. | +| `_allow_fetch` | `bool` | Stores whether remote references may be fetched. | +| `_dotall` | `bool` | Stores regex dot behavior. | +| `_raw_pattern` | `bool` | Stores raw pattern handling behavior. | +| `_rules` | `dict` | Accumulates generated grammar rules. | +| `_refs` | `dict` | Stores resolved JSON Schema references. | +| `_refs_being_resolved` | `set` | Tracks references currently being resolved. | + +### Key Methods + +| Method | Description | +| -------------------- | ---------------------------------------------------------------------------------------- | +| `resolve_refs` | Resolves local and supported HTTPS `$ref` references in a schema. | +| `visit` | Main schema visitor that generates grammar rules based on schema structure. | +| `format_grammar` | Formats generated rules into a GBNF grammar string. | +| `_build_object_rule` | Builds object grammar rules from properties, required fields, and additional properties. | +| `_visit_pattern` | Converts supported regex patterns into GBNF rules. | +| `_add_rule` | Adds or reuses a grammar rule name. | +| `_add_primitive` | Adds primitive rules and their dependencies. | + +## Supported JSON Schema Features + +Based on the current implementation, the converter includes handling for: + +* `type` +* `properties` +* `required` +* `additionalProperties` +* `$ref` +* `oneOf` +* `anyOf` +* `allOf` +* `const` +* `enum` +* `items` +* `prefixItems` +* `minItems` +* `maxItems` +* `pattern` +* `format` +* `minLength` +* `maxLength` +* integer bounds: + + * `minimum` + * `exclusiveMinimum` + * `maximum` + * `exclusiveMaximum` + +String formats handled by built-in rules include: + +* `date` +* `time` +* `date-time` +* UUID-like formats matching `uuid`, `uuid1`, `uuid2`, `uuid3`, `uuid4`, or `uuid5` + +The source includes a TODO comment for unsupported string formats such as `uri` and `email`. + +## Error Handling + +| API | Error | Condition | +| ------------------------------- | ------------------- | ----------------------------------------- | +| `LlamaGrammar.from_file` | `FileNotFoundError` | Grammar file path does not exist. | +| `LlamaGrammar.from_file` | `IOError` | Grammar file cannot be read. | +| `LlamaGrammar.from_file` | `ValueError` | Grammar file is empty. | +| `LlamaGrammar.from_json_schema` | `ValueError` | JSON Schema to GBNF conversion fails. | +| `json_schema_to_gbnf` | `TypeError` | Schema input is neither `str` nor `dict`. | + +## Common Usage + +### Use a Built-in Grammar + +```python +from llama_cpp.llama_grammar import LlamaGrammar, JSON_GBNF + +grammar = LlamaGrammar.from_string(JSON_GBNF) + +print(grammar.grammar) +``` + +### Load Grammar from a File + +```python +from llama_cpp.llama_grammar import LlamaGrammar + +grammar = LlamaGrammar.from_file("./grammar.gbnf") + +print(grammar.grammar) +``` + +### Convert JSON Schema to Grammar + +```python +from llama_cpp.llama_grammar import LlamaGrammar + +schema = { + "type": "object", + "properties": { + "answer": {"type": "string"}, + "confidence": {"type": "number"}, + }, + "required": ["answer"], +} + +grammar = LlamaGrammar.from_json_schema( + schema, + prop_order=["answer", "confidence"], +) + +print(grammar.grammar) +``` + +### Convert JSON Schema Directly to GBNF + +```python +from llama_cpp.llama_grammar import json_schema_to_gbnf + +schema = { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": {"type": "string"}, + } + }, +} + +gbnf = json_schema_to_gbnf(schema) + +print(gbnf) +``` + +## Best Practices & Common Patterns + +* Use `LlamaGrammar.from_string` when you already have a GBNF grammar string. +* Use `LlamaGrammar.from_file` when storing grammar definitions in `.gbnf` files. +* Use `LlamaGrammar.from_json_schema` when generating grammars from JSON Schema input. +* Use `json_schema_to_gbnf` directly when you only need the generated grammar string. +* Keep JSON Schemas small and explicit when targeting constrained generation. +* Use `prop_order` when output field order matters for stability. +* Keep `allow_fetch=False` unless remote `$ref` fetching is explicitly needed. +* Prefer public helpers over using `SchemaConverter` directly. +* Do not rely on internal converter methods as stable public APIs. + +## Limitations + +* `SchemaConverter` is implementation-oriented and may change. +* Remote `$ref` fetching is only attempted for HTTPS references and requires `allow_fetch=True`. +* The source includes TODO notes for unsupported string formats such as `uri` and `email`. +* Regex pattern conversion explicitly rejects unsupported pattern syntax such as lookaheads and non-greedy modifiers. +* The exact runtime integration between `LlamaGrammar` and model generation should be verified from the relevant generation APIs before documenting end-to-end constrained generation behavior. + +## Related Links + +* [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] +* [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] diff --git a/docs/wiki/modules/LlamaSpeculative.md b/docs/wiki/modules/LlamaSpeculative.md index d1463883b9..0c0ad099fb 100644 --- a/docs/wiki/modules/LlamaSpeculative.md +++ b/docs/wiki/modules/LlamaSpeculative.md @@ -233,5 +233,6 @@ Each class MUST have exactly the same internal structure and method names (creat ## Related Links +* [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] * [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] From de5a1e8dd9156966758e522d2b99d8739a56fc96 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 3 May 2026 16:12:43 +0800 Subject: [PATCH 062/304] Update /docs/wiki/index.md Signed-off-by: JamePeng --- docs/wiki/index.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/wiki/index.md b/docs/wiki/index.md index aadfd249e5..02f2dd5b9a 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -28,6 +28,7 @@ These pages document major source modules and related classes. |---|---| | [modules/LlamaCache\|Llama Cache] | Cache interfaces and implementations for reusing model state across repeated prompts. | | [modules/LlamaEmbedding\|Llama Embedding] | Embedding-related APIs and usage patterns. | +| [modules/LlamaGrammar\|Llama Grammar] | Provides grammar utilities for constrained generation. | | [modules/LlamaSpeculative\|Llama Speculative Decoding] | Draft model interfaces and prompt-based speculative decoding helpers. | --- @@ -48,9 +49,10 @@ These pages define how the wiki should be written, updated, and reviewed. If you are new to this wiki, read the pages in this order: 1. [[core/Llama|Llama](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] -2. [[modules/LlamaEmbedding|Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] -3. [[modules/LlamaCache|Llama Cache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] -4. [[modules/LlamaSpeculative|Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] +2. [[modules/LlamaCache|Llama Cache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] +3. [[modules/LlamaEmbedding|Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] +4. [[modules/LlamaGrammar|Llama Grammar](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaGrammar.md)] +5. [[modules/LlamaSpeculative|Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] If you are contributing documentation, start with: @@ -68,6 +70,7 @@ Currently available pages: - `core/Llama.md` - `modules/LlamaCache.md` - `modules/LlamaEmbedding.md` +- `modules/LlamaGrammar.md` - `modules/LlamaSpeculative.md` - `SCHEMA.md` - `contributing-to-wiki.md` From a63f60e8956342a90078421b439a626cd4d1cb17 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 4 May 2026 09:01:03 +0800 Subject: [PATCH 063/304] perf: Optimize detokenize buffer sizing for CJK-heavy outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Increase the initial detokenization buffer estimate from 1x tokens to `5x + 32` bytes. Performance analysis revealed that CJK-heavy outputs often require around 4.0xā€“5.04x bytes per token, with small-token edge cases reaching about 6.0x, so the previous estimate frequently forced a failed `llama_detokenize` call followed by a resize and retry. Almost twice as many calls to llama_detokenize. - This reduces avoidable detokenization retries and cuts call overhead in CJK-heavy cases, resulting in an observed ~3ā€“5% inference performance improvement. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 27dd4f80d3..b4ba1f4b21 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -298,7 +298,11 @@ def detokenize(self, tokens: List[int], special: bool = False) -> bytes: tokens_array = (llama_cpp.llama_token * n_tokens)(*tokens) # Initial buffer size estimation - buffer_size = max(n_tokens, 64) + # Note(JamePeng): + # Observed CJK heavy outputs are about 4.0x - 5.04x, + # with extreme values ā€‹ā€‹even reaching 6.0x, so this avoids most retry cases. + # In CJK use cases, the call overhead will be reduced by 50% compared to the previous detokenize method. + buffer_size = max(64, n_tokens * 5 + 32) buffer = (ctypes.c_char * buffer_size)() n_chars = llama_cpp.llama_detokenize( From 22a7fdbfe0c0e16a336ef9463b6b4df282b7928f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 4 May 2026 10:40:30 +0800 Subject: [PATCH 064/304] patch(logger): filter out verbose noisy CUDA Graph debug logs Add a temporary patch in `ggml_log_callback` to suppress the noisy `CUDA Graph id %zu reused` messages generated by the underlying C++ backend. A complete logger refactoring is planned for better log control in the future. Signed-off-by: JamePeng --- llama_cpp/_logger.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llama_cpp/_logger.py b/llama_cpp/_logger.py index 022ece22bf..015cec9faa 100644 --- a/llama_cpp/_logger.py +++ b/llama_cpp/_logger.py @@ -32,6 +32,12 @@ def ggml_log_callback( text: bytes, user_data: ctypes.c_void_p, ): + # Note(JamePeng): A temporary patch is used to filter out garbage debug information + # output from the underlying C++ `CUDA Graph id %zu reused`. + # The logger is planned to be refactored to meet control requirements. + if text: + if b"CUDA Graph" in text or b"CUDA graph" in text: + return # TODO: Correctly implement continue previous log global _last_log_level log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level] if level != 5 else _last_log_level From fe8657adf71856967320bfb932461d65ecf77b19 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 4 May 2026 11:00:32 +0800 Subject: [PATCH 065/304] Update Submodule vendor/llama.cpp 63d93d1..e48034d Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 63d93d1733..e48034dfc9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 63d93d17336e41e4cc73a64451e5b1d2477abdb1 +Subproject commit e48034dfc9e5705248fd39dc437ca887dc55a528 From ef27f333f367fdc53dc1a729ad8bb6c3c9362514 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 4 May 2026 11:15:14 +0800 Subject: [PATCH 066/304] Bump version to 0.3.38 Signed-off-by: JamePeng --- CHANGELOG.md | 24 ++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d144fe2267..253b2ae4cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.38] Optimized CJK Detokenization, Sync Grammar Parser, and Patched CUDA Graph Logs + +- perf: Optimize detokenize buffer sizing for CJK-heavy outputs + - Increase the initial detokenization buffer estimate from 1x tokens to `5x + 32` bytes. Performance analysis revealed that CJK-heavy outputs often require around 4.0xā€“5.04x bytes per token, with small-token edge cases reaching about 6.0x, so the previous estimate frequently forced a failed `llama_detokenize` call followed by a resize and retry. Previously, this resulted in almost twice as many calls to llama_detokenize. + - This reduces avoidable detokenization retries and cuts call overhead in CJK-heavy cases, resulting in an observed ~3ā€“5% inference performance improvement. + +- patch(logger): filter out verbose noisy CUDA Graph debug logs + - Add a temporary patch in `ggml_log_callback` to suppress the noisy `CUDA Graph id %zu reused` messages generated by the underlying C++ backend. A complete logger refactoring is planned for better log control in the future. + +- docs: add LlamaGrammar wiki page and Update /docs/wiki/index.md + - Now Github Wiki Online: https://github.com/JamePeng/llama-cpp-python/wiki + +- feat(grammar): sync JSON schema to GBNF converter with upstream + - Allow `LlamaGrammar.from_json_schema` and `json_schema_to_gbnf` to accept both string and dict schema inputs. + - Expose `allow_fetch`, `dotall`, and `raw_pattern` arguments to the public API to match the upstream script. + - Fix missing handling for empty/unconstrained schema objects (e.g. `{"description": "..."}`) which now correctly default to accepting any value. + - Fix bug where `has_min and has_max` evaluated incorrectly when variables were zero. Replaced `!= None` with `is not None` in `_generate_min_max_int`. + - Update internal constants and regex patterns (`INVALID_RULE_CHARS_RE`, `GRAMMAR_LITERAL_ESCAPE_RE`, `GRAMMAR_RANGE_LITERAL_ESCAPE_RE`) to resolve character escaping issues. + - Update reference link to point to the new `ggml-org` organization. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/e48034dfc9e5705248fd39dc437ca887dc55a528](https://github.com/ggml-org/llama.cpp/commit/e48034dfc9e5705248fd39dc437ca887dc55a528) + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/fe38cbfac424973ccd9cfaa199a64a02502af4d1...fe8657adf71856967320bfb932461d65ecf77b19 + ## [0.3.37] MoE CPU Offloading, O(1) Speculative Decoding, Thread-Safe Abort & New LLM Wiki - docs: A basic new documentation system for the LLM-Wiki has been initially established. diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 7fd2b4c492..438bf08b58 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.37" +__version__ = "0.3.38" From 1f5226b4882542545bec204d81f04171059566ee Mon Sep 17 00:00:00 2001 From: Alcoft Date: Mon, 4 May 2026 20:58:58 +0200 Subject: [PATCH 067/304] Implemented generic multimodal chat handler. --- llama_cpp/llama.py | 12 +++++++++ llama_cpp/llama_chat_format.py | 49 +++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 1241f81e26..848706a90d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -85,6 +85,7 @@ class Llama: def __init__( self, model_path: str, + clip_model_path: Optional[str] = None, *, # Model Params n_gpu_layers: Union[int, Literal["auto", "all"]] = "auto", @@ -608,6 +609,17 @@ def __init__( if self.verbose: print(f"Model metadata: {self.metadata}", file=sys.stderr) + + if clip_model_path is not None: + if self.chat_handler is not None and self.verbose: + print("Warning: Both `chat_handler` and `clip_model_path` are not null. Chat handler will be overwritten.", flush = True) + + self.chat_handler = llama_chat_format.GenericMTMDChatHandler( + gguf_metadata = self.metadata, + clip_model_path = clip_model_path, + model_arch = None, + verbose = self.verbose + ) eos_token_id = self.token_eos() bos_token_id = self.token_bos() diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index a0d8d25db4..468a73c077 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2887,10 +2887,14 @@ def __init__( raise ValueError(f"{self.log_prefix}(__init__): Clip model path does not exist: {clip_model_path}") # Pre-compile Jinja template + if not hasattr(self, "chat_format") or self.chat_format is None: + self.chat_format = self.CHAT_FORMAT + + self._chat_format_parser_tags = [] self.chat_template = ImmutableSandboxedEnvironment( trim_blocks=True, lstrip_blocks=True, - ).from_string(self.CHAT_FORMAT) + ).from_string(self.chat_format) self._exit_stack = ExitStack() @@ -3116,6 +3120,13 @@ def _process_mtmd_prompt( tool_choice=tool_choice, **getattr(self, 'extra_template_arguments', {}) ) + + for tag in self._chat_format_parser_tags: + if tag not in text: + continue + + text = text[:text.index(tag)] + media_marker + text[text.index(tag) + len(tag):] + # Replace image_url by media_marker in text for item in media_items: text = text.replace(item["url"], media_marker) @@ -3827,6 +3838,42 @@ def from_pretrained( **kwargs, ) +class GenericMTMDChatHandler(MTMDChatHandler): + def __init__( + self, + gguf_metadata: Dict[str, Any], + clip_model_path: str, + model_arch: Optional[str] = None, + verbose: bool = True, + **kwargs + ) -> None: + self.model_metadata = gguf_metadata + + self.chat_format = self.model_metadata.get("tokenizer.chat_template", None) + self.arch = self.model_metadata.get("general.architecture", None) if model_arch is None else model_arch + + if verbose: + print(f"Got chat template from model:\n```jinja\n{self.chat_format}\n```", flush = True) + + if self.arch is None: + if verbose: + print("Unknown model architecture. Will use general/most-common tags.") + + self.arch = "unknown" + + if self.chat_format is None: + raise ValueError("Failed to get model chat template automatically.") + + super().__init__(clip_model_path = clip_model_path, verbose = verbose, **kwargs) + + if self.arch in ["unknown", "qwen3vl", "qwen35moe", "qwen35"]: + self._chat_format_parser_tags += ["<|image_pad|>", "<|audio_pad|>", "<|video_pad|>"] + elif self.arch in ["gemma4"]: + self._chat_format_parser_tags += ["<|image|>", "<|audio|>", "<|video|>"] + elif self.arch in ["mistral3", "mistral4", "deepseek2"]: + self._chat_format_parser_tags += ["[IMG]"] + elif verbose: + print("Warning: Could not determine chat format parser tags.", flush = True) class Llava15ChatHandler(MTMDChatHandler): CHAT_FORMAT = ( From a8d19d3bbd18890693576b1f5ed6cd0b2d487eab Mon Sep 17 00:00:00 2001 From: Alcoft Date: Mon, 4 May 2026 21:19:20 +0200 Subject: [PATCH 068/304] Used text.replace() --- llama_cpp/llama_chat_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 468a73c077..ab5e438d3e 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3125,7 +3125,7 @@ def _process_mtmd_prompt( if tag not in text: continue - text = text[:text.index(tag)] + media_marker + text[text.index(tag) + len(tag):] + text = text.replace(tag, media_marker) # Replace image_url by media_marker in text for item in media_items: From 3e031d5de16d5bd81dd35ef2cc3b8e2d49fac063 Mon Sep 17 00:00:00 2001 From: Alcoft Date: Tue, 5 May 2026 17:46:08 +0200 Subject: [PATCH 069/304] Fixed some bugs. --- llama_cpp/llama_chat_format.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index ab5e438d3e..40491968a9 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3874,6 +3874,18 @@ def __init__( self._chat_format_parser_tags += ["[IMG]"] elif verbose: print("Warning: Could not determine chat format parser tags.", flush = True) + + def __call__(self, **kwargs): + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix} - Start processing") + + # Use parent implementation + return super().__call__(**kwargs) class Llava15ChatHandler(MTMDChatHandler): CHAT_FORMAT = ( From 389d0d97babca3edcf6fb74f476e306a21183b5f Mon Sep 17 00:00:00 2001 From: Alcoft Date: Tue, 5 May 2026 18:49:21 +0200 Subject: [PATCH 070/304] Implemented 'chat_handler_kwargs'. --- llama_cpp/llama.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 848706a90d..6dab44602d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -152,6 +152,7 @@ def __init__( spm_infill: bool = False, verbose: bool = True, # Extra Params + chat_handler_kwargs: Dict[str, Any] = {}, **kwargs, # type: ignore ): """Load a llama.cpp model from `model_path`. @@ -618,7 +619,8 @@ def __init__( gguf_metadata = self.metadata, clip_model_path = clip_model_path, model_arch = None, - verbose = self.verbose + verbose = self.verbose, + **chat_handler_kwargs ) eos_token_id = self.token_eos() From 867a579ef9440f02f9bf0849ff37e30b7fd4deda Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 6 May 2026 02:06:52 +0800 Subject: [PATCH 071/304] Update Submodule vendor/llama.cpp e48034d..bbeb89d Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e48034dfc9..bbeb89d76c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e48034dfc9e5705248fd39dc437ca887dc55a528 +Subproject commit bbeb89d76c41bc250f16e4a6fefcc9b530d6e3f3 From 3796562b0cb4397bc13b295a8d8e8433f4919005 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 6 May 2026 02:46:42 +0800 Subject: [PATCH 072/304] Sync llama : add option to save memory in device buffers Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 416e8b9357..1efd645150 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -2770,6 +2770,9 @@ def llama_state_seq_load_file( # // work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY = 1 +# // keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load) +LLAMA_STATE_SEQ_FLAGS_ON_DEVICE = 2 + llama_state_seq_flags = ctypes.c_uint32 # LLAMA_API size_t llama_state_seq_get_size_ext( From 8eafd9edacb20fbc461081ae16b5afc8b5cd883c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 6 May 2026 05:07:26 +0800 Subject: [PATCH 073/304] feat(HybridCheckpointCache): add `on-device` hybrid checkpoint support - Add dual-mode HybridCheckpointCache behavior Host mode keeps the existing Python-owned full checkpoint path. Device mode forwards checkpoint tensor payloads to llama.cpp-owned device buffers. - Add on_device support for llama.cpp sequence state APIs The cache now forwards LLAMA_STATE_SEQ_FLAGS_ON_DEVICE when requested. This aligns Python checkpoint behavior with upstream llama.cpp device-backed state storage. - Keep host checkpoint mode as the default on_device remains disabled by default for compatibility. Existing multi-checkpoint rollback behavior stays unchanged unless explicitly enabled. - Preserve multi-checkpoint history in host mode Host-backed checkpoints still store full serialized payloads in Python bytes. This keeps historical rollback safe for multi-turn reuse. - Add safe per-seq behavior for device mode Device-backed tensor payloads are owned by llama_context and keyed by seq_id. The cache now replaces old checkpoint metadata for the same seq_id before saving a new one. - Guard against stale on-device checkpoint restores Old Python checkpoint objects may outlive the current device payload. Restore now refuses stale on-device checkpoints to avoid mixing old metadata with newer device tensors. - Add shared FIFO eviction for checkpoint entries Checkpoint eviction is now handled through a common helper. This keeps max_checkpoints respected for both host metadata and device-mode metadata. - Clarify HybridCheckpoint data ownership semantics The dataclass docs now distinguish full host-side payloads from host-visible device-mode metadata. This makes it clear that Python does not own VRAM checkpoint tensors. - Improve cache_size documentation cache_size now describes host-visible memory usage. In device mode, it intentionally excludes llama_context-owned device tensor storage. - Expand save and restore diagnostics Verbose logs now include checkpoint mode, seq_id, position, count, and tracked memory usage. This should make hybrid rollback debugging much easier. - Rename internal state flags from _flag_partial to _flags The name now reflects that multiple sequence state flags may be combined. This is clearer now that PARTIAL_ONLY and ON_DEVICE can both be active. - Add checkpoint_on_device to Llama.__init__ Users can now enable device-backed hybrid checkpoints from the high-level Llama wrapper. The option is passed directly into HybridCheckpointCache as on_device. - Reduce default ctx_checkpoints from 32 to 16 This lowers default checkpoint memory pressure. Host mode can still be tuned higher when deeper rollback history is needed. - Document checkpoint_on_device in Llama init args The new argument explains that tensor payloads are stored in llama_context-owned device buffers. It also clarifies the tradeoff between lower device-to-host copy overhead and one active checkpoint per seq_id. - Improve hybrid cache initialization logs Llama.__init__ now prints ctx_checkpoints, checkpoint_interval, and on_device when hybrid checkpointing is enabled. This makes runtime configuration easier to verify from stderr. Signed-off-by: JamePeng --- llama_cpp/llama.py | 27 +++- llama_cpp/llama_cache.py | 293 ++++++++++++++++++++++++++++++++------- 2 files changed, 267 insertions(+), 53 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 1241f81e26..e50e3b9a3b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -131,8 +131,9 @@ def __init__( swa_full: Optional[bool] = None, kv_unified: Optional[bool] = None, # HybridCheckpointCache Params - ctx_checkpoints: int = 32, + ctx_checkpoints: int = 16, checkpoint_interval: int = 4096, + checkpoint_on_device: bool = False, # Sampling Params last_n_tokens_size: int = 64, # Backend Params @@ -227,6 +228,7 @@ def __init__( kv_unified: use single unified KV buffer for the KV cache of all sequences ctx_checkpoints: max number of context checkpoints to create per slot (default: 16)[(more info)](https://github.com/ggml-org/llama.cpp/pull/15293) checkpoint_interval: Hybrid model checkpoint token intervals, and archiving of text with interval sizes along the way. + checkpoint_on_device: Store hybrid/recurrent checkpoint tensor payloads in llama_context-owned device buffers via LLAMA_STATE_SEQ_FLAGS_ON_DEVICE. last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque. numa: numa policy chat_format: String specifying the chat format to use when calling create_chat_completion. @@ -541,6 +543,7 @@ def __init__( _is_recurrent = self._model.is_recurrent() _is_hybrid = self._model.is_hybrid() _n_swa = self._model.n_swa() + # Sync llama.cpp upstream (#20291): warn swa-full is not supported for non-SWA models. if _n_swa == 0: if (self.context_params.swa_full): @@ -555,13 +558,25 @@ def __init__( if self.is_hybrid: if self.verbose: - print(f"Llama.__init__: Hybrid/Recurrent model detected." - f"(is_recurrent: {_is_recurrent}, is_hybrid: {_is_hybrid}, n_swa: {_n_swa}, swa_full: {self.context_params.swa_full}). " - f" Enabling HybridCheckpointCache(ctx_checkpoints={ctx_checkpoints}, checkpoint_interval={checkpoint_interval}).", - file=sys.stderr) + print( + f"Llama.__init__: Hybrid/Recurrent model detected. " + f"(is_recurrent: {_is_recurrent}, is_hybrid: {_is_hybrid}, " + f"n_swa: {_n_swa}, swa_full: {self.context_params.swa_full}). " + f"Enabling HybridCheckpointCache(" + f"ctx_checkpoints={ctx_checkpoints}, " + f"checkpoint_interval={checkpoint_interval}, " + f"on_device={checkpoint_on_device}).", + file=sys.stderr, + ) self.ctx_checkpoints = ctx_checkpoints self.checkpoint_interval = checkpoint_interval - self._hybrid_cache_mgr = HybridCheckpointCache(self._ctx.ctx, max_checkpoints=self.ctx_checkpoints, verbose=self.verbose) + self.checkpoint_on_device = checkpoint_on_device + self._hybrid_cache_mgr = HybridCheckpointCache( + self._ctx.ctx, + max_checkpoints=self.ctx_checkpoints, + on_device=self.checkpoint_on_device, + verbose=self.verbose, + ) else: self._hybrid_cache_mgr = None diff --git a/llama_cpp/llama_cache.py b/llama_cpp/llama_cache.py index dc1dd20d7c..ee37df1200 100644 --- a/llama_cpp/llama_cache.py +++ b/llama_cpp/llama_cache.py @@ -352,58 +352,169 @@ def __setitem__(self, key: Sequence[int], value: "llama_core.LlamaState"): @dataclass class HybridCheckpoint: - """Represents a single snapshot of the RNN/Hybrid model's hidden state.""" - pos: int # The token position (cursor) where this snapshot was taken - data: bytes # The raw binary RNN state data - hash_val: str # SHA-256 hash of the token prefix to ensure exact sequence matching - size: int # Size of the state data in bytes - seq_id: int # Sequence ID this checkpoint belongs to + """ + Represents a single snapshot of the Hybrid/Recurrent model state. + + Notes: + - When on_device=False, `data` contains the full host-side serialized state. + - When on_device=True, `data` contains only the host-visible portion of the + serialized state. The tensor payload is stored in llama_context-owned + device buffers by llama.cpp, keyed by seq_id. + """ + pos: int # The token position (cursor) where this snapshot was taken. + data: bytes # The raw binary RNN state data. + hash_val: str # SHA-256 hash of the token prefix to ensure exact sequence matching. + size: int # Number of bytes written by llama_state_seq_get_data_ext(). + seq_id: int # Sequence id used by llama.cpp state APIs. class HybridCheckpointCache(BaseLlamaCache): """ - Manager for RNN state snapshots (Checkpoints) tailored for Hybrid/Recurrent models. - Provides rollback capabilities for models that cannot physically truncate KV cache. + Checkpoint manager for Hybrid/Recurrent model states. + + This cache is designed for models whose memory cannot be safely truncated like + a regular Transformer KV cache. For recurrent/hybrid architectures, rollback is + implemented by saving and restoring sequence state snapshots. + + Two operating modes are supported: + + 1. Host mode: on_device=False + - Full checkpoint payload is materialized as Python bytes. + - Multiple checkpoints per seq_id are safe. + - This mode is suitable for multi-turn rollback and longer conversation reuse. + + 2. Device mode: on_device=True + - LLAMA_STATE_SEQ_FLAGS_ON_DEVICE is forwarded to llama.cpp. + - Tensor payloads are stored in llama_context-owned device buffers. + - The device buffers are created per seq_id in llama.cpp. + - Therefore only one active checkpoint per seq_id is safe. + - This mode is suitable for fast speculative / branch rollback where avoiding + device-to-host tensor copies is more important than keeping many historical + checkpoints. + + Important: + Do not treat on_device=True as "Python owns a VRAM checkpoint". Python only + owns the host-visible serialized portion. The tensor payload lives inside the + llama_context and is keyed by seq_id. """ - def __init__(self, ctx: llama_cpp_lib.llama_context_p, max_checkpoints: int = 16, verbose: bool = False): + def __init__( + self, + ctx: llama_cpp_lib.llama_context_p, + max_checkpoints: int = 16, + on_device: bool = False, + verbose: bool = False + ): + """ + Args: + ctx (llama_context_p): + Borrowed llama.cpp context pointer used by the state sequence APIs. + This cache does not own the context and must not free it. + + max_checkpoints(int): Maximum number of Python-side checkpoint entries to keep. + - Host mode: This is the maximum number of historical checkpoints across all seq_ids. + - Device mode: This is still a global upper bound for Python-side metadata entries, + but this class also enforces at most one active checkpoint per seq_id, + because llama.cpp stores device tensor payloads per seq_id. + + on_device(bool): Whether to request llama.cpp to keep tensor checkpoint payloads in + context-owned device buffers via LLAMA_STATE_SEQ_FLAGS_ON_DEVICE. + + verbose(bool): Enables diagnostic logging to stderr for checkpoint save/restore/eviction. + """ if ctx is None: - raise ValueError("HybridCheckpointCache(__init__): Failed to create HybridCheckpointCache with model context") + raise ValueError("HybridCheckpointCache(__init__): Failed to create HybridCheckpointCache with a null model context") self._ctx = ctx + self.on_device = on_device + self.verbose = verbose + + # In host mode, max_checkpoints means "maximum number of Python-owned + # checkpoints across all seq_ids". + # + # In device mode, llama.cpp stores tensor payloads in device buffers keyed + # by seq_id. Multiple Python checkpoint metadata entries for the same seq_id + # would point to the same mutable device-side slot, so only one checkpoint + # per seq_id is safe. self.max_checkpoints = max_checkpoints + + # Python-side checkpoint registry. + # + # Host mode: + # Each HybridCheckpoint owns a full serialized checkpoint payload. + # + # Device mode: + # Each HybridCheckpoint owns only the host-visible serialized portion. + # The corresponding tensor payload is owned by llama_context. self.checkpoints: list[HybridCheckpoint] = [] + + # Total Python-tracked checkpoint size in bytes. + # + # Host mode: + # Roughly equals the total serialized checkpoint payload size. + # + # Device mode: + # Tracks only the host-visible part returned by llama.cpp, not the + # context-owned device tensor storage. self._current_size = 0 - # Cache C-type API function pointers for performance + # Cache C API function pointers for faster repeated calls. self._get_size_ext = llama_cpp_lib.llama_state_seq_get_size_ext self._get_data_ext = llama_cpp_lib.llama_state_seq_get_data_ext self._set_data_ext = llama_cpp_lib.llama_state_seq_set_data_ext - self._flag_partial = llama_cpp_lib.LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY - self.verbose = verbose - - if self.max_checkpoints <= 0: - if self.verbose: - import sys - print("HybridCheckpointCache(__init__): Cache is DISABLED (max_checkpoints <= 0). " - "Rollback capabilities are turned off. This is optimal for single-turn workflows.", - file=sys.stderr) + # State serialization flags forwarded to llama.cpp. + # + # LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY: + # Save only the sequence-specific / partial state needed for recurrent + # rollback instead of a full context state. + # + # LLAMA_STATE_SEQ_FLAGS_ON_DEVICE: + # Ask llama.cpp to store tensor payloads in context-owned device buffers. + self._flags = llama_cpp_lib.LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY + if on_device: + self._flags |= llama_cpp_lib.LLAMA_STATE_SEQ_FLAGS_ON_DEVICE + + if self.max_checkpoints <= 0 and self.verbose: + print("HybridCheckpointCache(__init__): Cache is DISABLED (max_checkpoints <= 0). " + "Rollback capabilities are turned off. This is optimal for single-turn workflows.", + file=sys.stderr) + + if self.on_device and self.max_checkpoints > 1 and self.verbose: + print( + "HybridCheckpointCache(__init__): on_device=True stores tensor payloads " + "in llama_context-owned device buffers keyed by seq_id. Multiple " + "historical checkpoints for the same seq_id are unsafe, so this cache " + "will keep only one checkpoint per seq_id.", + file=sys.stderr, + ) @property def cache_size(self) -> int: - """Returns the total memory used by all stored checkpoints in bytes.""" + """ + Returns the host-visible checkpoint size tracked by Python. + + In host mode, this is close to the full serialized checkpoint payload size. + In device mode, this is only the host-visible metadata/payload size returned + by llama.cpp. Device-side tensor storage is owned by llama_context and is not + fully represented by this number. + """ return self._current_size def clear(self): - """Clears all stored checkpoints and resets memory tracking.""" + """ + Clears Python-side checkpoint metadata. + + This does not explicitly release llama_context-owned device buffers. The + device buffers are managed by llama.cpp and are associated with the context. + """ if not self.checkpoints: # Empty Checkpoint: Return immediately, no need to clear. return self.checkpoints.clear() self._current_size = 0 if self.verbose: - print("HybridCheckpointCache: cleared") + print("HybridCheckpointCache(clear): cleared", file=sys.stderr) def close(self): - self.checkpoints = None + self.clear() self._ctx = None self._get_size_ext = None self._get_data_ext = None @@ -421,23 +532,72 @@ def _hash_prefix(self, tokens: List[int], length: int) -> str: """ if length <= 0: return "empty" - tokens_size = len(tokens) - if length > tokens_size: - length = tokens_size + length = min(length, len(tokens)) data = array.array('i', tokens[:length]).tobytes() return hashlib.sha256(data).hexdigest()[:32] + def _replace_checkpoint_for_seq_id(self, seq_id: int) -> None: + """ + Removes all Python-side checkpoints for one seq_id. + + Required for on_device=True because llama.cpp stores the device tensor + payload per seq_id, not per Python checkpoint object. + """ + kept: list[HybridCheckpoint] = [] + removed_size = 0 + + for cp in self.checkpoints: + if cp.seq_id == seq_id: + removed_size += cp.size + else: + kept.append(cp) + + self.checkpoints = kept + self._current_size -= removed_size + if self._current_size < 0: + self._current_size = 0 + + def _evict_checkpoints_if_needed(self) -> None: + """ + Evicts old checkpoints if needed + + Host mode: + This evicts full Python-owned checkpoint payloads, so FIFO historical + checkpoints are safe and useful. + + Device mode: + This evicts Python-side metadata only. The device tensor payload is owned + by llama_context and is keyed by seq_id. + """ + while len(self.checkpoints) > self.max_checkpoints: + old_cp = self.checkpoints.pop(0) + self._current_size -= old_cp.size + if self._current_size < 0: + self._current_size = 0 + + if self.verbose: + print( + f"HybridCheckpointCache: evicted checkpoint " + f"seq_id={old_cp.seq_id}, pos={old_cp.pos}", + file=sys.stderr, + ) + def find_best_checkpoint(self, tokens: List[int], seq_id: int = 0) -> Optional[HybridCheckpoint]: """ Finds the longest valid checkpoint that perfectly matches the provided token prefix. + + The hash check prevents restoring a checkpoint that has the same length but + belongs to a different prompt/history. + Returns None if no matching checkpoint is found. """ # Empty Checkpoint: Instant return, no hash calculation needed. if self.max_checkpoints <= 0 or len(self.checkpoints) == 0: return None - best_cp = None + best_cp: Optional[HybridCheckpoint] = None best_pos = -1 + for cp in self.checkpoints: if cp.seq_id != seq_id or cp.pos > len(tokens): # Skip if sequence ID mismatches or checkpoint is longer than the current prompt @@ -475,9 +635,17 @@ def save_checkpoint( file=sys.stderr) return False - flags = self._flag_partial + # In on-device mode, remove old Python metadata for this seq_id before saving + # the new checkpoint. The underlying llama.cpp device buffer for this seq_id + # will be overwritten by the get_data_ext() call. + if self.on_device: + self._replace_checkpoint_for_seq_id(seq_id) + + flags = self._flags - # 1. Query the required buffer size from the underlying C++ context + # 1. Query the required host-visible buffer size. + # In on_device mode this may exclude the large tensor payload + # that stays in device memory. size = self._get_size_ext(self._ctx, seq_id, flags) if size == 0: if self.verbose: @@ -487,9 +655,14 @@ def save_checkpoint( # 2. Allocate buffer and extract raw state data buffer = (ctypes.c_uint8 * size)() n_written = self._get_data_ext(self._ctx, buffer, size, seq_id, flags) + if n_written != size: if self.verbose: - print(f"HybridCheckpointCache(save_checkpoint): get failed {n_written}/{size}") + print( + f"HybridCheckpointCache(save_checkpoint): get_data_ext failed " + f"({n_written}/{size})", + file=sys.stderr, + ) return False # Note: This deep copy isolates the state from subsequent C++ backend mutations @@ -506,19 +679,18 @@ def save_checkpoint( ) self._current_size += n_written - # 4. Enforce capacity limits (FIFO eviction) - while len(self.checkpoints) > self.max_checkpoints: - if not self.checkpoints: - break - old_cp = self.checkpoints.pop(0) - self._current_size -= old_cp.size - if self.verbose: - print(f"HybridCheckpointCache(save_checkpoint): evicted pos={old_cp.pos}") + # 4. Evicts old checkpoints if needed + self._evict_checkpoints_if_needed() if self.verbose: - print(f"HybridCheckpointCache(save_checkpoint): Saved checkpoint at pos {current_pos} ({size / 1024 / 1024:.2f} MiB) " - f"total={len(self.checkpoints)} used={self._current_size / 1024 / 1024:.2f} MiB", - file=sys.stderr) + mode = "device" if self.on_device else "host" + print( + f"HybridCheckpointCache(save_checkpoint): saved {mode} checkpoint " + f"seq_id={seq_id}, pos={current_pos}, size={size / 1024 / 1024:.2f} MiB, " + f"hcc_count={len(self.checkpoints)}, " + f"hcc_mem_used={self._current_size / 1024 / 1024:.2f} MiB", + file=sys.stderr, + ) return True @@ -531,17 +703,38 @@ def restore_checkpoint(self, cp: HybridCheckpoint, seq_id: int = 0) -> bool: if self.verbose: print(f"HybridCheckpointCache(restore_checkpoint): [Error] Sequence ID mismatch: checkpoint has {cp.seq_id}, requested {seq_id}", file=sys.stderr) return False - flags = self._flag_partial - # 2. Verify the underlying C++ context still expects the exact same state size. + # 2. Guard against stale on-device checkpoint objects. + # + # In on_device mode, Python does not own the full checkpoint tensor payload. + # llama.cpp keeps the large tensor payload in llama_context-owned device + # buffers keyed by seq_id. Saving a newer checkpoint for the same seq_id may + # overwrite that device-side payload while an old HybridCheckpoint object can + # still exist outside this cache. + # + # Only checkpoint objects still tracked by this cache are considered valid. + # This avoids restoring old Python metadata together with newer device tensors. + if self.on_device and cp not in self.checkpoints: + if self.verbose: + print( + "HybridCheckpointCache(restore_checkpoint): stale on-device checkpoint; " + "refusing restore because device payload may have been overwritten.", + file=sys.stderr, + ) + return False + + flags = self._flags + + # 3. Verify the underlying C++ context still expects the exact same state size. # This prevents buffer overflows if the backend context was unexpectedly altered or reallocated. current_size = self._get_size_ext(self._ctx, seq_id, flags) if current_size != cp.size: if self.verbose: - print(f"HybridCheckpointCache(restore_checkpoint): [Warning] State size mismatch before restore: expected {cp.size}, got {current_size} -> possible invalidation") + print(f"HybridCheckpointCache(restore_checkpoint): [Warning] State size mismatch before restore: " + f"expected checkpoint size={cp.size}, got current size={current_size} -> possible invalidation") return False - # 3. Copy data back to a ctypes buffer and push to the C++ backend + # 4. Copy data back to a ctypes buffer and push to the C++ backend buffer = (ctypes.c_uint8 * cp.size).from_buffer_copy(cp.data) ret = self._set_data_ext( self._ctx, buffer, cp.size, seq_id, flags @@ -549,7 +742,13 @@ def restore_checkpoint(self, cp: HybridCheckpoint, seq_id: int = 0) -> bool: success = (ret == cp.size) if self.verbose: - print(f"HybridCheckpointCache(restore_checkpoint): restore {'OK' if success else 'FAIL'} pos={cp.pos}") + mode = "device" if self.on_device else "host" + print( + f"HybridCheckpointCache(restore_checkpoint): restore " + f"{'OK' if success else 'FAIL'} " + f"mode={mode}, seq_id={seq_id}, pos={cp.pos}", + file=sys.stderr, + ) return success # Disable BaseLlamaCache Dictionary Interfaces From 54115b4e86c5ffedc2e84237ff03b2473570d756 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 6 May 2026 05:08:14 +0800 Subject: [PATCH 074/304] Update /docs/wiki/core/Llama.md for `on_device` option Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 52 ++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 7a9b7bd6ad..0354d86150 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -4,13 +4,13 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-05-01 +last_updated: 2026-05-06 version_target: "latest" --- ``` ## Overview -The `Llama` class is the core, high-level Python wrapper for a `llama.cpp` model. It handles model loading, memory management (KV cache), tokenization, and generation (both base text completion and chat formatting). It includes advanced features like dynamic LoRA routing, hybrid model checkpointing, speculative decoding, and context shifting. +The `Llama` class is the core, high-level Python wrapper for a `llama.cpp` model. It handles model loading, memory management (KV cache), tokenization, and generation (both base text completion and chat formatting). It includes advanced features like dynamic LoRA routing, dual-mode hybrid/recurrent checkpointing, speculative decoding, and context shifting. ## Constructor (`__init__`) @@ -51,8 +51,9 @@ Initialize the model and context. Note that model loading will immediately alloc | `chat_format` | `str` | `None` | String specifying the chat template (e.g., `"llama-2"`, `"chatml"`). Guessed from GGUF if None. | | `chat_handler` | `LlamaChatCompletionHandler` | `None` | Optional custom handler. See [[ChatHandlers]]. | | `draft_model` | `LlamaDraftModel` | `None` | Optional draft model for speculative decoding. | -| `ctx_checkpoints` | `int` | `32` | Max context checkpoints per slot (Hybrid/SWA models). | -| `checkpoint_interval`| `int`| `4096` | Token interval for saving Hybrid model checkpoints. | +| `ctx_checkpoints` | `int` | `16` | Max hybrid/recurrent context checkpoints to keep. Set to `0` to disable checkpointing for single-turn fast paths. | +| `checkpoint_interval` | `int` | `4096` | Token interval for saving periodic Hybrid/Recurrent checkpoints during long prompt evaluation. | +| `checkpoint_on_device` | `bool` | `False` | Store Hybrid/Recurrent checkpoint tensor payloads in `llama_context`-owned device buffers via `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE`. Reduces device-to-host copy overhead, but only one active checkpoint per `seq_id` is safe. | *(Note: There are numerous additional RoPE/YaRN scaling parameters available for specialized context extension. Refer to the source code for the full list).* @@ -189,18 +190,41 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn 5. **Hybrid & Recurrent Architectures**: - The class natively detects Hybrid/Recurrent models (like LFM2VL/LFM2.5VL, Qwen3.5/3.6, Mamba or specialized SWA models(Gemma3/4)) and automatically enables the `HybridCheckpointCache`. This creates periodic save-states during large context pre-filling, allowing the model to roll back seamlessly if a generation is rejected (e.g., speculative decoding mismatches) without corrupting the recurrent state. + The class natively detects Hybrid/Recurrent models (for example LFM2VL/LFM2.5VL, Qwen3.5/3.6, Mamba, RWKV, or specialized SWA models such as Gemma3/4) and automatically enables the `HybridCheckpointCache`. - * Tips: If you are using hybrid multimodal model for building ComfyUI nodes or running single-turn API wrappers where you do not need multi-turn state rollbacks, simply initialize your Llama instance with `ctx_checkpoints=0`: + Unlike regular Transformer KV caches, Hybrid/Recurrent model memory cannot always be safely truncated token-by-token. The wrapper therefore saves periodic sequence-state checkpoints during long context prefill, allowing rollback to a verified prefix without corrupting recurrent state. + + `HybridCheckpointCache` supports two checkpoint storage modes: + + - **Host checkpoint mode** (`checkpoint_on_device=False`, default): checkpoint payloads are serialized into Python-owned bytes. This supports multiple historical checkpoints per `seq_id`, which is useful for multi-turn reuse and deeper rollback history. + - **Device checkpoint mode** (`checkpoint_on_device=True`): checkpoint tensor payloads are stored in `llama_context`-owned device buffers via `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE`. Python only keeps the host-visible serialized portion. This reduces device-to-host tensor copy overhead, but only one active checkpoint per `seq_id` is safe because device payloads are keyed by `seq_id`. + + *Tips*: If you are using a hybrid multimodal model for ComfyUI nodes or single-turn API wrappers where you do not need multi-turn state rollback, initialize your `Llama` instance with `ctx_checkpoints=0`: + + ```python + llm = Llama( + model_path="./Qwen3.5-VL-9B.gguf", + chat_handler=MTMDChatHandler(clip_model_path="./mmproj.gguf"), + n_ctx=4096, + ctx_checkpoints=0 # Disable checkpoints for zero-latency single-turn fast paths + ) + ``` + + For long prompts on GPU-backed Hybrid/Recurrent models, you can enable device-backed checkpoints to reduce device-to-host copy overhead: + + ```python + llm = Llama( + model_path="./Qwen3.6-27B.gguf", + n_ctx=32768, + n_gpu_layers=-1, + ctx_checkpoints=16, + checkpoint_interval=4096, + checkpoint_on_device=True + ) + ``` + + Use `checkpoint_on_device=False` if you need multiple historical checkpoints for the same `seq_id`. Use `checkpoint_on_device=True` when fast rollback/checkpointing is more important than keeping many historical checkpoint payloads. - ```python - llm = Llama( - model_path="./Qwen3.5-VL-9B.gguf", - chat_handler=MTMDChatHandler(clip_model_path="./mmproj.gguf"), - n_ctx=4096, - ctx_checkpoints=0 # <-- SET THIS TO 0 TO ENABLE ZERO-LATENCY FAST PATH - ) - ``` 6. **Assistant Prefill**: `llama-cpp-python` supports native **Assistant Prefill** for seamless message continuation. You can now simply use the `assistant_prefill=True` parameter in the `create_chat_completion` function. From f8d88b014889f7592d6fc3caaf23cdb9e3b088f2 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 6 May 2026 05:22:36 +0800 Subject: [PATCH 075/304] Update /docs/wiki/modules/LlamaCache.md for `on_device` option Signed-off-by: JamePeng --- docs/wiki/modules/LlamaCache.md | 285 +++++++++++++++++++++++--------- 1 file changed, 205 insertions(+), 80 deletions(-) diff --git a/docs/wiki/modules/LlamaCache.md b/docs/wiki/modules/LlamaCache.md index 64e6bbb5f8..d1db0a2097 100644 --- a/docs/wiki/modules/LlamaCache.md +++ b/docs/wiki/modules/LlamaCache.md @@ -2,7 +2,7 @@ title: Llama Cache module_name: llama_cpp.llama_cache source_file: llama_cpp/llama_cache.py -last_updated: 2026-05-02 +last_updated: 2026-05-06 version_target: "latest" --- @@ -21,10 +21,10 @@ It defines several cache classes: | `BaseLlamaCache` | Abstract base class for llama.cpp state caches. | | `LlamaRAMCache` | In-memory LRU cache for `LlamaState` objects. | | `LlamaDiskCache` | Disk-backed cache using the `diskcache` library. | -| `LlamaTrieCache` | Trie-based cache optimized for fast longest-prefix lookup. | -| `HybridCheckpointCache` | Checkpoint manager for RNN/Hybrid model hidden states. | -| `HybridCheckpoint` | Dataclass representing one saved hybrid model checkpoint. | | `TrieNode` | Internal trie node used by `LlamaTrieCache`. | +| `LlamaTrieCache` | Trie-based cache optimized for fast longest-prefix lookup. | +| `HybridCheckpoint` | Dataclass representing one saved Hybrid/Recurrent checkpoint and its host-visible payload. | +| `HybridCheckpointCache` | Checkpoint manager for Hybrid/Recurrent model state snapshots, with host and device-backed modes. | The public compatibility alias is: @@ -910,7 +910,7 @@ from llama_cpp.llama_cache import LlamaTrieCache as LlamaCache ## Overview -`HybridCheckpoint` is a dataclass representing one saved snapshot of a Hybrid or recurrent model's hidden state. +`HybridCheckpoint` is a dataclass representing one saved snapshot of a Hybrid or Recurrent model state. It is used by `HybridCheckpointCache`. @@ -920,9 +920,14 @@ Defined in: `llama_cpp/llama_cache.py` ## Role in the API -Hybrid or recurrent models may require hidden-state rollback rather than standard KV-cache truncation. +Hybrid or recurrent models may require sequence-state rollback rather than standard KV-cache truncation. + +`HybridCheckpoint` stores the checkpoint position, prefix verification hash, sequence id, and the serialized checkpoint payload visible to Python. -`HybridCheckpoint` stores enough metadata to verify and restore a specific recurrent state snapshot. +Its `data` field has different ownership semantics depending on the cache mode: + +* In host mode (`on_device=False`), `data` contains the full host-side serialized checkpoint state. +* In device mode (`on_device=True`), `data` contains only the host-visible serialized portion. The large tensor payload is stored in `llama_context`-owned device buffers by llama.cpp, keyed by `seq_id`. --- @@ -936,19 +941,19 @@ class HybridCheckpoint: hash_val: str size: int seq_id: int -``` +```` --- ## Fields -| Field | Type | Description | -| ---------- | ------- | --------------------------------------------------------------- | -| `pos` | `int` | Token position where this checkpoint was taken. | -| `data` | `bytes` | Raw binary RNN or Hybrid model state data. | -| `hash_val` | `str` | SHA-256 hash prefix used to verify exact token-prefix matching. | -| `size` | `int` | Size of the state data in bytes. | -| `seq_id` | `int` | Sequence ID associated with this checkpoint. | +| Field | Type | Description | +| ---------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `pos` | `int` | Token position where this checkpoint was taken. | +| `data` | `bytes` | Serialized checkpoint payload visible to Python. In host mode this is the full state; in device mode this is only the host-visible portion. | +| `hash_val` | `str` | SHA-256 hash prefix used to verify exact token-prefix matching. | +| `size` | `int` | Number of bytes written by `llama_state_seq_get_data_ext`. | +| `seq_id` | `int` | Sequence id used by llama.cpp sequence-state APIs. | --- @@ -958,23 +963,33 @@ class HybridCheckpoint: Users usually do not need to instantiate this dataclass manually. +In device mode, old `HybridCheckpoint` Python objects may become stale if a newer checkpoint is saved for the same `seq_id`, because the device-side tensor payload is keyed by `seq_id` and may be overwritten. + --- # `HybridCheckpointCache` ## Overview -`HybridCheckpointCache` manages RNN or Hybrid model hidden-state checkpoints. +`HybridCheckpointCache` manages Hybrid/Recurrent model state checkpoints. + +It is designed for models whose memory cannot always be safely truncated like a regular Transformer KV cache. Instead, rollback is implemented by saving and restoring sequence-state snapshots through llama.cpp state APIs. + +The cache supports two operating modes: -It is designed for models that cannot physically truncate KV cache in the same way as standard transformer-only models. +1. **Host mode** (`on_device=False`) -Instead of implementing dictionary-style cache operations, it provides explicit checkpoint operations: + * Full checkpoint payloads are materialized as Python-owned `bytes`. + * Multiple historical checkpoints per `seq_id` are safe. + * This is the default mode and is useful for multi-turn rollback or deeper prefix reuse. -* `save_checkpoint` -* `find_best_checkpoint` -* `restore_checkpoint` -* `clear` -* `close` +2. **Device mode** (`on_device=True`) + + * `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE` is forwarded to llama.cpp. + * Tensor payloads are stored in `llama_context`-owned device buffers. + * Python keeps only the host-visible serialized portion. + * Only one active checkpoint per `seq_id` is safe because device payloads are keyed by `seq_id`. + * This mode can reduce device-to-host copy overhead during checkpoint save/restore. Defined in: `llama_cpp/llama_cache.py` @@ -984,12 +999,14 @@ Defined in: `llama_cpp/llama_cache.py` `HybridCheckpointCache` is a specialized cache manager for Hybrid/Recurrent model rollback. -It stores raw state snapshots extracted from the llama.cpp backend through low-level C API functions: +It stores host-visible checkpoint data extracted from the llama.cpp backend through low-level C API functions: * `llama_state_seq_get_size_ext` * `llama_state_seq_get_data_ext` * `llama_state_seq_set_data_ext` +When `on_device=True`, tensor payloads are not treated as Python-owned bytes. They are stored by llama.cpp in `llama_context`-owned device buffers, while Python keeps the host-visible serialized portion and checkpoint metadata. + It is not a drop-in replacement for `LlamaRAMCache`, `LlamaDiskCache`, or `LlamaTrieCache`. --- @@ -1001,16 +1018,18 @@ def __init__( self, ctx: llama_cpp_lib.llama_context_p, max_checkpoints: int = 16, + on_device: bool = False, verbose: bool = False ): ... ``` -| Parameter | Type | Default | Required | Description | -| ----------------- | ------------------------------- | ------: | -------: | ------------------------------------------------------------------------------------------- | -| `ctx` | `llama_cpp_lib.llama_context_p` | ā€” | Yes | Low-level llama.cpp context pointer. Required for extracting and restoring sequence state. | -| `max_checkpoints` | `int` | `16` | No | Maximum number of checkpoints to retain. If set to `0` or below, checkpointing is disabled. | -| `verbose` | `bool` | `False` | No | Enables diagnostic messages printed to `stderr`. | +| Parameter | Type | Default | Required | Description | +| ----------------- | ------------------------------- | ------: | -------: | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `ctx` | `llama_cpp_lib.llama_context_p` | ā€” | Yes | Borrowed low-level llama.cpp context pointer used for sequence-state save/restore. The cache does not own or free this context. | +| `max_checkpoints` | `int` | `16` | No | Maximum number of Python-side checkpoint entries to retain. If set to `0` or below, checkpointing is disabled. | +| `on_device` | `bool` | `False` | No | Whether to request llama.cpp to store checkpoint tensor payloads in `llama_context`-owned device buffers via `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE`. | +| `verbose` | `bool` | `False` | No | Enables diagnostic messages printed to `stderr`. | --- @@ -1018,32 +1037,26 @@ def __init__( The constructor raises `ValueError` if `ctx` is `None`. -```python -if ctx is None: - raise ValueError( - "HybridCheckpointCache(__init__): Failed to create HybridCheckpointCache with model context" - ) -``` +If `max_checkpoints <= 0`, checkpointing is disabled. In verbose mode, the cache reports that rollback capabilities are turned off. This mode is intended to avoid expensive state extraction for single-turn workflows. -If `max_checkpoints <= 0`, checkpointing is disabled. In verbose mode, the cache reports that rollback capabilities are turned off. - -This mode is intended to avoid expensive state extraction for single-turn workflows. +When `on_device=True`, the cache forwards `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE` to llama.cpp. In this mode, the cache keeps only one active checkpoint per `seq_id` by replacing old Python-side checkpoint metadata before saving a new checkpoint for the same `seq_id`. --- ## Instance Variables -| Name | Type | Description | -| ----------------- | ------------------------------- | ------------------------------------------------------------------------------------------------ | -| `_ctx` | `llama_cpp_lib.llama_context_p` | Low-level llama.cpp context pointer used for state extraction and restoration. | -| `max_checkpoints` | `int` | Maximum number of checkpoints retained. Values less than or equal to zero disable checkpointing. | -| `checkpoints` | `list[HybridCheckpoint]` | Stored checkpoint objects. | -| `_current_size` | `int` | Total memory used by all stored checkpoints in bytes. | -| `_get_size_ext` | Callable | Cached reference to `llama_state_seq_get_size_ext`. | -| `_get_data_ext` | Callable | Cached reference to `llama_state_seq_get_data_ext`. | -| `_set_data_ext` | Callable | Cached reference to `llama_state_seq_set_data_ext`. | -| `_flag_partial` | int | Cached value of `LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY`. | -| `verbose` | `bool` | Enables debug output. | +| Name | Type | Description | +| ----------------- | ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `_ctx` | `llama_cpp_lib.llama_context_p` | Borrowed llama.cpp context pointer used for state extraction and restoration. | +| `on_device` | `bool` | Whether `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE` is forwarded to llama.cpp state APIs. | +| `verbose` | `bool` | Enables debug output. | +| `max_checkpoints` | `int` | Maximum number of Python-side checkpoint entries retained. Values less than or equal to zero disable checkpointing. | +| `checkpoints` | `list[HybridCheckpoint]` | Python-side checkpoint registry. In host mode, entries own full checkpoint payloads. In device mode, entries own only host-visible metadata/payload portions. | +| `_current_size` | `int` | Python-tracked host-visible checkpoint size in bytes. In device mode, this does not include `llama_context`-owned device tensor storage. | +| `_get_size_ext` | Callable | Cached reference to `llama_state_seq_get_size_ext`. | +| `_get_data_ext` | Callable | Cached reference to `llama_state_seq_get_data_ext`. | +| `_set_data_ext` | Callable | Cached reference to `llama_state_seq_set_data_ext`. | +| `_flags` | `int` | Combined llama.cpp sequence-state flags, always including `LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY` and optionally `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE`. | --- @@ -1057,7 +1070,11 @@ def cache_size(self) -> int: return self._current_size ``` -Returns the total memory used by stored checkpoints in bytes. +Returns the Python-tracked host-visible checkpoint size in bytes. + +In host mode, this is close to the full serialized checkpoint payload size. + +In device mode, this reports only the host-visible portion returned by llama.cpp. It does not include `llama_context`-owned device tensor storage. --- @@ -1070,14 +1087,16 @@ def clear(self): ... ``` -Clears all stored checkpoints and resets `_current_size` to `0`. +Clears Python-side checkpoint metadata and resets `_current_size` to `0`. If the checkpoint list is already empty, it returns immediately. +In device mode, this does not explicitly release `llama_context`-owned device buffers. Those buffers are managed by llama.cpp and are associated with the context. + In verbose mode, it prints: ```text -HybridCheckpointCache: cleared +HybridCheckpointCache(clear): cleared ``` --- @@ -1089,15 +1108,15 @@ def close(self): ... ``` -Releases references held by the cache. +Releases Python-side checkpoint metadata and detaches cached references held by the cache. Behavior: -* Sets `checkpoints` to `None`. +* Calls `clear()`. * Sets `_ctx` to `None`. * Sets cached C API function references to `None`. -This method is also called by `__del__`. +This method does not free the llama.cpp context itself, because the context is borrowed rather than owned by the cache. --- @@ -1133,6 +1152,50 @@ This hash is used to ensure checkpoints are restored only when the token prefix --- +### `_replace_checkpoint_for_seq_id` + +```python +def _replace_checkpoint_for_seq_id(self, seq_id: int) -> None: + ... +``` + +Removes all Python-side checkpoint entries for one `seq_id`. + +This is required in device mode because llama.cpp stores the device tensor payload per `seq_id`, not per Python checkpoint object. Keeping multiple checkpoint metadata entries for the same `seq_id` would be unsafe. + +Behavior: + +1. Iterates over all checkpoint entries. +2. Removes entries whose `seq_id` matches the requested `seq_id`. +3. Preserves entries for other sequence ids. +4. Subtracts removed checkpoint sizes from `_current_size`. +5. Clamps `_current_size` to `0` if needed. + +--- + +### `_evict_checkpoints_if_needed` + +```python +def _evict_checkpoints_if_needed(self) -> None: + ... +``` + +Evicts old checkpoint entries using FIFO order until `len(checkpoints) <= max_checkpoints`. + +In host mode, this evicts full Python-owned checkpoint payloads. + +In device mode, this evicts Python-side checkpoint metadata only. Device tensor payloads are owned by `llama_context`. + +Behavior: + +1. Checks whether the number of checkpoints exceeds `max_checkpoints`. +2. Pops the oldest checkpoint entry from the front of the list. +3. Subtracts its size from `_current_size`. +4. Clamps `_current_size` to `0` if needed. +5. Prints an eviction message in verbose mode. + +--- + ### `find_best_checkpoint` ```python @@ -1144,20 +1207,23 @@ def find_best_checkpoint( ... ``` -Finds the longest valid checkpoint matching the given token prefix and sequence ID. +Finds the longest valid checkpoint matching the given token prefix and sequence id. + +The hash check prevents restoring a checkpoint that has the same length but belongs to a different prompt/history. Returns `None` if: * Checkpointing is disabled. * There are no checkpoints. -* No checkpoint matches the requested sequence ID and token prefix. +* No checkpoint matches the requested sequence id and token prefix. Behavior: -1. Skips checkpoints whose `seq_id` differs. -2. Skips checkpoints whose `pos` is greater than the current token length. -3. Verifies token-prefix integrity using `_hash_prefix`. -4. Returns the checkpoint with the largest matching `pos`. +1. Returns immediately if `max_checkpoints <= 0` or no checkpoints exist. +2. Skips checkpoints whose `seq_id` differs from the requested `seq_id`. +3. Skips checkpoints whose `pos` is greater than the current token length. +4. Verifies token-prefix integrity using `_hash_prefix`. +5. Returns the checkpoint with the largest matching `pos`. --- @@ -1173,7 +1239,7 @@ def save_checkpoint( ... ``` -Extracts the current recurrent model state from the C++ backend and stores it as a `HybridCheckpoint`. +Extracts the current Hybrid/Recurrent model state from the C++ backend and stores it as a `HybridCheckpoint`. Returns `True` if the checkpoint was saved successfully. @@ -1186,20 +1252,24 @@ Returns `False` if: ### Behavior 1. Returns immediately if `max_checkpoints <= 0`. -2. Calls `_get_size_ext` to query the required state buffer size. -3. Allocates a `ctypes.c_uint8` buffer. -4. Calls `_get_data_ext` to extract state data. -5. Copies the state bytes into a Python `bytes` object. -6. Computes a hash of the token prefix. -7. Appends a new `HybridCheckpoint`. -8. Increments `_current_size`. -9. Evicts old checkpoints using FIFO order if the number of checkpoints exceeds `max_checkpoints`. +2. In device mode, removes old Python-side checkpoint metadata for the same `seq_id`. +3. Uses `_flags` to select partial-only state serialization, optionally with `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE`. +4. Calls `_get_size_ext` to query the required host-visible buffer size. +5. Allocates a `ctypes.c_uint8` buffer. +6. Calls `_get_data_ext` to extract the host-visible checkpoint data. +7. Copies the data into a Python `bytes` object. +8. Computes a hash of the token prefix. +9. Appends a new `HybridCheckpoint`. +10. Increments `_current_size`. +11. Evicts old checkpoint entries using FIFO order if the number of entries exceeds `max_checkpoints`. ### Important Performance Note The implementation intentionally bypasses checkpoint extraction when `max_checkpoints <= 0`. -This avoids potentially large synchronous VRAM-to-RAM transfers for single-turn workflows. +This avoids potentially large synchronous checkpoint extraction costs for single-turn workflows. + +When `on_device=True`, llama.cpp may keep large tensor payloads in context-owned device buffers instead of materializing them as Python-owned bytes. This can reduce device-to-host tensor copy overhead, but only one active checkpoint per `seq_id` is safe. --- @@ -1220,18 +1290,28 @@ Returns `True` if restoration succeeds. Returns `False` if: -* The checkpoint sequence ID does not match the requested `seq_id`. +* The checkpoint sequence id does not match the requested `seq_id`. +* `on_device=True` and the checkpoint object is no longer tracked by this cache. * The current backend state size differs from the checkpoint size. * The backend does not report the expected number of restored bytes. ### Behavior 1. Verifies `cp.seq_id == seq_id`. -2. Queries current expected state size from the backend. -3. Verifies it matches `cp.size`. -4. Copies checkpoint bytes into a ctypes buffer. -5. Calls `_set_data_ext` to restore the state. -6. Returns whether the number of restored bytes equals `cp.size`. +2. In device mode, rejects stale checkpoint objects that are no longer tracked by this cache. +3. Queries current expected host-visible state size from the backend. +4. Verifies it matches `cp.size`. +5. Copies checkpoint bytes into a ctypes buffer. +6. Calls `_set_data_ext` to restore the state. +7. Returns whether the number of restored bytes equals `cp.size`. + +### Stale Checkpoint Guard + +In device mode, Python does not own the full checkpoint tensor payload. The large tensor payload is stored inside `llama_context` device buffers keyed by `seq_id`. + +If a newer checkpoint is saved for the same `seq_id`, an older `HybridCheckpoint` Python object may still exist outside the cache, but its device-side tensor payload may have been overwritten. + +For this reason, `restore_checkpoint` refuses on-device checkpoint objects that are no longer tracked by the cache. This avoids restoring old Python metadata together with newer device tensors. --- @@ -1270,7 +1350,7 @@ Users should use checkpoint-specific methods instead. --- -## Example +## Example: Host-backed Checkpoints ```python from llama_cpp.llama_cache import HybridCheckpointCache @@ -1279,6 +1359,7 @@ from llama_cpp.llama_cache import HybridCheckpointCache checkpoint_cache = HybridCheckpointCache( ctx=ctx, max_checkpoints=16, + on_device=False, verbose=True, ) @@ -1299,16 +1380,57 @@ if saved: print("Restored:", restored) ``` -> Note: This example assumes `ctx` is already available from lower-level llama.cpp runtime code. Most high-level users do not manually create this cache. +Host mode stores full serialized checkpoint payloads in Python-owned `bytes`. Multiple historical checkpoints per `seq_id` are safe. + +--- + +## Example: Device-backed Checkpoints + +```python +from llama_cpp.llama_cache import HybridCheckpointCache + +# `ctx` must be a valid llama.cpp context pointer. +checkpoint_cache = HybridCheckpointCache( + ctx=ctx, + max_checkpoints=16, + on_device=True, + verbose=True, +) + +tokens = [1, 2, 3, 4] +current_pos = len(tokens) + +saved = checkpoint_cache.save_checkpoint( + current_pos=current_pos, + tokens=tokens, + seq_id=0, +) + +if saved: + checkpoint = checkpoint_cache.find_best_checkpoint(tokens, seq_id=0) + + if checkpoint is not None: + restored = checkpoint_cache.restore_checkpoint(checkpoint, seq_id=0) + print("Restored:", restored) +``` + +In device mode, llama.cpp owns the large tensor payload in context-owned device buffers. Python keeps only the host-visible checkpoint data and metadata. + +Only one active checkpoint per `seq_id` is safe. + +> Note: These examples assume `ctx` is already available from lower-level llama.cpp runtime code. Most high-level users do not manually create this cache. Instead, they configure it through the `Llama` constructor using `ctx_checkpoints`, `checkpoint_interval`, and `checkpoint_on_device`. --- ## Best Practices * Use `HybridCheckpointCache` only for Hybrid or recurrent model workflows that require hidden-state rollback. +* Keep `on_device=False` when you need multiple historical checkpoints for the same `seq_id`. +* Use `on_device=True` when reducing device-to-host checkpoint copy overhead is more important than keeping many historical checkpoint payloads. Only store the checkpoint seq_id and pos. * Set `max_checkpoints=0` for single-turn workflows where rollback is not needed. * Keep `max_checkpoints` small if checkpoint states are large. * Use `find_best_checkpoint` before calling `restore_checkpoint`. +* Do not hold and restore old on-device `HybridCheckpoint` objects after newer checkpoints have been saved for the same `seq_id`. * Do not use dictionary-style cache access with this class. --- @@ -1319,7 +1441,10 @@ if saved: * `max_checkpoints <= 0` disables checkpointing. * Restoring a checkpoint with the wrong `seq_id` fails. * Restore fails if the current backend state size no longer matches the checkpoint size. -* `close()` sets internal references to `None`; the object should not be reused afterward. +* In device mode, old `HybridCheckpoint` objects can become stale after a newer checkpoint is saved for the same `seq_id`. +* In device mode, `cache_size` does not include `llama_context`-owned device tensor storage. +* `clear()` removes Python-side checkpoint metadata but does not explicitly free llama.cpp-owned device buffers. +* `close()` detaches internal references; the object should not be reused afterward. * This class is not equivalent to `LlamaCache`. --- From 156226b00ebb6482cd83e26c917ff66aec65d104 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 12 May 2026 03:46:26 +0800 Subject: [PATCH 076/304] Update Submodule vendor/llama.cpp bbeb89d..a9883db --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index bbeb89d76c..a9883db8ee 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit bbeb89d76c41bc250f16e4a6fefcc9b530d6e3f3 +Subproject commit a9883db8ee021cf16783016a60996d41820b5195 From 89e90a74ec4823efb53baadec20197a6de08db2b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 13 May 2026 08:15:03 +0800 Subject: [PATCH 077/304] Sync llama.cpp API 20260513 Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 2 ++ llama_cpp/mtmd_cpp.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1efd645150..cc900c0648 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -2763,6 +2763,8 @@ def llama_state_seq_load_file( ) -> int: ... +# define LLAMA_STATE_SEQ_FLAGS_NONE 0 +LLAMA_STATE_SEQ_FLAGS_NONE = 0 # // for backwards-compat LLAMA_STATE_SEQ_FLAGS_SWA_ONLY = 1 diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 574d90e2bf..839c718ccd 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -723,6 +723,34 @@ def mtmd_log_set(log_callback: ggml_log_callback, user_data: c_void_p): # type: ... +# // EXPERIMENTAL API to get mmproj's capabilities without initializing the full context +# // This is only intended to be used by llama-server, breaking changes is expected +# struct mtmd_caps { +# bool inp_vision; +# bool inp_audio; +# }; +class mtmd_caps(Structure): + _fields_ = [ + ("inp_vision", c_bool), + ("inp_audio", c_bool), + ] + + if TYPE_CHECKING: + inp_vision: c_bool + inp_audio: c_bool + + +# MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname); +@ctypes_function_mtmd( + "mtmd_get_cap_from_file", [c_char_p], mtmd_caps) +def mtmd_get_cap_from_file(mmproj_fname: c_char_p) -> mtmd_caps: + """ + EXPERIMENTAL API to get mmproj's capabilities without initializing the full context. + This is only intended to be used by llama-server, breaking changes is expected + """ + ... + + # // test function, to be used in test-mtmd-c-api.c # MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); @ctypes_function_mtmd( From e67169dfd59a67ec500c38b42d0cfc41475f1051 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 14 May 2026 00:40:40 +0800 Subject: [PATCH 078/304] Implement `MiniCPMV46ChatHandler` for `MiniCPM-V-4.6` Signed-off-by: JamePeng --- README.md | 1 + llama_cpp/llama_chat_format.py | 204 +++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) diff --git a/README.md b/README.md index c9aba7d42d..cd03d217d7 100644 --- a/README.md +++ b/README.md @@ -835,6 +835,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [llama-3-vision-alpha](https://huggingface.co/abetlen/llama-3-vision-alpha-gguf) | `Llama3VisionAlphaChatHandler` | `llama-3-vision-alpha` | | [minicpm-v-2.6](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `MiniCPMv26ChatHandler` | `minicpm-v-2.6`, `minicpm-v-4.0` | | [minicpm-v-4.5](https://huggingface.co/openbmb/MiniCPM-V-4_5-gguf) | `MiniCPMv45ChatHandler` | `minicpm-v-4.5` | +| [minicpm-v-4.6](https://huggingface.co/openbmb/MiniCPM-V-4.6-gguf) | `MiniCPMv46ChatHandler` | `minicpm-v-4.6` | | [gemma3](https://huggingface.co/unsloth/gemma-3-27b-it-GGUF) | `Gemma3ChatHandler` | `gemma3` | | [gemma4](https://huggingface.co/unsloth/gemma-4-26B-A4B-it-GGUF) | `Gemma4ChatHandler` | `gemma4` | | [glm4.1v](https://huggingface.co/unsloth/GLM-4.1V-9B-Thinking-GGUF) | `GLM41VChatHandler` | `glm4.1v` | diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index a0d8d25db4..2ab627c89b 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -4280,6 +4280,210 @@ def __call__(self, **kwargs): return super().__call__(**kwargs) +class MiniCPMV46ChatHandler(MTMDChatHandler): + """ + Handler for MiniCPM-V-4.6 models. + + Features: + - Aligned with official tokenizer_config.json special tokens. + - Custom `<|image_pad|>` and `<|video_pad|>` multimodal tokens. + - Integrated MTMD-style URL and Base64 injection for visual content. + - Specialized `` and `` block generation. + - Autonomously folds previous reasoning paths using `last_query_index`. + - Toggles `` block generation via `enable_thinking` (Defaults to False). + """ + + # Core tokens + MINICPM_BOS_TOKEN = "<|im_start|>" + MINICPM_EOS_TOKEN = "<|im_end|>" + MINICPM_PAD_TOKEN = "<|endoftext|>" + + # Vision tokens + MINICPM_VISION_BOS_TOKEN = "<|vision_start|>" + MINICPM_VISION_EOS_TOKEN = "<|vision_end|>" + MINICPM_IMAGE_TOKEN = "<|image_pad|>" + MINICPM_VIDEO_TOKEN = "<|video_pad|>" + + CHAT_FORMAT = ( + "{%- if enable_thinking is not defined -%}\n" + " {%- set enable_thinking = false -%}\n" + "{%- endif -%}\n" + "{%- macro render_content(content, is_system_content=false) -%}\n" + " {%- if content is string -%}\n" + " {{- content -}}\n" + " {%- elif content is iterable and content is not mapping -%}\n" + " {%- set ns = namespace(parts=[]) -%}\n" + " {%- for item in content -%}\n" + " {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}\n" + " {%- if is_system_content -%}\n" + " {{- raise_exception('System message cannot contain images.') -}}\n" + " {%- endif -%}\n" + " {%- set url_val = '' -%}\n" + " {%- if item.type == 'image_url' -%}\n" + " {%- set url_val = item.image_url if item.image_url is string else item.image_url.url -%}\n" + " {%- endif -%}\n" + " {%- set ns.parts = ns.parts + ['<|image_pad|>' + url_val] -%}\n" + # " {%- elif 'video' in item or 'video_url' in item or item.type == 'video' -%}\n" + # " {%- if is_system_content -%}\n" + # " {{- raise_exception('System message cannot contain videos.') -}}\n" + # " {%- endif -%}\n" + # " {%- set url_val = '' -%}\n" + # " {%- if item.type == 'video_url' -%}\n" + # " {%- set url_val = item.video_url if item.video_url is string else item.video_url.url -%}\n" + # " {%- endif -%}\n" + # " {%- set ns.parts = ns.parts + ['<|video_pad|>' + url_val] -%}\n" + " {%- elif 'text' in item -%}\n" + " {%- set ns.parts = ns.parts + [item.text] -%}\n" + " {%- else -%}\n" + " {{- raise_exception('Unexpected item type in content.') -}}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {{- ns.parts | join('\\n') -}}\n" + " {%- elif content is none or content is undefined -%}\n" + " {{- '' -}}\n" + " {%- else -%}\n" + " {{- raise_exception('Unexpected content type.') -}}\n" + " {%- endif -%}\n" + "{%- endmacro -%}\n" + "{%- if not messages %}\n" + " {{- raise_exception('No messages provided.') }}\n" + "{%- endif %}\n" + "{%- if tools and tools is iterable and tools is not mapping %}\n" + " {{- '<|im_start|>system\\n' }}\n" + " {{- '# Tools\\n\\nYou have access to the following functions:\\n\\n' }}\n" + " {%- for tool in tools %}\n" + " {{- '\\n' }}\n" + " {{- tool | tojson }}\n" + " {%- endfor %}\n" + " {{- '\\n' }}\n" + " {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n' }}\n" + " {%- if messages[0].role == 'system' %}\n" + " {%- set content = render_content(messages[0].content, true)|trim %}\n" + " {%- if content %}\n" + " {{- '\\n\\n' + content }}\n" + " {%- endif %}\n" + " {%- endif %}\n" + " {{- '<|im_end|>\\n' }}\n" + "{%- else %}\n" + " {%- if messages[0].role == 'system' %}\n" + " {%- set content = render_content(messages[0].content, true)|trim %}\n" + " {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n" + " {%- endif %}\n" + "{%- endif %}\n" + "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n" + "{%- for message in messages[::-1] %}\n" + " {%- set index = (messages|length - 1) - loop.index0 %}\n" + " {%- if ns.multi_step_tool and message.role == 'user' %}\n" + " {%- set content = render_content(message.content)|trim %}\n" + " {%- if not(content.startswith('') and content.endswith('')) %}\n" + " {%- set ns.multi_step_tool = false %}\n" + " {%- set ns.last_query_index = index %}\n" + " {%- endif %}\n" + " {%- endif %}\n" + "{%- endfor %}\n" + "{%- if ns.multi_step_tool %}\n" + " {{- raise_exception('No user query found in messages.') }}\n" + "{%- endif %}\n" + "{%- for message in messages %}\n" + " {%- set content = render_content(message.content)|trim %}\n" + " {%- if message.role == 'system' %}\n" + " {%- if not loop.first %}\n" + " {{- raise_exception('System message must be at the beginning.') }}\n" + " {%- endif %}\n" + " {%- elif message.role == 'user' %}\n" + " {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n" + " {%- elif message.role == 'assistant' %}\n" + " {%- set reasoning_content = '' %}\n" + " {%- if message.reasoning_content is string %}\n" + " {%- set reasoning_content = message.reasoning_content %}\n" + " {%- else %}\n" + " {%- if '' in content %}\n" + " {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n" + " {%- set content = content.split('')[-1].lstrip('\\n') %}\n" + " {%- endif %}\n" + " {%- endif %}\n" + " {%- set reasoning_content = reasoning_content|trim %}\n" + " {%- if loop.index0 > ns.last_query_index %}\n" + " {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n\\n' + content }}\n" + " {%- else %}\n" + " {{- '<|im_start|>' + message.role + '\\n' + content }}\n" + " {%- endif %}\n" + " {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n" + " {%- for tool_call in message.tool_calls %}\n" + " {%- if tool_call.function is defined %}\n" + " {%- set tool_call = tool_call.function %}\n" + " {%- endif %}\n" + " {%- if loop.first %}\n" + " {%- if content|trim %}\n" + " {{- '\\n\\n\\n\\n' }}\n" + " {%- else %}\n" + " {{- '\\n\\n' }}\n" + " {%- endif %}\n" + " {%- else %}\n" + " {{- '\\n\\n\\n' }}\n" + " {%- endif %}\n" + " {%- if tool_call.arguments is defined %}\n" + " {%- for args_name, args_value in tool_call.arguments|items %}\n" + " {{- '\\n' }}\n" + " {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n" + " {{- args_value }}\n" + " {{- '\\n\\n' }}\n" + " {%- endfor %}\n" + " {%- endif %}\n" + " {{- '\\n' }}\n" + " {%- endfor %}\n" + " {%- endif %}\n" + " {{- '<|im_end|>\\n' }}\n" + " {%- elif message.role == 'tool' %}\n" + " {%- if loop.previtem and loop.previtem.role != 'tool' %}\n" + " {{- '<|im_start|>user' }}\n" + " {%- endif %}\n" + " {{- '\\n\\n' }}\n" + " {{- content }}\n" + " {{- '\\n' }}\n" + " {%- if not loop.last and loop.nextitem.role != 'tool' %}\n" + " {{- '<|im_end|>\\n' }}\n" + " {%- elif loop.last %}\n" + " {{- '<|im_end|>\\n' }}\n" + " {%- endif %}\n" + " {%- else %}\n" + " {{- raise_exception('Unexpected message role.') }}\n" + " {%- endif %}\n" + "{%- endfor %}\n" + "{%- if add_generation_prompt %}\n" + " {{- '<|im_start|>assistant\\n' }}\n" + " {%- if enable_thinking is defined and enable_thinking is false %}\n" + " {{- '\\n\\n\\n\\n' }}\n" + " {%- else %}\n" + " {{- '\\n' }}\n" + " {%- endif %}\n" + "{%- endif %}\n" + ) + + def __init__(self, enable_thinking: bool = True, **kwargs): + """ + Initializes the MiniCPM-V-4.6 Handler. + + Args: + enable_thinking (bool): Controls whether to open a `` block for reasoning. + Defaults to False as per the standard template logic. + """ + self.enable_thinking = enable_thinking + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Inject the thinking variable into the Jinja environment + self.extra_template_arguments["enable_thinking"] = self.enable_thinking + + # MiniCPM uses standard <|im_end|> ChatML stop formatting + kwargs['stop'] = [self.MINICPM_PAD_TOKEN, self.MINICPM_EOS_TOKEN] + + if self.verbose: + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") + + return super().__call__(**kwargs) + + class Gemma3ChatHandler(MTMDChatHandler): GEMMA3_BOI_TOKEN = "" From 99543936f58145314cde6c7cfbf88ab119a664b5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 14 May 2026 07:17:58 +0800 Subject: [PATCH 079/304] fix(MTMDChatHandler): correct audio_url content type check and improve variable handling - Changed condition from `content == "audio_url"` to `content_type == "audio_url"` for proper type-based dispatching. - Extracted `audio_url` variable for better readability. - Converted `else` to `elif content_type == "input_audio"` to make the control flow explicit and safer. Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 2ab627c89b..1c41beb40f 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2997,11 +2997,12 @@ def _get_media_items(self, messages: List[llama_types.ChatCompletionRequestMessa raise ValueError(f"{self.log_prefix}: This mmproj model instance does not support audio inputs.") # Case A: Handle custom/forward-compatible audio_url format - if content == "audio_url": - url = content["audio_url"] if isinstance(content["audio_url"], str) else content["audio_url"]["url"] + if content_type == "audio_url": + audio_url = content["audio_url"] + url = audio_url if isinstance(audio_url, str) else audio_url["url"] media_items.append({"url": url, "type": "audio"}) # Case B: Handle OpenAI standard input_audio format - else: + elif content_type == "input_audio": input_audio = content.get("input_audio", {}) if isinstance(input_audio, dict) and "data" in input_audio: # It might just be raw base64 data, we can format it as a data URI to reuse load_audio logic From 0295d0c62fd5173e0704d33a8ced4d2c8e590d9c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 15 May 2026 04:21:06 +0800 Subject: [PATCH 080/304] Update Submodule vendor/llama.cpp a9883db..834a243 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a9883db8ee..834a243664 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a9883db8ee021cf16783016a60996d41820b5195 +Subproject commit 834a243664114487f99520370a7a7b00fc7a486f From 1fb6a6665726e6abce52959bc38f162e6a0cb2dc Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 15 May 2026 07:22:11 +0800 Subject: [PATCH 081/304] feat(logger): refactor and enhance ggml logging configuration system - Introduce a `LoggerConfig` dataclass to provide fine-grained control over native ggml/llama.cpp runtime logging. - Align verbosity levels (0 to 5) with upstream `llama.cpp` conventions (`common/log.h`). - Implement a dynamic, configurable substring filtering system, replacing the hardcoded "CUDA Graph" patch with `DEFAULT_LOG_FILTERS`. - Add comprehensive public APIs for log management: `configure_logging`, `set_verbosity`, `set_quiet`, `set_silent`, `set_log_filters`, and `add_log_filters`. - Maintain backwards compatibility for the existing `set_verbose(bool)` function. - Improve the `ggml_log_callback` to correctly handle `GGML_LOG_LEVEL_CONT` by inheriting the verbosity of the preceding log message. - Route `GGML_LOG_LEVEL_NONE` to `stdout` and all other diagnostic logs to `stderr` by default. Signed-off-by: JamePeng --- llama_cpp/_logger.py | 406 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 383 insertions(+), 23 deletions(-) diff --git a/llama_cpp/_logger.py b/llama_cpp/_logger.py index 015cec9faa..7669e2a722 100644 --- a/llama_cpp/_logger.py +++ b/llama_cpp/_logger.py @@ -1,6 +1,9 @@ import sys import ctypes import logging +from dataclasses import dataclass, field +from typing import Iterable, Optional, TextIO, Union + import llama_cpp._ggml as _ggml import llama_cpp.llama_cpp as llama_cpp_lib @@ -12,42 +15,399 @@ # GGML_LOG_LEVEL_DEBUG = 4, # GGML_LOG_LEVEL_CONT = 5, // continue previous log # }; -GGML_LOG_LEVEL_TO_LOGGING_LEVEL = { - 0: logging.CRITICAL, - 1: logging.INFO, - 2: logging.WARNING, - 3: logging.ERROR, - 4: logging.DEBUG, - 5: logging.DEBUG, +GGML_LOG_LEVEL_NONE = 0 +GGML_LOG_LEVEL_INFO = 1 +GGML_LOG_LEVEL_WARN = 2 +GGML_LOG_LEVEL_ERROR = 3 +GGML_LOG_LEVEL_DEBUG = 4 +GGML_LOG_LEVEL_CONT = 5 + +# common/log.h model: +# +# LOG_LEVEL_OUTPUT = 0 +# LOG_LEVEL_ERROR = 1 +# LOG_LEVEL_WARN = 2 +# LOG_LEVEL_INFO = 3 +# LOG_LEVEL_TRACE = 4 +# LOG_LEVEL_DEBUG = 5 +# +# Rule: +# +# event_verbosity <= verbosity_threshold => print +# +# Larger threshold means more verbose output. +# +LOG_LEVEL_OUTPUT = 0 +LOG_LEVEL_ERROR = 1 +LOG_LEVEL_WARN = 2 +LOG_LEVEL_INFO = 3 +LOG_LEVEL_TRACE = 4 +LOG_LEVEL_DEBUG = 5 + +LOG_DEFAULT_LLAMA = LOG_LEVEL_INFO +LOG_DEFAULT_DEBUG = LOG_LEVEL_DEBUG + +# Match the updated common_log_default_callback behavior: +# INFO -> TRACE +# CONT -> TRACE +# +# This is slightly more conservative for verbosity=3: +# if the backend emits INFO through ggml_log_callback, Python will hide it unless +# verbosity >= 4. This mirrors the current upstream default callback behavior. +GGML_LEVEL_TO_VERBOSITY = { + GGML_LOG_LEVEL_NONE: LOG_LEVEL_OUTPUT, + GGML_LOG_LEVEL_ERROR: LOG_LEVEL_ERROR, + GGML_LOG_LEVEL_WARN: LOG_LEVEL_WARN, + GGML_LOG_LEVEL_INFO: LOG_LEVEL_TRACE, + GGML_LOG_LEVEL_DEBUG: LOG_LEVEL_DEBUG, + GGML_LOG_LEVEL_CONT: LOG_LEVEL_TRACE, # fallback only; CONT inherits previous +} + +GGML_LEVEL_TO_PYTHON_LEVEL = { + GGML_LOG_LEVEL_NONE: logging.INFO, + GGML_LOG_LEVEL_ERROR: logging.ERROR, + GGML_LOG_LEVEL_WARN: logging.WARNING, + GGML_LOG_LEVEL_INFO: logging.INFO, + GGML_LOG_LEVEL_DEBUG: logging.DEBUG, + GGML_LOG_LEVEL_CONT: logging.INFO, # fallback only; CONT inherits previous } + +# Default substring filters. +# +# These are intentionally simple substring filters instead of hard-coded +# special branches. Users can replace or clear them with set_log_filters(). +DEFAULT_LOG_FILTERS = [ + "CUDA Graph", + "CUDA graph" +] + + +VerbosityLike = Union[bool, int, str, None] + logger = logging.getLogger("llama-cpp-python") -_last_log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[0] -# typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); +@dataclass +class LoggerConfig: + # 0=output, 1=error, 2=warn, 3=info, 4=trace, 5=debug + verbosity: int = LOG_DEFAULT_LLAMA + + show_output: bool = True + + stdout: TextIO = sys.stdout + stderr: TextIO = sys.stderr + + # If any substring is contained in a log message, the message is dropped. + log_filters: list[str] = field(default_factory=lambda: list(DEFAULT_LOG_FILTERS)) + log_filters_case_sensitive: bool = True + + +_config = LoggerConfig() +_last_verbosity = LOG_LEVEL_INFO + + +def _normalize_verbosity( + value: VerbosityLike, + *, + default: int = LOG_DEFAULT_LLAMA, +) -> int: + """ + Convert user input to llama.cpp-style verbosity 0..5. + + Compatibility: + verbose=False -> ERROR (1) + verbose=True -> DEBUG (5) + + Numeric levels: + 0 = output + 1 = error + 2 = warn + 3 = info + 4 = trace + 5 = debug + """ + if value is None: + return default + + if isinstance(value, bool): + return LOG_LEVEL_DEBUG if value else LOG_LEVEL_ERROR + + if isinstance(value, int): + return max(LOG_LEVEL_OUTPUT, min(LOG_LEVEL_DEBUG, value)) + + if isinstance(value, str): + key = value.strip().lower() + aliases = { + "0": LOG_LEVEL_OUTPUT, + "output": LOG_LEVEL_OUTPUT, + "none": LOG_LEVEL_OUTPUT, + + "1": LOG_LEVEL_ERROR, + "error": LOG_LEVEL_ERROR, + "err": LOG_LEVEL_ERROR, + "silent": LOG_LEVEL_ERROR, + + "2": LOG_LEVEL_WARN, + "warn": LOG_LEVEL_WARN, + "warning": LOG_LEVEL_WARN, + "quiet": LOG_LEVEL_WARN, + + "3": LOG_LEVEL_INFO, + "info": LOG_LEVEL_INFO, + "default": LOG_DEFAULT_LLAMA, + "normal": LOG_DEFAULT_LLAMA, + + "4": LOG_LEVEL_TRACE, + "trace": LOG_LEVEL_TRACE, + "trc": LOG_LEVEL_TRACE, + + "5": LOG_LEVEL_DEBUG, + "debug": LOG_LEVEL_DEBUG, + "verbose": LOG_LEVEL_DEBUG, + } + + if key in aliases: + return aliases[key] + + try: + parsed = int(key) + except ValueError as exc: + raise ValueError( + "_logger._normalize_verbosity: " + "verbosity must be one of 0..5, bool, None, or " + "'silent'/'quiet'/'info'/'trace'/'debug'" + ) from exc + + return max(LOG_LEVEL_OUTPUT, min(LOG_LEVEL_DEBUG, parsed)) + + raise TypeError(f"_logger._normalize_verbosity: unsupported verbosity type: {type(value)!r}") + + +def _verbosity_to_python_level(verbosity: int) -> int: + if verbosity >= LOG_LEVEL_DEBUG: + return logging.DEBUG + if verbosity >= LOG_LEVEL_INFO: + return logging.INFO + if verbosity >= LOG_LEVEL_WARN: + return logging.WARNING + return logging.ERROR + + +def _get_verbosity(level: int) -> int: + """ + Map ggml log level to Python-side verbosity. + + GGML_LOG_LEVEL_INFO maps to LOG_LEVEL_INFO so that verbosity=3 remains + useful as the default info level. + """ + if level == GGML_LOG_LEVEL_NONE: + return LOG_LEVEL_OUTPUT + if level == GGML_LOG_LEVEL_ERROR: + return LOG_LEVEL_ERROR + if level == GGML_LOG_LEVEL_WARN: + return LOG_LEVEL_WARN + if level == GGML_LOG_LEVEL_INFO: + return LOG_LEVEL_INFO + if level == GGML_LOG_LEVEL_DEBUG: + return LOG_LEVEL_DEBUG + if level == GGML_LOG_LEVEL_CONT: + return LOG_LEVEL_INFO + return LOG_LEVEL_DEBUG + + +def _decode_log_text(text: bytes) -> str: + return text.decode("utf-8", errors="replace") + + +def _matches_log_filter(msg: str) -> bool: + filters = _config.log_filters + if not filters: + return False + + if _config.log_filters_case_sensitive: + return any(item and item in msg for item in filters) + + msg_lower = msg.lower() + return any(item and item.lower() in msg_lower for item in filters) + + +def _should_drop(level: int, verbosity: int, msg: str) -> bool: + if verbosity > _config.verbosity: + return True + + if level == GGML_LOG_LEVEL_NONE and not _config.show_output: + return True + + if _matches_log_filter(msg): + return True + + return False + + @_ggml.ggml_log_callback def ggml_log_callback( level: int, text: bytes, user_data: ctypes.c_void_p, ): - # Note(JamePeng): A temporary patch is used to filter out garbage debug information - # output from the underlying C++ `CUDA Graph id %zu reused`. - # The logger is planned to be refactored to meet control requirements. - if text: - if b"CUDA Graph" in text or b"CUDA graph" in text: - return - # TODO: Correctly implement continue previous log - global _last_log_level - log_level = GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level] if level != 5 else _last_log_level - if logger.level <= GGML_LOG_LEVEL_TO_LOGGING_LEVEL[level]: - print(text.decode("utf-8"), end="", flush=True, file=sys.stderr) - _last_log_level = log_level + global _last_verbosity + + msg = _decode_log_text(text) + + if level == GGML_LOG_LEVEL_CONT: + verbosity = _last_verbosity + else: + verbosity = _get_verbosity(level) + _last_verbosity = verbosity + if _should_drop(level, verbosity, msg): + return -llama_cpp_lib.llama_log_set(ggml_log_callback, ctypes.c_void_p(0)) + out = _config.stdout if level == GGML_LOG_LEVEL_NONE else _config.stderr + print(msg, end="", flush=True, file=out) + + +# Keep a global reference to avoid ctypes callback being garbage-collected. +_ggml_log_callback_ref = ggml_log_callback + +llama_cpp_lib.llama_log_set(_ggml_log_callback_ref, ctypes.c_void_p(0)) + + +def configure_logging( + *, + verbosity: VerbosityLike = None, + verbose: Optional[bool] = None, + quiet: Optional[bool] = None, + silent: Optional[bool] = None, + show_output: Optional[bool] = None, + log_filters: Optional[Iterable[str]] = None, + append_log_filters: Optional[Iterable[str]] = None, + log_filters_case_sensitive: Optional[bool] = None, +): + """ + Configure native ggml/llama.cpp runtime logging. + + Priority: + silent > quiet > verbosity > verbose > current config + + Compatibility: + verbose=False -> ERROR + verbose=True -> DEBUG + + Numeric levels: + 0 = output + 1 = error + 2 = warn + 3 = info + 4 = trace + 5 = debug + """ + if silent is True: + v = LOG_LEVEL_ERROR + elif quiet is True: + v = LOG_LEVEL_WARN + elif verbosity is not None: + v = _normalize_verbosity(verbosity) + elif verbose is not None: + v = _normalize_verbosity(verbose) + else: + v = _config.verbosity + + _config.verbosity = v + logger.setLevel(_verbosity_to_python_level(v)) + + if show_output is not None: + _config.show_output = show_output + + if log_filters is not None: + _config.log_filters = [s for s in log_filters if s] + + if append_log_filters is not None: + _config.log_filters.extend(s for s in append_log_filters if s) + + if log_filters_case_sensitive is not None: + _config.log_filters_case_sensitive = log_filters_case_sensitive def set_verbose(verbose: bool): - logger.setLevel(logging.DEBUG if verbose else logging.ERROR) + """ + Backward-compatible bool API. + + False -> ERROR + True -> DEBUG + """ + configure_logging(verbose=verbose) + + +def set_verbosity(verbosity: VerbosityLike): + configure_logging(verbosity=verbosity) + + +def get_verbosity() -> int: + return _config.verbosity + + +def set_quiet(quiet: bool = True): + configure_logging(quiet=quiet) + + +def set_silent(silent: bool = True): + configure_logging(silent=silent) + + +def set_log_filters( + filters: Iterable[str], + *, + case_sensitive: bool = True, +): + """ + Replace all substring log filters. + + Example: + set_log_filters(["CUDA Graph id", "clip_model_loader: tensor"]) + """ + configure_logging( + log_filters=filters, + log_filters_case_sensitive=case_sensitive, + ) + + +def get_log_filters() -> list[str]: + return list(_config.log_filters) + + +def add_log_filters(filters: Iterable[str]): + """ + Append substring log filters. + """ + configure_logging(append_log_filters=filters) + + +def clear_log_filters(): + """ + Clear all substring log filters, including default filters. + """ + _config.log_filters.clear() + + +def reset_log_filters(): + """ + Restore default substring log filters. + """ + _config.log_filters = list(DEFAULT_LOG_FILTERS) + + +def get_log_filters_case_sensitive() -> bool: + return _config.log_filters_case_sensitive + + +def reset_logging(): + """ + Reset logging to default llama.cpp-style INFO verbosity and default filters. + """ + _config.verbosity = LOG_DEFAULT_LLAMA + _config.show_output = True + _config.log_filters = list(DEFAULT_LOG_FILTERS) + _config.log_filters_case_sensitive = True + logger.setLevel(_verbosity_to_python_level(_config.verbosity)) From f64320de4fedcad1f28cb46526803ff68784c546 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 15 May 2026 07:30:34 +0800 Subject: [PATCH 082/304] feat(core): integrate fine-grained logging API into Llama class This commit exposes the newly refactored `_logger` configuration system directly through the `Llama` class, providing users with robust, programmatic control over native `llama.cpp` backend logs. Key changes: - Expand `Llama.__init__` with `verbosity`, `log_filters`, and `log_filters_case_sensitive` parameters. - Add instance methods for runtime log management (`set_verbosity`, `get_verbosity`, `set_log_filters`, `add_log_filters`, `clear_log_filters`, etc.). - Add comprehensive docstrings explaining the 0-5 verbosity scale and explicitly noting the process-global nature of the native backend logger. Advantages over the legacy implementation: - Granular Control: Replaces the restrictive binary `verbose=True/False` flag (which only toggled between ERROR and DEBUG) with a granular 6-tier scale (output, error, warn, info, trace, debug). - Dynamic Filtering: Empowers users to actively suppress specific noisy C++ logs using custom substring filters, removing the need for hardcoded internal patches. - Better Discoverability: Attaches logging controls directly to the `Llama` object, making log management much more accessible and intuitive without requiring users to import internal logger modules. Signed-off-by: JamePeng --- llama_cpp/llama.py | 112 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index e50e3b9a3b..19ede6bcfd 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -58,7 +58,16 @@ from ._ggml import ( ggml_backend_cpu_buffer_type, ) -from ._logger import set_verbose +from ._logger import ( + configure_logging, + get_verbosity, + set_verbosity, + get_log_filters, + set_log_filters, + add_log_filters, + clear_log_filters, + reset_log_filters, +) from ._utils import suppress_stdout_stderr @@ -150,7 +159,11 @@ def __init__( type_v: Optional[int] = None, # Misc spm_infill: bool = False, + # Log verbose: bool = True, + verbosity: Optional[Union[int, str, bool]] = None, + log_filters: Optional[Sequence[str]] = None, + log_filters_case_sensitive: bool = True, # Extra Params **kwargs, # type: ignore ): @@ -235,11 +248,31 @@ def __init__( chat_handler: Optional chat handler to use when calling create_chat_completion. draft_model: Optional draft model to use for speculative decoding. tokenizer: Optional tokenizer to override the default tokenizer from llama.cpp. - verbose: Print verbose output to stderr. type_k: KV cache data type for K (default: f16) type_v: KV cache data type for V (default: f16) spm_infill: Use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this. - + verbose: Backward-compatible boolean switch for native llama.cpp / ggml runtime logs. + False keeps only error-level native logs; True enables debug-level native logs. + If `verbosity` is provided, `verbosity` takes precedence over `verbose`. + verbosity: Fine-grained llama.cpp-style native runtime log verbosity. + Accepts 0-5, bool, or string aliases. + Numeric levels: + 0 = output only + 1 = error + 2 = warning + 3 = info + 4 = trace + 5 = debug + Use `verbosity=3` for llama.cpp-style default info logs. + `verbose=False` remains equivalent to error-only logging, while + `verbose=True` remains equivalent to debug logging. + log_filters: Optional substring filters for native runtime logs. + If any provided substring appears in a decoded backend log message, + that message is suppressed. By default, the logger may include built-in + filters for noisy low-level logs such as CUDA Graph reuse spam messages. + Pass an empty list to disable all substring filtering for this instance. + log_filters_case_sensitive: Whether `log_filters` should match case-sensitively. + Defaults to True for predictable low-level backend log filtering. Raises: ValueError: If the model path does not exist. @@ -247,9 +280,15 @@ def __init__( A Llama instance. """ self.verbose = verbose + self.verbosity = verbosity self._stack = contextlib.ExitStack() - set_verbose(verbose) + configure_logging( + verbose=verbose, + verbosity=verbosity, + log_filters=log_filters, + log_filters_case_sensitive=log_filters_case_sensitive, + ) if not Llama.__backend_initialized: with suppress_stdout_stderr(disable=verbose): @@ -795,6 +834,71 @@ def eval_logits(self) -> Deque[List[float]]: maxlen=self._n_ctx if self._logits_all else 1, ) + # Logger API + + def set_verbosity(self, verbosity: Union[int, str, bool, None]) -> None: + """Set native llama.cpp / ggml runtime log verbosity for this process. + + Levels: + 0 = output only + 1 = error + 2 = warning + 3 = info + 4 = trace + 5 = debug + + Note: + Native backend logging is process-global because llama.cpp / ggml use + a global log callback. Changing this affects all Llama instances in + the current Python process. + """ + set_verbosity(verbosity) + self.verbosity = get_verbosity() + self.verbose = self.verbosity >= 5 + + + def get_verbosity(self) -> int: + """Return the current native runtime log verbosity.""" + return get_verbosity() + + + def set_log_filters( + self, + filters: Sequence[str], + *, + case_sensitive: bool = True, + ) -> None: + """Replace substring filters for native runtime logs. + + Any backend log message containing one of these substrings will be + suppressed. Pass an empty list to disable all substring filtering. + + Note: + Native backend logging is process-global, so this affects all Llama + instances in the current Python process. + """ + set_log_filters(filters, case_sensitive=case_sensitive) + + + def add_log_filters(self, filters: Sequence[str]) -> None: + """Append substring filters for native runtime logs.""" + add_log_filters(filters) + + + def get_log_filters(self) -> List[str]: + """Return the current substring filters for native runtime logs.""" + return get_log_filters() + + + def clear_log_filters(self) -> None: + """Clear all substring filters, including default filters.""" + clear_log_filters() + + + def reset_log_filters(self) -> None: + """Restore default substring filters for native runtime logs.""" + reset_log_filters() + # LoRA / Adapter Management API def load_lora(self, name: str, path: str): From d89aa5a7d7ea3629c8767b332d692e9f3b9a9e5f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 15 May 2026 07:43:27 +0800 Subject: [PATCH 083/304] docs(wiki): document runtime verbosity and log filters for Llama Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 106 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 0354d86150..a061861ece 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -4,7 +4,7 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-05-06 +last_updated: 2026-05-15 version_target: "latest" --- ``` @@ -55,6 +55,15 @@ Initialize the model and context. Note that model loading will immediately alloc | `checkpoint_interval` | `int` | `4096` | Token interval for saving periodic Hybrid/Recurrent checkpoints during long prompt evaluation. | | `checkpoint_on_device` | `bool` | `False` | Store Hybrid/Recurrent checkpoint tensor payloads in `llama_context`-owned device buffers via `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE`. Reduces device-to-host copy overhead, but only one active checkpoint per `seq_id` is safe. | +### Runtime Logging Parameters + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `verbose` | `bool` | `True` | Backward-compatible boolean native logging switch. `False` keeps only error-level llama.cpp / ggml logs; `True` enables debug-level native logs. If `verbosity` is provided, `verbosity` takes precedence over `verbose`. | +| `verbosity` | `Optional[Union[int, str, bool]]` | `None` | Fine-grained llama.cpp-style native runtime log verbosity. Numeric levels: `0=output`, `1=error`, `2=warning`, `3=info`, `4=trace`, `5=debug`. Use `verbosity=3` for llama.cpp-style default info logs. String aliases such as `"silent"`, `"quiet"`, `"info"`, `"trace"`, and `"debug"` are also accepted. | +| `log_filters` | `Optional[Sequence[str]]` | `None` | Optional substring filters for native runtime logs. If any provided substring appears in a decoded backend log message, that message is suppressed. The default logger may include built-in filters for noisy low-level logs such as `CUDA Graph id %d reuse` messages. Pass an empty list `[]` to disable default substring filtering. | +| `log_filters_case_sensitive` | `bool` | `True` | Whether `log_filters` should match case-sensitively. Defaults to `True` for predictable low-level backend log filtering. | + *(Note: There are numerous additional RoPE/YaRN scaling parameters available for specialized context extension. Refer to the source code for the full list).* --- @@ -112,6 +121,42 @@ model.eval(tokens=[1, 453, 234, 987], active_loras=[{"name": "coding_adapter", " Immediately halts an active generation loop safely. * **Usage**: Typically called from a separate monitoring thread (like a timer). When triggered, the running stream will exit and the final chunk will contain `"finish_reason": "abort"`. +### Runtime Logging Control + +The `Llama` class exposes lightweight runtime helpers for adjusting native llama.cpp / ggml logging after initialization. + +> **Note:** Native backend logging is process-global because llama.cpp / ggml use a global log callback. Changing verbosity or log filters affects all `Llama` instances in the current Python process. + +* `set_verbosity(verbosity: Union[int, str, bool, None])`: Set native runtime log verbosity. +* `get_verbosity() -> int`: Return the current native runtime log verbosity. +* `set_log_filters(filters: Sequence[str], case_sensitive: bool = True)`: Replace substring filters for native runtime logs. +* `add_log_filters(filters: Sequence[str])`: Append substring filters. +* `get_log_filters() -> List[str]`: Return the current substring filters. +* `clear_log_filters()`: Clear all substring filters, including default filters. +* `reset_log_filters()`: Restore default substring filters. + +```python +from llama_cpp import Llama + +llm = Llama( + model_path="models/qwen3.gguf", + verbosity=3, # llama.cpp-style info logs +) + +# Temporarily enable debug-level native logs. +llm.set_verbosity(5) + +# Suppress noisy backend messages by substring. +llm.add_log_filters([ + "CUDA Graph", + "CUDA graph", + "clip_model_loader: tensor", +]) + +# Return to quiet error-only logging. +llm.set_verbosity(1) +``` + ### Dynamic LoRA Management The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dynamically per-generation or per-eval. * `load_lora(name: str, path: str)`: Loads an adapter into VRAM (does not apply it yet). @@ -185,7 +230,7 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn llm.create_completion("Once upon a time", active_loras=[{"name": "story", "scale": 0.9}]) # Use sql adapter - llm.create_completion("SELECT *", active_loras=[{"name": "sql_expert", "scale": 0.8}])v + llm.create_completion("SELECT *", active_loras=[{"name": "sql_expert", "scale": 0.8}]) ``` 5. **Hybrid & Recurrent Architectures**: @@ -321,6 +366,63 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn run_controlled_generation("Explain quantum mechanics in a way that relates to bugs in code.", timeout_seconds=8) ``` +8. **Runtime Logging & Backend Noise Filtering**: + + `Llama` supports fine-grained native llama.cpp / ggml logging through `verbosity`. This is more precise than the legacy `verbose` boolean flag. + + ```python + from llama_cpp import Llama + + # Legacy behavior: + # verbose=False -> error-only logs + llm_quiet = Llama( + model_path="models/qwen3.gguf", + verbose=False, + ) + + # Recommended precise logging: + # 0 = output, 1 = error, 2 = warning, 3 = info, 4 = trace, 5 = debug + llm = Llama( + model_path="models/qwen3.gguf", + verbosity=3, # llama.cpp-style default info logs + ) + ``` + + For low-level debugging, use `verbosity=5`. By default, the logger may suppress known noisy backend messages such as CUDA Graph reuse logs. Pass `log_filters=[]` to disable all substring filtering. + + ```python + llm = Llama( + model_path="models/qwen3.gguf", + verbosity=5, + log_filters=[], # show all debug logs, including normally filtered ones + ) + ``` + + To suppress additional noisy messages, pass substring filters: + + ```python + llm = Llama( + model_path="models/qwen3.gguf", + verbosity=5, + log_filters=[ + "CUDA Graph id", + "clip_model_loader: tensor", + "ggml_cuda_graph_update_required", + ], + ) + ``` + + You can also adjust logging at runtime: + + ```python + llm.set_verbosity(5) + llm.add_log_filters(["llama_perf_context_print"]) + + # Later, return to warning-level logs. + llm.set_verbosity(2) + ``` + + **Important:** native backend logging is process-global. Runtime changes affect all `Llama` instances in the same Python process. --- From c14d769a4408e516bb03bd13f68e838ab6edbe4a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 07:30:10 +0800 Subject: [PATCH 084/304] Update Submodule vendor/llama.cpp 834a243..49d1701 --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 834a243664..49d1701bd2 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 834a243664114487f99520370a7a7b00fc7a486f +Subproject commit 49d1701bd24e4cedf6dfec9e50e185111203946b From 4d3e320b321db7c505721c92c7d3d26641e95623 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 09:01:49 +0800 Subject: [PATCH 085/304] Implement `Qwen3ASRChatHandler` for `Qwen3-ASR` models. - Integrate MTMD multimodal logic to extract and inject `audio_url` and base64 `input_audio` data directly into the `<|audio_start|><|audio_pad|>[DATA]<|audio_end|>` sequence. - Define a default multilingual transcription system prompt and configure model-specific stop tokens. Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 78 ++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 1c41beb40f..0365d8f871 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -5473,6 +5473,84 @@ def __call__(self, **kwargs): # Use parent implementation return super().__call__(**kwargs) +class Qwen3ASRChatHandler(MTMDChatHandler): + """ + Handler for Qwen 3 ASR (Automatic Speech Recognition) models. + + Features: + - Highly specialized for Speech-to-Text tasks. + - Aggregates all system text into a single cohesive system block. + - Drops user text entirely, extracting ONLY audio data into a unified user turn. + - Wraps audio with <|audio_start|><|audio_pad|>[DATA]<|audio_end|>. + - Integrated MTMD-style URL and Base64 injection for input_audio and audio_url. + """ + + DEFAULT_SYSTEM_MESSAGE = """ + You are an advanced multilingual Speech-to-Text model. Accurately transcribe the audio into text in its original spoken language. + You should ignore background noise, filler words, and stutters where possible, and format the final output with correct grammar and capitalization. + """ + + QWEN3_ASR_BOS_TOKEN = "<|im_start|>" + QWEN3_ASR_PAD_TOKEN = "<|endoftext|>" + QWEN3_ASR_EOS_TOKEN = "<|im_end|>" + + + QWEN3_ASR_AUDIO_BOS_TOKEN = "<|audio_start|>" + QWEN3_ASR_AUDIO_PAD_TOKEN = "<|audio_pad|>" + QWEN3_ASR_AUDIO_EOS_TOKEN = "<|audio_end|>" + + CHAT_FORMAT = ( + "{%- set ns = namespace(system_text='') -%}\n" + "{%- for m in messages -%}\n" + " {%- if m.role == 'system' -%}\n" + " {%- if m.content is string -%}\n" + " {%- set ns.system_text = ns.system_text + m.content -%}\n" + " {%- else -%}\n" + " {%- for c in m.content -%}\n" + " {%- if c.type == 'text' and (c.text is defined) -%}\n" + " {%- set ns.system_text = ns.system_text + c.text -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "\n" + "{%- set ns2 = namespace(audio_tokens='') -%}\n" + "{%- for m in messages -%}\n" + " {%- if m.content is not string -%}\n" + " {%- for c in m.content -%}\n" + " {%- if c.type == 'audio' or ('audio' in c) or ('audio_url' in c) or c.type == 'input_audio' -%}\n" + " {#- MTMD Audio Injection -#}\n" + " {%- set audio_val = '' -%}\n" + " {%- if c.type == 'audio_url' or 'audio_url' in c -%}\n" + " {%- set audio_val = c.audio_url if c.audio_url is string else c.audio_url.url -%}\n" + " {%- elif c.type == 'input_audio' or 'input_audio' in c -%}\n" + " {%- set audio_val = c.input_audio if c.input_audio is string else ('data:audio/' + c.input_audio.format + ';base64,' + c.input_audio.data) -%}\n" + " {%- endif -%}\n" + " {%- set ns2.audio_tokens = ns2.audio_tokens + '<|audio_start|><|audio_pad|>' + audio_val + '<|audio_end|>' -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "\n" + "{{- '<|im_start|>system\\n' + (ns.system_text if ns.system_text is string else '') + '<|im_end|>\\n' -}}\n" + "{{- '<|im_start|>user\\n' + ns2.audio_tokens + '<|im_end|>\\n' -}}\n" + "{%- if add_generation_prompt -%}\n" + " {{- '<|im_start|>assistant\\n' -}}\n" + "{%- endif -%}\n" + ) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Qwen3 models universally use `<|endoftext|>` and `<|im_end|>` as the stop token + kwargs['stop'] = [self.QWEN3_ASR_AUDIO_PAD_TOKEN, self.QWEN3_ASR_AUDIO_EOS_TOKEN] + + if self.verbose: + print(f"{self.log_prefix} - Start processing Qwen3-ASR (Audio Only)") + + return super().__call__(**kwargs) class Qwen3VLChatHandler(MTMDChatHandler): CHAT_FORMAT = ( From ad67e0e979620e3dc18c91460bd7a96a3dfc1934 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 09:05:21 +0800 Subject: [PATCH 086/304] docs(README.md): add Qwen3-ASR documentation and usage example - Update the supported multi-modal models table to include `qwen3-asr` and the `Qwen3ASRChatHandler`. - Add a new dedicated section for Speech-to-Text inference with a complete, collapsible Python script. - Provide a `build_media_payload` helper function to demonstrate proper Base64 encoding of local `.wav` and `.mp3` files into OpenAI-compatible `input_audio` schemas. - Include a critical warning advising users to use BF16 quantization for the multimodal projector (`mmproj`) to prevent audio degradation. - Clarify usage mechanics, specifically that all instructions must be placed in the `system` role due to the ASR template's text-dropping behavior. Signed-off-by: JamePeng --- README.md | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/README.md b/README.md index cd03d217d7..26aa11a55f 100644 --- a/README.md +++ b/README.md @@ -845,6 +845,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [lfm2.5-vl](https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B-GGUF) | `LFM25VLChatHandler` | `lfm2.5-vl` | | [paddleocr-vl-1.5](https://huggingface.co/JamePeng2023/PaddleOCR-VL-1.5-GGUF) | `PaddleOCRChatHandler` | `paddleocr` | | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | +| [qwen3-asr](https://huggingface.co/JamePeng2023/Qwen3-ASR-1.7B-GGUF) | `Qwen3ASRChatHandler` | `qwen3-asr` | | [qwen3-vl](https://huggingface.co/unsloth/Qwen3-VL-8B-Thinking-GGUF) | `Qwen3VLChatHandler` | `qwen3-vl` | | [qwen3.5](https://huggingface.co/unsloth/Qwen3.5-27B-GGUF) | `Qwen35ChatHandler` | `qwen3.5` | | [qwen3.6](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF) | `Qwen35ChatHandler` | `qwen3.6` | @@ -1072,6 +1073,111 @@ print(res["choices"][0]["message"]["content"])
+## Speech Recognition With Qwen3-ASR (Speech-to-Text) + +The `Qwen3ASRChatHandler` is specifically designed for the Qwen3 Automatic Speech Recognition (ASR) models. Unlike standard multimodal models, this handler aggregates system prompts for instructions and automatically extracts audio data from the user's message, ignoring any user text. + +> **āš ļø Important Note on Quantization:** > For Qwen3-ASR models, it is highly recommended to use the **BF16** version of the multimodal projector (`mmproj`). Other quantizations are known to cause severe audio degradation. + +**Example Code**:
+ +```python +from llama_cpp import Llama +from llama_cpp.llama_chat_format import Qwen3ASRChatHandler +import base64 +import os + +# 1. Define paths to the model and the BF16 multimodal projector +MODEL_PATH = r"./Qwen3-ASR-1.7B-BF16.gguf" +MMPROJ_PATH = r"./mmproj-Qwen3-ASR-1.7b-BF16.gguf" + +# 2. Initialize the Llama model with the dedicated ASR handler +llm = Llama( + model_path=MODEL_PATH, + chat_handler=Qwen3ASRChatHandler( + clip_model_path=MMPROJ_PATH, + verbose=False, + ), + n_gpu_layers=-1, + n_ctx=10240, + verbose=False, + verbosity=0 +) + +# 3. Helper function to encode audio files into OpenAI-compatible payloads +_MEDIA_MIME_TYPES = { + '.wav': ('audio', 'wav'), + '.mp3': ('audio', 'mp3'), +} + +def build_media_payload(file_path: str) -> dict: + """Reads a local audio file and converts it into the LLM input structure.""" + if not os.path.isfile(file_path): + raise FileNotFoundError(f"Media file not found: {file_path}") + + extension = os.path.splitext(file_path)[1].lower() + media_category, mime_or_format = _MEDIA_MIME_TYPES.get(extension, ('unknown', 'application/octet-stream')) + + if media_category == 'unknown': + print(f"Warning: Unknown extension '{extension}'.") + + # Read and Base64 encode the file + with open(file_path, "rb") as f: + encoded_data = base64.b64encode(f.read()).decode("utf-8") + + if media_category == 'audio': + return { + "type": "input_audio", + "input_audio": { + "data": encoded_data, + "format": mime_or_format + } + } + else: + return {"type": "text", "text": f"[Attached unsupported file: {file_path}]"} + + +# ======================== +# Main Inference Section +# ======================== + +media_paths = ["./audio/test.wav"] +user_content = [build_media_payload(path) for path in media_paths] + +# 4. Generate the transcription +response = llm.create_chat_completion( + messages=[ + { + "role": "system", + "content": ( + "You are an advanced multilingual Speech-to-Text model. " + "Accurately transcribe the audio into text in its original spoken language. " + "You should ignore background noise, filler words, and stutters where possible, " + "and format the final output with correct grammar and capitalization." + ) + }, + { + "role": "user", + "content": user_content + } + ], + temperature=1.0, + top_p=0.95, + top_k=64, + max_tokens=10240, +) + +print(f"Transcribe: {response['choices'][0]['message']['content']}") + +``` + +#### How it works: + +* **`input_audio` Schema:** The script reads the local `.wav` or `.mp3` file, encodes it in Base64, and wraps it in an OpenAI-compatible `"type": "input_audio"` dictionary. +* **System Prompt:** Because the Qwen3-ASR template strips out user text, all instructions (like translation requests or formatting rules) **must** be placed in the `"system"` role. + +
+ ## Comprehensive Omni MultiModal Example: Gemma-4 (Vision + Audio + Text) Below is a complete, production-ready example demonstrating how to dynamically route and process both image and audio files. It includes a universal media processor that automatically converts local files into the correct payload structure (Data URIs for images, and `input_audio` for audio files). From 43b85f38ed9b55ab1cff4646e41796f93b4b1129 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 09:12:34 +0800 Subject: [PATCH 087/304] docs(README.md): Update the jump link for Qwen3-ASR in the top directory. Signed-off-by: JamePeng --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 26aa11a55f..caec7e32e0 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ This package provides: - [Multi-modal Models Support](https://github.com/JamePeng/llama-cpp-python#multi-modal-models) - Support Models Lists - [Loading a Local Image With Qwen3VL(Thinking/Instruct)](https://github.com/JamePeng/llama-cpp-python#loading-a-local-image-with-qwen3vlthinkinginstruct) + - [Speech Recognition With Qwen3-ASR (Speech-to-Text)](https://github.com/JamePeng/llama-cpp-python#speech-recognition-with-qwen3-asr-speech-to-text) - [Comprehensive Omni MultiModal Example: Gemma-4 (Vision + Audio + Text)](https://github.com/JamePeng/llama-cpp-python#comprehensive-omni-multimodal-example-gemma-4-vision--audio--text) - [Embeddings & Reranking (GGUF)](https://github.com/JamePeng/llama-cpp-python#embeddings--reranking-gguf) - [1. Text Embeddings (Vector Search)](https://github.com/JamePeng/llama-cpp-python#1-text-embeddings-vector-search) From 4fb074682bf18c0c8097e9b9e4800940eb49bc07 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 09:57:46 +0800 Subject: [PATCH 088/304] Update SCHEMA.md --- docs/wiki/SCHEMA.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/wiki/SCHEMA.md b/docs/wiki/SCHEMA.md index b96ec964c7..23954a156e 100644 --- a/docs/wiki/SCHEMA.md +++ b/docs/wiki/SCHEMA.md @@ -4,7 +4,7 @@ - **Author**: JamePeng - **Maintainer**: LLM-assisted documentation workflow - **Project**: [llama-cpp-python](https://github.com/JamePeng/llama-cpp-python) wiki -- **Last Modified**: 2026-05-02 +- **Last Modified**: 2026-05-16 - **Version Target**: latest source code - **Schema Version**: 0.3 @@ -24,6 +24,7 @@ - `llama_cpp.py` - `mtmd_cpp.py` - `_ggml.py` + - `_logger.py` - Never invent parameters or behavior. Always read the current source code before writing/updating a page. - Prefer documenting public and user-facing APIs first. Internal implementation details may be documented only when they help users understand behavior, extension points, debugging, or advanced usage. - All examples must be complete, runnable with the latest API, and include necessary imports. From 1064cf17361c394f38f6f3d89fc68367d45d8720 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 10:00:53 +0800 Subject: [PATCH 089/304] docs(Llama.md): update `verbose=False` vs. `verbosity=0` note Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index a061861ece..1f7cce206b 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -4,7 +4,7 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-05-15 +last_updated: 2026-05-16 version_target: "latest" --- ``` @@ -424,6 +424,10 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn **Important:** native backend logging is process-global. Runtime changes affect all `Llama` instances in the same Python process. + **verbose=False** vs. **verbosity=0**: These have distinct behaviors. + - `verbose=False` silences Python wrapper prints but not backend diagnostics; like `if self.verbose: print()` + - `verbosity=0` silences all backend non-error output. + --- ## Deprecated / Changed APIs From 7eab8d3ad3f178d9cae6bebdd6013213176415d8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 10:04:43 +0800 Subject: [PATCH 090/304] docs(Logger.md): Upload Logger documentation Signed-off-by: JamePeng --- docs/wiki/modules/Logger.md | 216 ++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 docs/wiki/modules/Logger.md diff --git a/docs/wiki/modules/Logger.md b/docs/wiki/modules/Logger.md new file mode 100644 index 0000000000..f24f7f43a8 --- /dev/null +++ b/docs/wiki/modules/Logger.md @@ -0,0 +1,216 @@ +--- +title: Logger +class_name: Logger (module) +module_name: llama_cpp._logger +source_file: llama_cpp/_logger.py +last_updated: 2026-05-16 +version_target: latest +--- + +## Overview + +The `Logger` module provides configuration for runtime logging in `llama-cpp-python`, wrapping the native `ggml`/`llama.cpp` logging infrastructure. It controls verbosity levels, output streams, substring filtering, and callback integration, allowing fine-grained control over diagnostic and informational output from the underlying bindings. + +## Role in the Library + +- **Wraps low-level logging**: It intercepts and transforms log events from the C/C++ backend (`ggml_log_callback`). +- **Connects to Python logging**: Maps `ggml` verbosity levels (0ā€“5) to `logging` levels (ERROR, WARNING, INFO, DEBUG), and routes output to `stdout`/`stderr` based on severity. +- **Provides filtering**: Substring-based message filtering to suppress specific log categories (e.g., CUDA Graph output). +- **Extends the API surface**: Offers both explicit configuration functions and convenient shorthand setters (`set_verbose`, `set_quiet`), while preserving full control through `configure_logging`. + +## Core Methods + +### `configure_logging(*, verbosity=None, verbose=None, quiet=None, silent=None, show_output=None, log_filters=None, append_log_filters=None, log_filters_case_sensitive=None)` + +The primary configuration function. Combines multiple parameters into a unified verbosity level. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `verbosity` | int \| bool \| None | None | Numeric level (0ā€“5). `False` maps to `ERROR` (1), `True` to `DEBUG` (5). | +| `verbose` | bool | None | Shorthand: `True` ā†’ `DEBUG`, `False` ā†’ `ERROR`. | +| `quiet` | bool | None | Shorthand: `True` ā†’ `WARN` (2). | +| `silent` | bool | None | Shorthand: `True` ā†’ `ERROR` (1). | +| `show_output` | bool | None | Whether `GGML_LOG_LEVEL_NONE` (output) should be shown. | +| `log_filters` | Iterable[str] | None | List of substring patterns to filter out. | +| `append_log_filters` | Iterable[str] | None | Append additional filter patterns. | +| `log_filters_case_sensitive` | bool | None | Whether filters are case-sensitive. | + +### `set_verbose(verbose: bool)` + +Shorthand setter. `verbose=True` sets `verbosity=DEBUG`, `verbose=False` sets `verbosity=ERROR`. + +### `set_verbosity(verbosity: VerbosityLike)` + +Sets verbosity to any value accepted by `configure_logging`. + +### `get_verbosity() -> int` + +Returns current configured verbosity level (0ā€“5). + +### `set_quiet(quiet: bool = True)` + +Sets `verbosity=WARN` (`2`). + +### `set_silent(silent: bool = True)` + +Sets `verbosity=ERROR` (`1`). + +### `set_log_filters(filters: Iterable[str], *, case_sensitive: bool = True)` + +Replaces all substring log filters. + +### `get_log_filters() -> list[str]` + +Returns current filter list. + +### `add_log_filters(filters: Iterable[str])` + +Appends filters to the current list. + +### `clear_log_filters()` + +Removes all user-defined filters. + +### `reset_log_filters()` + +Restores the default filter list: `["CUDA Graph", "CUDA graph"]`. + +### `reset_logging()` + +Resets to default: `verbosity=INFO` (`3`), `show_output=True`, default filters. + +## Important Attributes / State + +| Attribute | Type | Source | Description | +|-----------|------|--------|-------------| +| `_config` | LoggerConfig | Internal | Holds the current configuration: verbosity, output streams, filters. | +| `_last_verbosity` | int | Internal | Tracks the last verbosity level set by `ggml_log_callback`. | + +## Best Practices & Common Patterns + +### 1. Default Behavior +Use `reset_logging()` to start with `INFO` verbosity, which shows warnings and errors but hides internal debug output. + +```python +from llama_cpp import Llama +from llama_cpp import reset_logging + +reset_logging() # Default verbosity=3 (INFO), show warnings and errors +llm = Llama(model_path="models/qwen3.gguf") +llm("Explain quantum physics.") +``` + +### 2. Precise Logging via `verbosity` +Replace the legacy `verbose` boolean with the precise `verbosity` parameter. `verbose=False` maps to `ERROR` (1), `verbose=True` to `DEBUG` (5). + +```python +from llama_cpp import Llama + +# Legacy (coarse control): +llm_quiet = Llama(model_path="models/qwen3.gguf", verbose=False) +llm_quiet("What is a neural network?") + +# Modern (fine-grained control): +llm = Llama(model_path="models/qwen3.gguf", verbosity=3) +llm("What is a neural network?") +``` + +### 3. Low-Level Debugging +For deep backend debugging, set `verbosity=5` (DEBUG) and optionally disable substring filters to see all diagnostic output. + +```python +from llama_cpp import Llama + +# Debug-level logs, showing all backend diagnostics +llm = Llama(model_path="models/qwen3.gguf", verbosity=5) + +# If you want to see normally filtered CUDA Graph messages: +llm = Llama( + model_path="models/qwen3.gguf", + verbosity=5, + log_filters=[], # Disable all substring filters +) +``` + +### 4. Substring-Based Backend Noise Filtering +Suppress known noisy backend messages by passing substring filters. This prevents "CUDA Graph" and model loading chatter from flooding the console. + +```python +from llama_cpp import Llama + +llm = Llama( + model_path="models/qwen3.gguf", + verbosity=3, # INFO level + log_filters=[ + "CUDA Graph id", + "clip_model_loader: tensor", + "ggml_cuda_graph_update_required", + "llama_perf_context_print", + ], +) +llm("What is a transformer?") +``` + +### 5. Runtime Logging Adjustments +Since logging is process-global, you can adjust verbosity or filters at runtime ā€” changes apply to all `Llama` instances in the same process. + +```python +from llama_cpp import Llama + +llm = Llama(model_path="models/qwen3.gguf", verbosity=2) # QUIET: only show warnings and errors +llm("Quick answer: What is machine learning?") + +# Temporarily increase verbosity for diagnostics +llm.set_verbosity(5) +llm("Show me the full debug log for this prompt") +llm.set_verbosity(2) # Return to QUIET + +# Add a specific filter without resetting everything +llm.add_log_filters(["llama_perf_context_print"]) +llm("Final answer: What is machine learning?") +``` + +### 6. Complete Diagnostic Session +For a full diagnostic session, combine precise verbosity, custom filters, and runtime control: + +```python +from llama_cpp import Llama + +# 1. Start with info-level verbosity +llm = Llama(model_path="models/qwen3.gguf", verbosity=3) + +# 2. Suppress backend noise +llm.set_log_filters([ + "CUDA Graph", + "CUDA graph", + "clip_model_loader: tensor", + "ggml_cuda_graph_update_required", +]) + +# 3. Run inference +llm("Explain the llama.cpp inference pipeline") + +# 4. Temporarily increase verbosity for a specific call +llm.set_verbosity(5) +llm("Show debug output for cache hit details") +llm.set_verbosity(2) # Return to normal + +# 5. Remove filters after session +llm.clear_log_filters() +``` + +## Key Considerations + +- **Process-global**: Logging configuration affects all `Llama` instances in the same process. Use `add_log_filters` or `set_log_filters` carefully when multiple instances run concurrently. +- **Flushed immediately**: Every log call flushes to `stdout`/`stderr`, so output appears immediately. +- **Shorthand vs. precise**: Prefer `verbosity`/`set_verbosity` over `verbose`/`set_verbose`/`set_quiet`/`set_silent` for precision, though the shorthands remain for backward compatibility. +- **verbose=False** vs. **verbosity=0**: These have distinct behaviors ā€” `verbose=False` silences Python wrapper prints but not backend diagnostics; `verbosity=0` silences all backend non-error output. + +## Deprecated / Changed APIs + +None documented. + +## Related Links + +* [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] +* [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] From 43a96e2e098ca2845d3e4c6acd92643b80579240 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 10:08:29 +0800 Subject: [PATCH 091/304] docs(index): Append Logger.md info and link Signed-off-by: JamePeng --- docs/wiki/index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/wiki/index.md b/docs/wiki/index.md index 02f2dd5b9a..143d6e629b 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -30,6 +30,7 @@ These pages document major source modules and related classes. | [modules/LlamaEmbedding\|Llama Embedding] | Embedding-related APIs and usage patterns. | | [modules/LlamaGrammar\|Llama Grammar] | Provides grammar utilities for constrained generation. | | [modules/LlamaSpeculative\|Llama Speculative Decoding] | Draft model interfaces and prompt-based speculative decoding helpers. | +| [modules/Logger\|Logger] | provides configuration for runtime logging in `llama-cpp-python`, wrapping the native `ggml`/`llama.cpp` logging infrastructure. It controls verbosity levels, output streams, substring filtering, and callback integration, allowing fine-grained control over diagnostic and informational output from the underlying bindings. | --- @@ -53,6 +54,7 @@ If you are new to this wiki, read the pages in this order: 3. [[modules/LlamaEmbedding|Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] 4. [[modules/LlamaGrammar|Llama Grammar](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaGrammar.md)] 5. [[modules/LlamaSpeculative|Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] +6. [[modules/Logger\|Logger](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/Logger.md)] If you are contributing documentation, start with: @@ -72,6 +74,7 @@ Currently available pages: - `modules/LlamaEmbedding.md` - `modules/LlamaGrammar.md` - `modules/LlamaSpeculative.md` +- `modules/Logger.md` - `SCHEMA.md` - `contributing-to-wiki.md` From 9187910e35e6f4d063f33364a10812727a05e58d Mon Sep 17 00:00:00 2001 From: Alcoft Date: Sat, 16 May 2026 06:41:17 +0200 Subject: [PATCH 092/304] fix --- llama_cpp/llama.py | 1 - llama_cpp/llama_chat_format.py | 33 +++++++++++---------------------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 6dab44602d..7666b822a8 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -618,7 +618,6 @@ def __init__( self.chat_handler = llama_chat_format.GenericMTMDChatHandler( gguf_metadata = self.metadata, clip_model_path = clip_model_path, - model_arch = None, verbose = self.verbose, **chat_handler_kwargs ) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 40491968a9..0be38a19d3 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3839,47 +3839,36 @@ def from_pretrained( ) class GenericMTMDChatHandler(MTMDChatHandler): + KNOWN_MEDIA_TAGS = [ + "<|image_pad|>", + "<|audio_pad|>", + "<|video_pad|>", + "<|image|>", + "<|audio|>", + "<|video|>", + "[IMG]" + ] + def __init__( self, gguf_metadata: Dict[str, Any], clip_model_path: str, - model_arch: Optional[str] = None, verbose: bool = True, **kwargs ) -> None: self.model_metadata = gguf_metadata - self.chat_format = self.model_metadata.get("tokenizer.chat_template", None) - self.arch = self.model_metadata.get("general.architecture", None) if model_arch is None else model_arch if verbose: print(f"Got chat template from model:\n```jinja\n{self.chat_format}\n```", flush = True) - - if self.arch is None: - if verbose: - print("Unknown model architecture. Will use general/most-common tags.") - - self.arch = "unknown" if self.chat_format is None: raise ValueError("Failed to get model chat template automatically.") super().__init__(clip_model_path = clip_model_path, verbose = verbose, **kwargs) - - if self.arch in ["unknown", "qwen3vl", "qwen35moe", "qwen35"]: - self._chat_format_parser_tags += ["<|image_pad|>", "<|audio_pad|>", "<|video_pad|>"] - elif self.arch in ["gemma4"]: - self._chat_format_parser_tags += ["<|image|>", "<|audio|>", "<|video|>"] - elif self.arch in ["mistral3", "mistral4", "deepseek2"]: - self._chat_format_parser_tags += ["[IMG]"] - elif verbose: - print("Warning: Could not determine chat format parser tags.", flush = True) def __call__(self, **kwargs): - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) + self._chat_format_parser_tags = [tag for tag in self.KNOWN_MEDIA_TAGS if tag in self.chat_format] if self.verbose: print(f"{self.log_prefix} - Start processing") From 2dad6dc407c6b56af281a29bbe2f7a3e15fb712f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 12:46:38 +0800 Subject: [PATCH 093/304] build(cmake): refactor install target lists for new GGML backend layout - Categorize build targets into logical groups (`LLAMA_CPP_TARGETS`, `GGML_CORE_TARGETS`, `GGML_CPU_VARIANT_TARGETS`, and `GGML_BACKEND_TARGETS`) to improve maintainability and keep the Python package installation in sync with the updated upstream GGML backend layout. - Add missing targets such as `llama-common` and the separated `ggml-cpu-*` CPU variant backends. - Ensure all grouped targets are passed through `llama_cpp_python_install_target`. Signed-off-by: JamePeng --- CMakeLists.txt | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 04d3ec1fff..c42bbe95f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,14 +117,38 @@ if (LLAMA_BUILD) set_target_properties(llama PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) endif() - # Define list of GGML targets to install - set(GGML_TARGETS + # Define list of LLAMA_CPP/GGML targets to install + set(LLAMA_CPP_TARGETS llama + llama-common + ) + set(GGML_CORE_TARGETS ggml ggml-base ggml-blas - ggml-cann ggml-cpu + ggml-rpc + ) + + set(GGML_CPU_VARIANT_TARGETS + ggml-cpu-x64 + ggml-cpu-sse42 + ggml-cpu-sandybridge + ggml-cpu-ivybridge + ggml-cpu-piledriver + ggml-cpu-haswell + ggml-cpu-skylakex + ggml-cpu-cannonlake + ggml-cpu-cascadelake + ggml-cpu-cooperlake + ggml-cpu-icelake + ggml-cpu-alderlake + ggml-cpu-sapphirerapids + ggml-cpu-zen4 + ) + + set(GGML_BACKEND_TARGETS + ggml-cann ggml-cuda ggml-hexagon ggml-hip @@ -132,7 +156,6 @@ if (LLAMA_BUILD) ggml-musa ggml-opencl ggml-openvino - ggml-rpc ggml-sycl ggml-virtgpu ggml-vulkan @@ -141,8 +164,12 @@ if (LLAMA_BUILD) ggml-zendnn ) - # Loop through targets to avoid repetitive function calls - foreach(TARGET_NAME ${GGML_TARGETS}) + foreach(TARGET_NAME + ${LLAMA_CPP_TARGETS} + ${GGML_CORE_TARGETS} + ${GGML_CPU_VARIANT_TARGETS} + ${GGML_BACKEND_TARGETS} + ) llama_cpp_python_install_target(${TARGET_NAME}) endforeach() From 24b1dc859cba8b2dce7fb2463c78faacc1955997 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 13:07:25 +0800 Subject: [PATCH 094/304] build(cmake): sync llama build options and disable server UI - Update llama build option descriptions to match the current upstream naming style. - Explicitly disable `LLAMA_BUILD_SERVER` to avoid building the server target for Python package wheels. - Explicitly disable `LLAMA_BUILD_UI` and `LLAMA_USE_PREBUILT_UI` because the embedded server Web UI is not needed for wheel builds. - Keep examples, tests, and curl support disabled for minimal wheel artifacts. Signed-off-by: JamePeng --- CMakeLists.txt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c42bbe95f0..ee72ae9582 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,16 +72,23 @@ if (LLAMA_BUILD) set(CMAKE_SKIP_RPATH FALSE) # Enable building of the common library - set(LLAMA_BUILD_COMMON ON CACHE BOOL "llama.cpp: build common utils library" FORCE) + set(LLAMA_BUILD_COMMON ON CACHE BOOL "llama: build common utils library" FORCE) # Enable build and link OpenSSL - set(LLAMA_OPENSSL ON CACHE BOOL "llama.cpp: build and link OpenSSL" FORCE) + set(LLAMA_OPENSSL ON CACHE BOOL "llama: use openssl to support HTTPS" FORCE) # Disable building of examples - set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "llama.cpp: build examples" FORCE) + set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "llama: build examples" FORCE) # Disable building of tests - set(LLAMA_BUILD_TESTS OFF CACHE BOOL "llama.cpp: build tests" FORCE) + set(LLAMA_BUILD_TESTS OFF CACHE BOOL "llama: build tests" FORCE) + + # Disable building of server + set(LLAMA_BUILD_SERVER OFF CACHE BOOL "llama: build server example" FORCE) + + # Disable build the embedded Web UI for server + set(LLAMA_BUILD_UI OFF CACHE BOOL "llama: build the embedded Web UI for server" FORCE) + set(LLAMA_USE_PREBUILT_UI OFF CACHE BOOL "llama: use prebuilt UI from HF Bucket when available (requires LLAMA_BUILD_UI=ON)" FORCE) # Disable building curl support set(LLAMA_CURL OFF CACHE BOOL "llama.cpp: use libcurl to download model from an URL" FORCE) From 4c4e3d007a649e65e8f38a6ce387807299310bdf Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 13:55:17 +0800 Subject: [PATCH 095/304] build(cmake): clean up dev files and import libs from Windows wheels - Remove `ARCHIVE DESTINATION` for Windows targets to avoid installing `.lib` files. - Add a cleanup function to strip `cmake`, `pkgconfig`, and import libraries from the python wheel runtime directories. - Ensures Windows builds only package the required runtime DLLs. Signed-off-by: JamePeng --- CMakeLists.txt | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ee72ae9582..8e5d583d90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,13 +11,12 @@ set(CMAKE_INSTALL_LIBDIR llama_cpp/lib CACHE PATH "" FORCE) set(CMAKE_INSTALL_INCLUDEDIR llama_cpp/include CACHE PATH "" FORCE) -# Helper function to install targets to Python package directories +# Install a built target into the Python package runtime directory. function(llama_cpp_python_install_target target) if(NOT TARGET ${target}) return() endif() - # Define install destinations to avoid code duplication set(INSTALL_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib" "${SKBUILD_PLATLIB_DIR}/llama_cpp/lib" @@ -33,6 +32,9 @@ function(llama_cpp_python_install_target target) RESOURCE DESTINATION ${DIR} ) + # Copy runtime DLL dependencies of this target when available. + # This does not replace explicit installation of dynamic backend + # targets such as ggml-cpu-*; those are installed as targets below. # Automatically handle Windows DLL installation for each target if (WIN32) install( @@ -57,6 +59,40 @@ function(llama_cpp_python_install_target target) endif() endfunction() + +# Remove development-only artifacts from Python wheel runtime directories. +# +# Upstream install rules may place CMake package files, pkg-config files, and +# Windows import libraries under llama_cpp/lib because CMAKE_INSTALL_LIBDIR is +# redirected there for wheel builds. They are not needed at runtime. +function(llama_cpp_python_cleanup_dev_files) + if(NOT WIN32) + return() + endif() + + set(INSTALL_DIRS + "${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib" + "${SKBUILD_PLATLIB_DIR}/llama_cpp/lib" + ) + + foreach(DIR ${INSTALL_DIRS}) + install(CODE " + if(EXISTS \"${DIR}\") + file(GLOB LLAMA_CPP_IMPORT_LIBS \"${DIR}/*.lib\") + if(LLAMA_CPP_IMPORT_LIBS) + file(REMOVE \${LLAMA_CPP_IMPORT_LIBS}) + endif() + + file(REMOVE_RECURSE + \"${DIR}/cmake\" + \"${DIR}/pkgconfig\" + ) + endif() + ") + endforeach() +endfunction() + + if (LLAMA_BUILD) set(BUILD_SHARED_LIBS "On") @@ -204,4 +240,8 @@ if (LLAMA_BUILD) llama_cpp_python_install_target(mtmd) endif() + + # Run after all runtime targets are installed, including mtmd. + llama_cpp_python_cleanup_dev_files() + endif() From 6af3cd7df808cb9725b2da0273c23098111c25ea Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 15:18:32 +0800 Subject: [PATCH 096/304] fix(_ggml): correct `ggml_backend_unload` function name --- llama_cpp/_ggml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 8f4cb1187f..c4ae7c94bf 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -1295,8 +1295,8 @@ def ggml_backend_load(path: ctypes.c_char_p) -> ggml_backend_reg_t: # // Unload a backend if loaded dynamically and unregister it # GGML_API void ggml_backend_unload(ggml_backend_reg_t reg); -@ggml_function("ggml_backend_load_all", [ctypes.c_void_p], None) -def ggml_backend_load_all(reg: ggml_backend_reg_t): +@ggml_function("ggml_backend_unload", [ctypes.c_void_p], None) +def ggml_backend_unload(reg: ggml_backend_reg_t): """ Unload a backend if loaded dynamically and unregister it """ From 038a953079126fd4a81e574c9c06680e36b0a10e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 16:13:53 +0800 Subject: [PATCH 097/304] feat(core): support loading GGML_BACKEND_DL dynamic backend libraries from wheel lib - Import `ggml_backend_load_all_from_path` and `ggml_backend_reg_count` from `_ggml`. - Load dynamic ggml backend libraries from the packaged `llama_cpp/lib` directory after `llama_backend_init()`. - Support wheels built with `GGML_BACKEND_DL`, where CPU variants and accelerator backends such as `ggml-cpu-*` and `ggml-cuda` are shipped as separate runtime libraries. - Print the registered backend count in verbose mode to help diagnose backend discovery issues. Signed-off-by: JamePeng --- llama_cpp/llama.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 19ede6bcfd..8b1070be4f 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -57,6 +57,8 @@ ) from ._ggml import ( ggml_backend_cpu_buffer_type, + ggml_backend_load_all_from_path, + ggml_backend_reg_count ) from ._logger import ( configure_logging, @@ -290,9 +292,40 @@ def __init__( log_filters_case_sensitive=log_filters_case_sensitive, ) + # llama.cpp / ggml backend initialization is process-global. + # Run it once before loading any model. if not Llama.__backend_initialized: with suppress_stdout_stderr(disable=verbose): llama_cpp_lib.llama_backend_init() + + # Wheels built with `GGML_BACKEND_DL` ship ggml backends as separate + # dynamic libraries under llama_cpp/lib, for example: + # + # ggml-cpu-x64.dll + # ggml-cpu-haswell.dll + # ggml-cpu-alderlake.dll + # ggml-cuda.dll + # + # With the dynamic backend layout, llama_backend_init() initializes + # the global backend system but does not necessarily register every + # packaged backend. Loading the package lib directory ensures ggml can + # discover CPU variants and optional accelerator backends before model + # loading. + lib_dir = Path(llama_cpp_lib.__file__).resolve().parent / "lib" + + if not lib_dir.exists(): + raise FileNotFoundError(f"Llama.__init__: llama_cpp lib directory not found: {lib_dir}") + + # Load all dynamic ggml backend plugins from the packaged lib directory. + ggml_backend_load_all_from_path( + ctypes.c_char_p(str(lib_dir).encode("utf-8")) + ) + + # Print the number of backend registrations to confirm whether the DLL is loaded. + if self.verbose: + count = ggml_backend_reg_count() + print(f"Llama.__init__: Loaded ggml backend registry count: {count}", file=sys.stderr) + Llama.__backend_initialized = True if isinstance(numa, bool): From a8f928c9b134b61d5f6370d1e02de29efdd95227 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 17:14:15 +0800 Subject: [PATCH 098/304] Bump version to 0.3.39-preview Signed-off-by: JamePeng --- llama_cpp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 438bf08b58..b32fbfd36e 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.38" +__version__ = "0.3.39-preview" From 628373c1af97935a8c00e5273c5e9dd90dcd6b4c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 16 May 2026 17:29:56 +0800 Subject: [PATCH 099/304] ci(cu131+windows): build CU131 wheels with GGML dynamic backends for windows - Replace the old CPU/AVX release tag matrix with a single CU131 backend wheel layout. - Enable `GGML_BACKEND_DL` and `GGML_CPU_ALL_VARIANTS` so Windows wheels ship runtime-loadable GGML backend DLLs and CPU variant backends. - Use the Windows LLVM toolchain and disable non-wheel targets such as examples, tests, tools, server, embedded UI, and curl. - Remove the `.basic` style local version suffix and publish wheels as `+cu130`. - Update CUDA architectures to CUDA 13.1 and simplify CMake argument handling. Signed-off-by: JamePeng --- .github/workflows/build-wheels-cu130-win.yml | 134 ------------- .github/workflows/build-wheels-cu131-win.yml | 191 +++++++++++++++++++ 2 files changed, 191 insertions(+), 134 deletions(-) delete mode 100644 .github/workflows/build-wheels-cu130-win.yml create mode 100644 .github/workflows/build-wheels-cu131-win.yml diff --git a/.github/workflows/build-wheels-cu130-win.yml b/.github/workflows/build-wheels-cu130-win.yml deleted file mode 100644 index d6187d7bf4..0000000000 --- a/.github/workflows/build-wheels-cu130-win.yml +++ /dev/null @@ -1,134 +0,0 @@ -name: Build Wheels (CU130) for Windows - -on: - workflow_dispatch: - -permissions: - contents: write - -jobs: - build_wheels: - name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ['windows-2022'] - pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] - cuda: ["13.0.2"] - releasetag: ["Basic"] - cudaarch: ["75-real;80-real;86-real;87-real;89-real;90-real;100-real;120-real"] - defaults: - run: - shell: pwsh - env: - CUDAVER: ${{ matrix.cuda }} - AVXVER: ${{ matrix.releasetag }} - CUDAARCHVER: ${{ matrix.cudaarch }} - # https://cmake.org/cmake/help/latest/prop_tgt/CUDA_ARCHITECTURES.html - # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/#gpu-feature-list - # e.g. "all" "89" "90" "100" "120" - MAX_JOBS: 8 - - steps: - - name: Add MSBuild to PATH - if: runner.os == 'Windows' - uses: microsoft/setup-msbuild@v3 - with: - msbuild-architecture: x64 - - - uses: actions/checkout@v6 - with: - submodules: "recursive" - - # from kingbri1/flash-attention build-wheels.yml - - name: Install CUDA ${{ matrix.cuda }} - uses: Jimver/cuda-toolkit@v0.2.35 - id: cuda-toolkit - with: - cuda: "${{ matrix.cuda }}" - use-github-cache: false - - # from astral-sh/setup-uv - - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v7 - with: - python-version: ${{ matrix.pyver }} - activate-environment: true - enable-cache: true - - - name: Install Dependencies - run: | - git config --system core.longpaths true - uv pip install --upgrade build setuptools wheel packaging - - - name: Build Wheel - run: | - $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','') - $env:CUDA_HOME = $env:CUDA_PATH - $env:CUDA_TOOLKIT_ROOT_DIR = $env:CUDA_PATH - $env:VERBOSE = '1' - $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=' + $env:CUDAARCHVER + ' -DCMAKE_BUILD_PARALLEL_LEVEL=' + $env:MAX_JOBS - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=on -DCUDA_SEPARABLE_COMPILATION=on $env:CMAKE_ARGS" - $env:CMAKE_ARGS = "-DENABLE_CCACHE=on -DLLAMA_CURL=off $env:CMAKE_ARGS" - - if ($env:AVXVER -eq 'AVX') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off' - } - if ($env:AVXVER -eq 'AVX2') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off' - } - if ($env:AVXVER -eq 'AVXVNNI') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX_VNNI=on' - } - # if ($env:AVXVER -eq 'AVX512') { - # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on' - # } - # Basic options for compiling without AVX instructions - if ($env:AVXVER -eq 'Basic') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX_VNNI=off -DGGML_AVX512=off -DGGML_AVX512_VBMI=off -DGGML_AVX512_VNNI=off -DGGML_AVX512_BF16=off -DGGML_FMA=off -DGGML_F16C=off' - } - python -m build --wheel - - # Check if wheel was built - if (!(Test-Path '.\dist\*.whl')) { - Write-Error "No wheel built in dist/ directory" - exit 1 - } - - $wheelFile = Get-Item '.\dist\*.whl' | Select-Object -First 1 - - # Split file name: name-ver-py-abi-plat.whl - $parts = $wheelFile.Name.Split('-') - $distName = $parts[0] - $version = $parts[1] - $pyTag = $parts[2] - $abiTag = $parts[3] - $platTag = $parts[4] - - $newVersion = "$version+cu$cudaVersion.$($env:AVXVER.ToLower())" - - $newName = "$distName-$newVersion-$pyTag-$abiTag-$platTag" - - # Rename wheel file - Rename-Item -Path $wheelFile.FullName -NewName $newName - Write-Output "Renamed wheel to: $newName" - - # write the build tag to the output - Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV - Write-Output "TAG_VERSION=$version" >> $env:GITHUB_ENV - - - name: Get Current Date - id: get-date - run: | - $currentDate = Get-Date -UFormat "%Y%m%d" - Write-Output "BUILD_DATE=$currentDate" >> $env:GITHUB_ENV - - - name: Create Release - if: always() && env.TAG_VERSION != '' - uses: softprops/action-gh-release@v3 - with: - files: dist/* - # Set tag_name to -cu--win- - tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-${{ env.AVXVER }}-win-${{ env.BUILD_DATE }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-wheels-cu131-win.yml b/.github/workflows/build-wheels-cu131-win.yml new file mode 100644 index 0000000000..14bea65d19 --- /dev/null +++ b/.github/workflows/build-wheels-cu131-win.yml @@ -0,0 +1,191 @@ +name: Build Wheels (CU131) for Windows + +on: + workflow_dispatch: + +permissions: + contents: write + +jobs: + build_wheels: + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu131 + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: ["windows-2022"] + pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] + cuda: ["13.1.1"] + cudaarch: ["75-real;80-real;86-real;87-real;89-real;90-real;100-real;120-real"] + + defaults: + run: + shell: pwsh + + env: + CUDAVER: ${{ matrix.cuda }} + CUDAARCHVER: ${{ matrix.cudaarch }} + MAX_JOBS: 12 + + steps: + - name: Add MSBuild to PATH + uses: microsoft/setup-msbuild@v3 + with: + msbuild-architecture: x64 + + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Install CUDA ${{ matrix.cuda }} + uses: Jimver/cuda-toolkit@v0.2.35 + id: cuda-toolkit + with: + cuda: ${{ matrix.cuda }} + use-github-cache: false + + - name: Install uv and Python ${{ matrix.pyver }} + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.pyver }} + activate-environment: true + enable-cache: true + + - name: Install dependencies + run: | + git config --system core.longpaths true + uv pip install --upgrade build setuptools wheel packaging + + - name: Setup MSVC environment for nvcc + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + echo PATH=%PATH%>>%GITHUB_ENV% + echo INCLUDE=%INCLUDE%>>%GITHUB_ENV% + echo LIB=%LIB%>>%GITHUB_ENV% + echo LIBPATH=%LIBPATH%>>%GITHUB_ENV% + + - name: Build wheel + run: | + $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.', '') + + $env:CUDA_HOME = $env:CUDA_PATH + $env:CUDA_TOOLKIT_ROOT_DIR = $env:CUDA_PATH + $env:VERBOSE = '1' + + # Force CMake to use Ninja + LLVM/Clang instead of the default + # Visual Studio generator. MSVC skips several GGML CPU all-variant + # backends, such as ivybridge, piledriver, cooperlake, zen4, and + # sapphirerapids. + $env:CMAKE_GENERATOR = 'Ninja Multi-Config' + + $toolchainCandidates = @( + (Join-Path $env:GITHUB_WORKSPACE "vendor\llama.cpp\cmake\x64-windows-llvm.cmake"), + (Join-Path $env:GITHUB_WORKSPACE "cmake\x64-windows-llvm.cmake") + ) + + $toolchainFile = $toolchainCandidates | + Where-Object { Test-Path $_ } | + Select-Object -First 1 + + if (!$toolchainFile) { + Write-Error "Toolchain file not found. Checked: $($toolchainCandidates -join ', ')" + exit 1 + } + + $toolchainFile = $toolchainFile.Replace('\', '/') + Write-Output "Using toolchain file: $toolchainFile" + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend DLLs. + # - GGML_CPU_ALL_VARIANTS builds CPU variant DLLs such as ggml-cpu-x64, + # ggml-cpu-haswell, ggml-cpu-alderlake, etc. + # - GGML_NATIVE=OFF avoids binding the wheel to the runner CPU. + + # Suppress CUDA compiler warnings + $cudaDiagSuppress = '--diag-suppress=177,221,550' + + $cmakeArgs = @( + # Windows toolchain / common runtime + '-DCMAKE_TOOLCHAIN_FILE=vendor/llama.cpp/cmake/x64-windows-llvm.cmake' + '-DLLAMA_BUILD_BORINGSSL=ON' + + # Disable non-wheel targets + '-DLLAMA_BUILD_EXAMPLES=OFF' + '-DLLAMA_BUILD_TESTS=OFF' + '-DLLAMA_BUILD_TOOLS=OFF' + '-DLLAMA_BUILD_SERVER=OFF' + '-DLLAMA_BUILD_UI=OFF' + '-DLLAMA_USE_PREBUILT_UI=OFF' + '-DLLAMA_CURL=OFF' + + # GGML dynamic backend layout + '-DGGML_CPU=ON' + '-DGGML_CUDA=ON' + '-DGGML_NATIVE=OFF' + '-DGGML_BACKEND_DL=ON' + '-DGGML_CPU_ALL_VARIANTS=ON' + '-DGGML_OPENMP=ON' + + # CUDA backend + "-DCMAKE_CUDA_ARCHITECTURES=$env:CUDAARCHVER" + '-DGGML_CUDA_FORCE_MMQ=ON' + '-DCUDA_SEPARABLE_COMPILATION=ON' + "-DCMAKE_CUDA_FLAGS=$cudaDiagSuppress" + + # Build behavior + "-DCMAKE_BUILD_PARALLEL_LEVEL=$env:MAX_JOBS" + '-DENABLE_CCACHE=ON' + ) + + $env:CMAKE_ARGS = $cmakeArgs -join ' ' + Write-Output "CMAKE_ARGS=$env:CMAKE_ARGS" + + python -m build --wheel + + # Check if wheel was built + if (!(Test-Path '.\dist\*.whl')) { + Write-Error "No wheel built in dist/ directory" + exit 1 + } + + $wheelFile = Get-Item '.\dist\*.whl' | Select-Object -First 1 + + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl + $parts = $wheelFile.Name.Split('-') + $distName = $parts[0] + $version = $parts[1] + $pyTag = $parts[2] + $abiTag = $parts[3] + $platTag = $parts[4] + + # CPU all-variants is now an internal runtime layout detail. + $newVersion = "$version+cu$cudaVersion" + $newName = "$distName-$newVersion-$pyTag-$abiTag-$platTag" + + # Rename wheel file + Rename-Item -Path $wheelFile.FullName -NewName $newName + Write-Output "Renamed wheel to: $newName" + + # Write the build tag to the output + Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV + Write-Output "TAG_VERSION=$version" >> $env:GITHUB_ENV + + - name: Get current date + id: get-date + run: | + $currentDate = Get-Date -UFormat "%Y%m%d" + Write-Output "BUILD_DATE=$currentDate" >> $env:GITHUB_ENV + + - name: Create release + if: always() && env.TAG_VERSION != '' + uses: softprops/action-gh-release@v3 + with: + files: dist/* + # Set tag_name to v-cu-win- + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-win-${{ env.BUILD_DATE }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From df07219dee25ae2cc95f842bd1a7c81e6bfb599f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 07:57:05 +0800 Subject: [PATCH 100/304] ci(cu12+windows): build CU124-128 wheels with GGML dynamic backends for windows - Replace the old CPU/AVX release tag matrix with a single CU124-128 backend wheel layout. - Enable `GGML_BACKEND_DL` and `GGML_CPU_ALL_VARIANTS` so Windows wheels ship runtime-loadable GGML backend DLLs and CPU variant backends. - Use the Windows LLVM toolchain and disable non-wheel targets such as examples, tests, tools, server, embedded UI, and curl. - Remove the `.basic` style local version suffix and publish wheels as `+cu124/cu126/cu128`. - Update CUDA architectures to CUDA 13.1 and simplify CMake argument handling. Signed-off-by: JamePeng --- .github/workflows/build-wheels-cu124-win.yml | 145 ++++++++++++------ .github/workflows/build-wheels-cu126-win.yml | 145 ++++++++++++------ .github/workflows/build-wheels-cu128-win.yml | 147 +++++++++++++------ 3 files changed, 304 insertions(+), 133 deletions(-) diff --git a/.github/workflows/build-wheels-cu124-win.yml b/.github/workflows/build-wheels-cu124-win.yml index 01bd48e7de..e856533410 100644 --- a/.github/workflows/build-wheels-cu124-win.yml +++ b/.github/workflows/build-wheels-cu124-win.yml @@ -8,85 +8,141 @@ permissions: jobs: build_wheels: - name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag }} + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu124 runs-on: ${{ matrix.os }} + strategy: + fail-fast: false matrix: - os: ['windows-2022'] + os: ["windows-2022"] pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] cuda: ["12.4.1"] - releasetag: ["Basic"] cudaarch: ["70-real;75-real;80-real;86-real;87-real;89-real;90-real"] + defaults: run: shell: pwsh + env: CUDAVER: ${{ matrix.cuda }} - AVXVER: ${{ matrix.releasetag }} CUDAARCHVER: ${{ matrix.cudaarch }} - # https://cmake.org/cmake/help/latest/prop_tgt/CUDA_ARCHITECTURES.html - # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/#gpu-feature-list - # e.g. "all" "89" "90" "100" "120" - MAX_JOBS: 8 + MAX_JOBS: 12 steps: - name: Add MSBuild to PATH - if: runner.os == 'Windows' uses: microsoft/setup-msbuild@v3 with: msbuild-architecture: x64 - - uses: actions/checkout@v6 + - name: Checkout + uses: actions/checkout@v6 with: - submodules: "recursive" + submodules: recursive - # from kingbri1/flash-attention build-wheels.yml - name: Install CUDA ${{ matrix.cuda }} uses: Jimver/cuda-toolkit@v0.2.35 id: cuda-toolkit with: - cuda: "${{ matrix.cuda }}" + cuda: ${{ matrix.cuda }} use-github-cache: false - # from astral-sh/setup-uv - - name: Install the latest version of uv and set the python version + - name: Install uv and Python ${{ matrix.pyver }} uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true enable-cache: true - - name: Install Dependencies + - name: Install dependencies run: | git config --system core.longpaths true uv pip install --upgrade build setuptools wheel packaging - - name: Build Wheel + - name: Setup MSVC environment for nvcc + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + echo PATH=%PATH%>>%GITHUB_ENV% + echo INCLUDE=%INCLUDE%>>%GITHUB_ENV% + echo LIB=%LIB%>>%GITHUB_ENV% + echo LIBPATH=%LIBPATH%>>%GITHUB_ENV% + + - name: Build wheel run: | - $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','') + $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.', '') + $env:CUDA_HOME = $env:CUDA_PATH $env:CUDA_TOOLKIT_ROOT_DIR = $env:CUDA_PATH $env:VERBOSE = '1' - $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=' + $env:CUDAARCHVER + ' -DCMAKE_BUILD_PARALLEL_LEVEL=' + $env:MAX_JOBS - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=on -DCUDA_SEPARABLE_COMPILATION=on $env:CMAKE_ARGS" - $env:CMAKE_ARGS = "-DENABLE_CCACHE=on -DLLAMA_CURL=off $env:CMAKE_ARGS" - if ($env:AVXVER -eq 'AVX') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off' - } - if ($env:AVXVER -eq 'AVX2') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off' - } - if ($env:AVXVER -eq 'AVXVNNI') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX_VNNI=on' - } - # if ($env:AVXVER -eq 'AVX512') { - # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on' - # } - # Basic options for compiling without AVX instructions - if ($env:AVXVER -eq 'Basic') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX_VNNI=off -DGGML_AVX512=off -DGGML_AVX512_VBMI=off -DGGML_AVX512_VNNI=off -DGGML_AVX512_BF16=off -DGGML_FMA=off -DGGML_F16C=off' + # Force CMake to use Ninja + LLVM/Clang instead of the default + # Visual Studio generator. MSVC skips several GGML CPU all-variant + # backends, such as ivybridge, piledriver, cooperlake, zen4, and + # sapphirerapids. + $env:CMAKE_GENERATOR = 'Ninja Multi-Config' + + $toolchainCandidates = @( + (Join-Path $env:GITHUB_WORKSPACE "vendor\llama.cpp\cmake\x64-windows-llvm.cmake"), + (Join-Path $env:GITHUB_WORKSPACE "cmake\x64-windows-llvm.cmake") + ) + + $toolchainFile = $toolchainCandidates | + Where-Object { Test-Path $_ } | + Select-Object -First 1 + + if (!$toolchainFile) { + Write-Error "Toolchain file not found. Checked: $($toolchainCandidates -join ', ')" + exit 1 } + + $toolchainFile = $toolchainFile.Replace('\', '/') + Write-Output "Using toolchain file: $toolchainFile" + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend DLLs. + # - GGML_CPU_ALL_VARIANTS builds CPU variant DLLs such as ggml-cpu-x64, + # ggml-cpu-haswell, ggml-cpu-alderlake, etc. + # - GGML_NATIVE=OFF avoids binding the wheel to the runner CPU. + + # Suppress CUDA compiler warnings + $cudaDiagSuppress = '--diag-suppress=177,221,550' + + $cmakeArgs = @( + # Windows toolchain / common runtime + '-DCMAKE_TOOLCHAIN_FILE=vendor/llama.cpp/cmake/x64-windows-llvm.cmake' + '-DLLAMA_BUILD_BORINGSSL=ON' + + # Disable non-wheel targets + '-DLLAMA_BUILD_EXAMPLES=OFF' + '-DLLAMA_BUILD_TESTS=OFF' + '-DLLAMA_BUILD_TOOLS=OFF' + '-DLLAMA_BUILD_SERVER=OFF' + '-DLLAMA_BUILD_UI=OFF' + '-DLLAMA_USE_PREBUILT_UI=OFF' + '-DLLAMA_CURL=OFF' + + # GGML dynamic backend layout + '-DGGML_CPU=ON' + '-DGGML_CUDA=ON' + '-DGGML_NATIVE=OFF' + '-DGGML_BACKEND_DL=ON' + '-DGGML_CPU_ALL_VARIANTS=ON' + '-DGGML_OPENMP=ON' + + # CUDA backend + "-DCMAKE_CUDA_ARCHITECTURES=$env:CUDAARCHVER" + '-DGGML_CUDA_FORCE_MMQ=ON' + '-DCUDA_SEPARABLE_COMPILATION=ON' + "-DCMAKE_CUDA_FLAGS=$cudaDiagSuppress" + + # Build behavior + "-DCMAKE_BUILD_PARALLEL_LEVEL=$env:MAX_JOBS" + '-DENABLE_CCACHE=ON' + ) + + $env:CMAKE_ARGS = $cmakeArgs -join ' ' + Write-Output "CMAKE_ARGS=$env:CMAKE_ARGS" + python -m build --wheel # Check if wheel was built @@ -97,7 +153,8 @@ jobs: $wheelFile = Get-Item '.\dist\*.whl' | Select-Object -First 1 - # Split wheel filename: name-ver-py-abi-plat.whl + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl $parts = $wheelFile.Name.Split('-') $distName = $parts[0] $version = $parts[1] @@ -105,30 +162,30 @@ jobs: $abiTag = $parts[3] $platTag = $parts[4] - $newVersion = "$version+cu$cudaVersion.$($env:AVXVER.ToLower())" - + # CPU all-variants is now an internal runtime layout detail. + $newVersion = "$version+cu$cudaVersion" $newName = "$distName-$newVersion-$pyTag-$abiTag-$platTag" # Rename wheel file Rename-Item -Path $wheelFile.FullName -NewName $newName Write-Output "Renamed wheel to: $newName" - # write the build tag to the output + # Write the build tag to the output Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV Write-Output "TAG_VERSION=$version" >> $env:GITHUB_ENV - - name: Get Current Date + - name: Get current date id: get-date run: | $currentDate = Get-Date -UFormat "%Y%m%d" Write-Output "BUILD_DATE=$currentDate" >> $env:GITHUB_ENV - - name: Create Release + - name: Create release if: always() && env.TAG_VERSION != '' uses: softprops/action-gh-release@v3 with: files: dist/* - # Set tag_name to -cu--win- - tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-${{ env.AVXVER }}-win-${{ env.BUILD_DATE }} + # Set tag_name to v-cu-win- + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-win-${{ env.BUILD_DATE }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-wheels-cu126-win.yml b/.github/workflows/build-wheels-cu126-win.yml index 9330cb130b..b77b17917f 100644 --- a/.github/workflows/build-wheels-cu126-win.yml +++ b/.github/workflows/build-wheels-cu126-win.yml @@ -8,85 +8,141 @@ permissions: jobs: build_wheels: - name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag }} + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu126 runs-on: ${{ matrix.os }} + strategy: + fail-fast: false matrix: - os: ['windows-2022'] + os: ["windows-2022"] pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] cuda: ["12.6.3"] - releasetag: ["Basic"] cudaarch: ["70-real;75-real;80-real;86-real;87-real;89-real;90-real"] + defaults: run: shell: pwsh + env: CUDAVER: ${{ matrix.cuda }} - AVXVER: ${{ matrix.releasetag }} CUDAARCHVER: ${{ matrix.cudaarch }} - # https://cmake.org/cmake/help/latest/prop_tgt/CUDA_ARCHITECTURES.html - # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/#gpu-feature-list - # e.g. "all" "89" "90" "100" "120" - MAX_JOBS: 8 + MAX_JOBS: 12 steps: - name: Add MSBuild to PATH - if: runner.os == 'Windows' uses: microsoft/setup-msbuild@v3 with: msbuild-architecture: x64 - - uses: actions/checkout@v6 + - name: Checkout + uses: actions/checkout@v6 with: - submodules: "recursive" + submodules: recursive - # from kingbri1/flash-attention build-wheels.yml - name: Install CUDA ${{ matrix.cuda }} uses: Jimver/cuda-toolkit@v0.2.35 id: cuda-toolkit with: - cuda: "${{ matrix.cuda }}" + cuda: ${{ matrix.cuda }} use-github-cache: false - # from astral-sh/setup-uv - - name: Install the latest version of uv and set the python version + - name: Install uv and Python ${{ matrix.pyver }} uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true enable-cache: true - - name: Install Dependencies + - name: Install dependencies run: | git config --system core.longpaths true uv pip install --upgrade build setuptools wheel packaging - - name: Build Wheel + - name: Setup MSVC environment for nvcc + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + echo PATH=%PATH%>>%GITHUB_ENV% + echo INCLUDE=%INCLUDE%>>%GITHUB_ENV% + echo LIB=%LIB%>>%GITHUB_ENV% + echo LIBPATH=%LIBPATH%>>%GITHUB_ENV% + + - name: Build wheel run: | - $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','') + $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.', '') + $env:CUDA_HOME = $env:CUDA_PATH $env:CUDA_TOOLKIT_ROOT_DIR = $env:CUDA_PATH $env:VERBOSE = '1' - $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=' + $env:CUDAARCHVER + ' -DCMAKE_BUILD_PARALLEL_LEVEL=' + $env:MAX_JOBS - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=on -DCUDA_SEPARABLE_COMPILATION=on $env:CMAKE_ARGS" - $env:CMAKE_ARGS = "-DENABLE_CCACHE=on -DLLAMA_CURL=off $env:CMAKE_ARGS" - if ($env:AVXVER -eq 'AVX') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off' - } - if ($env:AVXVER -eq 'AVX2') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off' - } - if ($env:AVXVER -eq 'AVXVNNI') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX_VNNI=on' - } - # if ($env:AVXVER -eq 'AVX512') { - # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on' - # } - # Basic options for compiling without AVX instructions - if ($env:AVXVER -eq 'Basic') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX_VNNI=off -DGGML_AVX512=off -DGGML_AVX512_VBMI=off -DGGML_AVX512_VNNI=off -DGGML_AVX512_BF16=off -DGGML_FMA=off -DGGML_F16C=off' + # Force CMake to use Ninja + LLVM/Clang instead of the default + # Visual Studio generator. MSVC skips several GGML CPU all-variant + # backends, such as ivybridge, piledriver, cooperlake, zen4, and + # sapphirerapids. + $env:CMAKE_GENERATOR = 'Ninja Multi-Config' + + $toolchainCandidates = @( + (Join-Path $env:GITHUB_WORKSPACE "vendor\llama.cpp\cmake\x64-windows-llvm.cmake"), + (Join-Path $env:GITHUB_WORKSPACE "cmake\x64-windows-llvm.cmake") + ) + + $toolchainFile = $toolchainCandidates | + Where-Object { Test-Path $_ } | + Select-Object -First 1 + + if (!$toolchainFile) { + Write-Error "Toolchain file not found. Checked: $($toolchainCandidates -join ', ')" + exit 1 } + + $toolchainFile = $toolchainFile.Replace('\', '/') + Write-Output "Using toolchain file: $toolchainFile" + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend DLLs. + # - GGML_CPU_ALL_VARIANTS builds CPU variant DLLs such as ggml-cpu-x64, + # ggml-cpu-haswell, ggml-cpu-alderlake, etc. + # - GGML_NATIVE=OFF avoids binding the wheel to the runner CPU. + + # Suppress CUDA compiler warnings + $cudaDiagSuppress = '--diag-suppress=177,221,550' + + $cmakeArgs = @( + # Windows toolchain / common runtime + '-DCMAKE_TOOLCHAIN_FILE=vendor/llama.cpp/cmake/x64-windows-llvm.cmake' + '-DLLAMA_BUILD_BORINGSSL=ON' + + # Disable non-wheel targets + '-DLLAMA_BUILD_EXAMPLES=OFF' + '-DLLAMA_BUILD_TESTS=OFF' + '-DLLAMA_BUILD_TOOLS=OFF' + '-DLLAMA_BUILD_SERVER=OFF' + '-DLLAMA_BUILD_UI=OFF' + '-DLLAMA_USE_PREBUILT_UI=OFF' + '-DLLAMA_CURL=OFF' + + # GGML dynamic backend layout + '-DGGML_CPU=ON' + '-DGGML_CUDA=ON' + '-DGGML_NATIVE=OFF' + '-DGGML_BACKEND_DL=ON' + '-DGGML_CPU_ALL_VARIANTS=ON' + '-DGGML_OPENMP=ON' + + # CUDA backend + "-DCMAKE_CUDA_ARCHITECTURES=$env:CUDAARCHVER" + '-DGGML_CUDA_FORCE_MMQ=ON' + '-DCUDA_SEPARABLE_COMPILATION=ON' + "-DCMAKE_CUDA_FLAGS=$cudaDiagSuppress" + + # Build behavior + "-DCMAKE_BUILD_PARALLEL_LEVEL=$env:MAX_JOBS" + '-DENABLE_CCACHE=ON' + ) + + $env:CMAKE_ARGS = $cmakeArgs -join ' ' + Write-Output "CMAKE_ARGS=$env:CMAKE_ARGS" + python -m build --wheel # Check if wheel was built @@ -97,7 +153,8 @@ jobs: $wheelFile = Get-Item '.\dist\*.whl' | Select-Object -First 1 - # Split wheel filename: name-ver-py-abi-plat.whl + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl $parts = $wheelFile.Name.Split('-') $distName = $parts[0] $version = $parts[1] @@ -105,30 +162,30 @@ jobs: $abiTag = $parts[3] $platTag = $parts[4] - $newVersion = "$version+cu$cudaVersion.$($env:AVXVER.ToLower())" - + # CPU all-variants is now an internal runtime layout detail. + $newVersion = "$version+cu$cudaVersion" $newName = "$distName-$newVersion-$pyTag-$abiTag-$platTag" # Rename wheel file Rename-Item -Path $wheelFile.FullName -NewName $newName Write-Output "Renamed wheel to: $newName" - # write the build tag to the output + # Write the build tag to the output Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV Write-Output "TAG_VERSION=$version" >> $env:GITHUB_ENV - - name: Get Current Date + - name: Get current date id: get-date run: | $currentDate = Get-Date -UFormat "%Y%m%d" Write-Output "BUILD_DATE=$currentDate" >> $env:GITHUB_ENV - - name: Create Release + - name: Create release if: always() && env.TAG_VERSION != '' uses: softprops/action-gh-release@v3 with: files: dist/* - # Set tag_name to -cu--win- - tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-${{ env.AVXVER }}-win-${{ env.BUILD_DATE }} + # Set tag_name to v-cu-win- + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-win-${{ env.BUILD_DATE }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-wheels-cu128-win.yml b/.github/workflows/build-wheels-cu128-win.yml index 98ebbc4127..223473dde6 100644 --- a/.github/workflows/build-wheels-cu128-win.yml +++ b/.github/workflows/build-wheels-cu128-win.yml @@ -8,85 +8,141 @@ permissions: jobs: build_wheels: - name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag }} + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu128 runs-on: ${{ matrix.os }} + strategy: + fail-fast: false matrix: - os: ['windows-2022'] + os: ["windows-2022"] pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] cuda: ["12.8.1"] - releasetag: ["Basic"] - cudaarch: ["75-real;80-real;86-real;87-real;89-real;90-real;100-real;101-real;120-real"] + cudaarch: ["75-real;80-real;86-real;87-real;89-real;90-real;100-real;120-real"] + defaults: run: shell: pwsh + env: CUDAVER: ${{ matrix.cuda }} - AVXVER: ${{ matrix.releasetag }} CUDAARCHVER: ${{ matrix.cudaarch }} - # https://cmake.org/cmake/help/latest/prop_tgt/CUDA_ARCHITECTURES.html - # https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/#gpu-feature-list - # e.g. "all" "89" "90" "100" "120" - MAX_JOBS: 8 + MAX_JOBS: 12 steps: - name: Add MSBuild to PATH - if: runner.os == 'Windows' uses: microsoft/setup-msbuild@v3 with: msbuild-architecture: x64 - - uses: actions/checkout@v6 + - name: Checkout + uses: actions/checkout@v6 with: - submodules: "recursive" + submodules: recursive - # from kingbri1/flash-attention build-wheels.yml - name: Install CUDA ${{ matrix.cuda }} uses: Jimver/cuda-toolkit@v0.2.35 id: cuda-toolkit with: - cuda: "${{ matrix.cuda }}" + cuda: ${{ matrix.cuda }} use-github-cache: false - # from astral-sh/setup-uv - - name: Install the latest version of uv and set the python version + - name: Install uv and Python ${{ matrix.pyver }} uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true enable-cache: true - - name: Install Dependencies + - name: Install dependencies run: | git config --system core.longpaths true uv pip install --upgrade build setuptools wheel packaging - - name: Build Wheel + - name: Setup MSVC environment for nvcc + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + echo PATH=%PATH%>>%GITHUB_ENV% + echo INCLUDE=%INCLUDE%>>%GITHUB_ENV% + echo LIB=%LIB%>>%GITHUB_ENV% + echo LIBPATH=%LIBPATH%>>%GITHUB_ENV% + + - name: Build wheel run: | - $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','') + $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.', '') + $env:CUDA_HOME = $env:CUDA_PATH $env:CUDA_TOOLKIT_ROOT_DIR = $env:CUDA_PATH $env:VERBOSE = '1' - $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=' + $env:CUDAARCHVER + ' -DCMAKE_BUILD_PARALLEL_LEVEL=' + $env:MAX_JOBS - $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=on -DCUDA_SEPARABLE_COMPILATION=on $env:CMAKE_ARGS" - $env:CMAKE_ARGS = "-DENABLE_CCACHE=on -DLLAMA_CURL=off $env:CMAKE_ARGS" - if ($env:AVXVER -eq 'AVX') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' - } - if ($env:AVXVER -eq 'AVX2') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off' - } - if ($env:AVXVER -eq 'AVXVNNI') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX_VNNI=on' - } - # if ($env:AVXVER -eq 'AVX512') { - # $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on' - # } - # Basic options for compiling without AVX instructions - if ($env:AVXVER -eq 'Basic') { - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX_VNNI=off -DGGML_AVX512=off -DGGML_AVX512_VBMI=off -DGGML_AVX512_VNNI=off -DGGML_AVX512_BF16=off -DGGML_FMA=off -DGGML_F16C=off' + # Force CMake to use Ninja + LLVM/Clang instead of the default + # Visual Studio generator. MSVC skips several GGML CPU all-variant + # backends, such as ivybridge, piledriver, cooperlake, zen4, and + # sapphirerapids. + $env:CMAKE_GENERATOR = 'Ninja Multi-Config' + + $toolchainCandidates = @( + (Join-Path $env:GITHUB_WORKSPACE "vendor\llama.cpp\cmake\x64-windows-llvm.cmake"), + (Join-Path $env:GITHUB_WORKSPACE "cmake\x64-windows-llvm.cmake") + ) + + $toolchainFile = $toolchainCandidates | + Where-Object { Test-Path $_ } | + Select-Object -First 1 + + if (!$toolchainFile) { + Write-Error "Toolchain file not found. Checked: $($toolchainCandidates -join ', ')" + exit 1 } + + $toolchainFile = $toolchainFile.Replace('\', '/') + Write-Output "Using toolchain file: $toolchainFile" + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend DLLs. + # - GGML_CPU_ALL_VARIANTS builds CPU variant DLLs such as ggml-cpu-x64, + # ggml-cpu-haswell, ggml-cpu-alderlake, etc. + # - GGML_NATIVE=OFF avoids binding the wheel to the runner CPU. + + # Suppress CUDA compiler warnings + $cudaDiagSuppress = '--diag-suppress=177,221,550' + + $cmakeArgs = @( + # Windows toolchain / common runtime + '-DCMAKE_TOOLCHAIN_FILE=vendor/llama.cpp/cmake/x64-windows-llvm.cmake' + '-DLLAMA_BUILD_BORINGSSL=ON' + + # Disable non-wheel targets + '-DLLAMA_BUILD_EXAMPLES=OFF' + '-DLLAMA_BUILD_TESTS=OFF' + '-DLLAMA_BUILD_TOOLS=OFF' + '-DLLAMA_BUILD_SERVER=OFF' + '-DLLAMA_BUILD_UI=OFF' + '-DLLAMA_USE_PREBUILT_UI=OFF' + '-DLLAMA_CURL=OFF' + + # GGML dynamic backend layout + '-DGGML_CPU=ON' + '-DGGML_CUDA=ON' + '-DGGML_NATIVE=OFF' + '-DGGML_BACKEND_DL=ON' + '-DGGML_CPU_ALL_VARIANTS=ON' + '-DGGML_OPENMP=ON' + + # CUDA backend + "-DCMAKE_CUDA_ARCHITECTURES=$env:CUDAARCHVER" + '-DGGML_CUDA_FORCE_MMQ=ON' + '-DCUDA_SEPARABLE_COMPILATION=ON' + "-DCMAKE_CUDA_FLAGS=$cudaDiagSuppress" + + # Build behavior + "-DCMAKE_BUILD_PARALLEL_LEVEL=$env:MAX_JOBS" + '-DENABLE_CCACHE=ON' + ) + + $env:CMAKE_ARGS = $cmakeArgs -join ' ' + Write-Output "CMAKE_ARGS=$env:CMAKE_ARGS" + python -m build --wheel # Check if wheel was built @@ -97,7 +153,8 @@ jobs: $wheelFile = Get-Item '.\dist\*.whl' | Select-Object -First 1 - # Split file name: name-ver-py-abi-plat.whl + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl $parts = $wheelFile.Name.Split('-') $distName = $parts[0] $version = $parts[1] @@ -105,30 +162,30 @@ jobs: $abiTag = $parts[3] $platTag = $parts[4] - $newVersion = "$version+cu$cudaVersion.$($env:AVXVER.ToLower())" - + # CPU all-variants is now an internal runtime layout detail. + $newVersion = "$version+cu$cudaVersion" $newName = "$distName-$newVersion-$pyTag-$abiTag-$platTag" # Rename wheel file Rename-Item -Path $wheelFile.FullName -NewName $newName Write-Output "Renamed wheel to: $newName" - # write the build tag to the output + # Write the build tag to the output Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV Write-Output "TAG_VERSION=$version" >> $env:GITHUB_ENV - - name: Get Current Date + - name: Get current date id: get-date run: | $currentDate = Get-Date -UFormat "%Y%m%d" Write-Output "BUILD_DATE=$currentDate" >> $env:GITHUB_ENV - - name: Create Release + - name: Create release if: always() && env.TAG_VERSION != '' uses: softprops/action-gh-release@v3 with: files: dist/* - # Set tag_name to -cu--win- - tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-${{ env.AVXVER }}-win-${{ env.BUILD_DATE }} + # Set tag_name to v-cu-win- + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-win-${{ env.BUILD_DATE }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e47843f591f1b879e461bcce3d1877ff943c3f71 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 07:58:19 +0800 Subject: [PATCH 101/304] Update Submodule vendor/llama.cpp 49d1701..b64739e --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 49d1701bd2..b64739ea39 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 49d1701bd24e4cedf6dfec9e50e185111203946b +Subproject commit b64739ea393b3c9d07cc9907e0a611f707838051 From 39785d0efb7a45490fdc45ac340ff2ec1a2eae8c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 08:08:45 +0800 Subject: [PATCH 102/304] fix(_internals): Remove unnecessary free operations; models should not be released within the context. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b4ba1f4b21..277d22aebf 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -494,7 +494,6 @@ def __init__( ctx = llama_cpp.llama_init_from_model(self.model.model, self.params) if ctx is None: - llama_cpp.llama_model_free(self.model.model) raise ValueError("Failed to create context with model") self.ctx = ctx From 127881293e712ffdc3cae43af969d2a46e52c80e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 08:47:19 +0800 Subject: [PATCH 103/304] Sync llama.cpp API 20260517 - llama + spec: MTP Support Signed-off-by: JamePeng --- llama_cpp/_internals.py | 3 +++ llama_cpp/llama.py | 8 ++++++++ llama_cpp/llama_cpp.py | 41 ++++++++++++++++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 277d22aebf..a8dd56083b 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -533,6 +533,9 @@ def n_ubatch(self) -> int: def n_seq_max(self) -> int: return llama_cpp.llama_n_seq_max(self.ctx) + def n_rs_seq(self) -> int: + return llama_cpp.llama_n_rs_seq(self.ctx) + def pooling_type(self) -> int: return llama_cpp.llama_pooling_type(self.ctx) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 8b1070be4f..734485802e 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -119,8 +119,12 @@ def __init__( n_batch: int = 2048, n_ubatch: int = 512, n_seq_max: int = 1, + n_rs_seq: int = 0, n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, + ctx_type: Optional[ + int + ] = llama_cpp_lib.llama_context_type.LLAMA_CONTEXT_TYPE_DEFAULT, rope_scaling_type: Optional[ int ] = llama_cpp_lib.llama_rope_scaling_type.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, @@ -474,6 +478,7 @@ def __init__( self.n_batch = min(n_ctx, n_batch) # ??? self.n_keep = n_keep if n_keep > 0 else 256 self.n_seq_max = n_seq_max + self.n_rs_seq = n_rs_seq self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count() @@ -486,8 +491,11 @@ def __init__( self.context_params.n_batch = self.n_batch self.context_params.n_ubatch = min(self.n_batch, n_ubatch) self.context_params.n_seq_max = self.n_seq_max + self.context_params.n_rs_seq = self.n_rs_seq self.context_params.n_threads = self.n_threads self.context_params.n_threads_batch = self.n_threads_batch + + self.context_params.ctx_type = ctx_type self.context_params.rope_scaling_type = ( rope_scaling_type if rope_scaling_type is not None diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index cc900c0648..ec2b665a16 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -471,6 +471,14 @@ class llama_split_mode(enum.IntEnum): LLAMA_SPLIT_MODE_ROW = 2 LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_context_type { +# LLAMA_CONTEXT_TYPE_DEFAULT = 0, +# LLAMA_CONTEXT_TYPE_MTP = 1, +# }; +class llama_context_type(enum.IntEnum): + LLAMA_CONTEXT_TYPE_DEFAULT = 0 + LLAMA_CONTEXT_TYPE_MTP = 1 + # typedef struct llama_token_data { # llama_token id; // token id # float logit; // log-odds of the token @@ -827,9 +835,11 @@ class llama_sampler_seq_config(ctypes.Structure): # uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode # uint32_t n_ubatch; // physical maximum batch size # uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) +# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] # int32_t n_threads; // number of threads to use for generation # int32_t n_threads_batch; // number of threads to use for batch processing +# enum llama_context_type ctx_type; // set the context type (e.g. MTP) # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` # enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id # enum llama_attention_type attention_type; // attention type to use for embeddings @@ -843,13 +853,14 @@ class llama_sampler_seq_config(ctypes.Structure): # float yarn_beta_fast; // YaRN low correction dim # float yarn_beta_slow; // YaRN high correction dim # uint32_t yarn_orig_ctx; // YaRN original context size -# float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, < 0 disabled (default) +# float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default) # ggml_backend_sched_eval_callback cb_eval; # void * cb_eval_user_data; # enum ggml_type type_k; // data type for K cache [EXPERIMENTAL] # enum ggml_type type_v; // data type for V cache [EXPERIMENTAL] + # // Abort callback # // if it returns true, execution of llama_decode() will be aborted # // currently works only with CPU execution @@ -862,11 +873,12 @@ class llama_sampler_seq_config(ctypes.Structure): # bool no_perf; // measure performance timings # bool op_offload; // offload host tensor operations to device # bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) -# // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some casesAdd commentMore actions -# // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 +# // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases +# // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 # bool kv_unified; // use a unified buffer across the input sequences when computing the attention -# // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix -# // ref: https://github.com/ggml-org/llama.cpp/pull/14363 +# // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix +# // ref: https://github.com/ggml-org/llama.cpp/pull/14363 + # // [EXPERIMENTAL] # // backend sampler chain configuration (make sure the caller keeps the sampler chains alive) # // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init) @@ -881,12 +893,16 @@ class llama_context_params(ctypes.Structure): n_batch (int): logical maximum batch size that can be submitted to llama_decode n_ubatch (int): physical maximum batch size n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) + n_rs_seq (int): number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing + + ctx_type (int): set the context type (e.g. MTP) rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type` pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer) attention_type (int): attention type to use for embeddings flash_attn_type (int): when to enable Flash Attention + rope_freq_base (float): RoPE base frequency, 0 = from model rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model @@ -895,18 +911,23 @@ class llama_context_params(ctypes.Structure): yarn_beta_slow (float): YaRN high correction dim yarn_orig_ctx (int): YaRN original context size defrag_thold (float): [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default) + cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval + type_k (int): data type for K cache type_v (int): data type for V cache + abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback + embeddings (bool): if true, extract embeddings (together with logits) offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU no_perf (bool): whether to measure performance timings op_offload(bool): whether to offload host tensor operations to device swa_full(bool): whether to use full-size SWA cache kv_unified(bool): use a unified buffer across the input sequences when computing the attention + samplers(llama_sampler_seq_config *): the samplers must be sampler chains (i.e. use llama_sampler_chain_init) n_samplers(size_t): numbers of sampler chains """ @@ -916,8 +937,10 @@ class llama_context_params(ctypes.Structure): n_batch: int n_ubatch: int n_seq_max: int + n_rs_seq: int n_threads: int n_threads_batch: int + ctx_type: int rope_scaling_type: int pooling_type: int attention_type: int @@ -950,8 +973,10 @@ class llama_context_params(ctypes.Structure): ("n_batch", ctypes.c_uint32), ("n_ubatch", ctypes.c_uint32), ("n_seq_max", ctypes.c_uint32), + ("n_rs_seq", ctypes.c_uint32), ("n_threads", ctypes.c_int32), ("n_threads_batch", ctypes.c_int32), + ("ctx_type", ctypes.c_int), ("rope_scaling_type", ctypes.c_int), ("pooling_type", ctypes.c_int), ("attention_type", ctypes.c_int), @@ -1602,6 +1627,12 @@ def llama_n_seq_max(ctx: llama_context_p, /) -> int: ... +# LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx); +@ctypes_function("llama_n_rs_seq", [llama_context_p_ctypes], ctypes.c_uint32) +def llama_n_rs_seq(ctx: llama_context_p, /) -> int: + ... + + # DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); @ctypes_function("llama_n_ctx_train", [llama_model_p_ctypes], ctypes.c_int32) def llama_n_ctx_train(model: llama_model_p, /) -> int: From 50627bfa9b16a5061df699b04006029070935c62 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 09:06:03 +0800 Subject: [PATCH 104/304] fix(context): prevent operations on uninitialized or closed contexts - Introduce an internal `_assert_ctx()` method to verify that the underlying C context (`self.ctx`) is valid before invoking dependent `llama.cpp` operations. - Apply `_assert_ctx()` to critical methods (`encode`, `decode`, `get_logits*`, `get_embeddings*`) to prevent hard crashes (segfaults) caused by passing null pointers to the C API. - Upgrade the context initialization failure exception from a generic `ValueError` to a detailed `RuntimeError`, providing developers with actionable hints about potentially out-of-sync `llama_context_params`. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index a8dd56083b..c026440a2d 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -493,8 +493,13 @@ def __init__( ctx = llama_cpp.llama_init_from_model(self.model.model, self.params) - if ctx is None: - raise ValueError("Failed to create context with model") + if not ctx: + raise RuntimeError( + "Failed to create llama context with model. " + "This may indicate that llama_context_params is out of sync with " + "the bundled llama.cpp version, or that required context parameters " + "were not initialized correctly." + ) self.ctx = ctx @@ -518,6 +523,13 @@ def close(self): def __del__(self): self.close() + def _assert_ctx(self): + if not getattr(self, "ctx", None): + raise RuntimeError( + "LlamaContext is not initialized or has already been closed. " + "Context-dependent llama.cpp operations cannot continue." + ) + def n_ctx(self) -> int: return llama_cpp.llama_n_ctx(self.ctx) @@ -654,6 +666,7 @@ def set_state_seq_data_ext( # // Decoding API def encode(self, batch: LlamaBatch): + self._assert_ctx() return_code = llama_cpp.llama_encode( self.ctx, batch.batch, @@ -678,6 +691,7 @@ def decode(self, batch: 'LlamaBatch') -> int: RuntimeError: If a fatal, non-recoverable error occurs during decoding (e.g., negative error codes or invalid batch structures). """ + self._assert_ctx() return_code = llama_cpp.llama_decode(self.ctx, batch.batch) if return_code == 0: @@ -741,21 +755,27 @@ def synchronize(self): llama_cpp.llama_synchronize(self.ctx) def get_logits(self): + self._assert_ctx() return llama_cpp.llama_get_logits(self.ctx) def get_logits_ith(self, i: int): + self._assert_ctx() return llama_cpp.llama_get_logits_ith(self.ctx, i) def set_embeddings(self, embeddings: bool): + self._assert_ctx() llama_cpp.llama_set_embeddings(self.ctx, embeddings) def get_embeddings(self): + self._assert_ctx() return llama_cpp.llama_get_embeddings(self.ctx) def get_embeddings_ith(self, i: int): + self._assert_ctx() return llama_cpp.llama_get_embeddings_ith(self.ctx, i) def get_embeddings_seq(self, seq_id: int): + self._assert_ctx() return llama_cpp.llama_get_embeddings_seq(self.ctx, seq_id) def reset_timings(self): From a4c8d77d1817bcf7255b8e93aa4733e5378d9211 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 12:29:48 +0800 Subject: [PATCH 105/304] ci(cu131+linux): build CU131 wheels with GGML dynamic backends for linux - Replace the old CPU/AVX release tag matrix with a single CU131 backend wheel layout. - Enable `GGML_BACKEND_DL` and `GGML_CPU_ALL_VARIANTS` so Linux wheels ship runtime-loadable GGML backend DLLs and CPU variant backends. - Disable non-wheel targets such as examples, tests, tools, server, embedded UI, and curl. - Remove the `.basic` style local version suffix and publish wheels as `+cu131`. - Update CUDA architectures to CUDA 13.1 and simplify CMake argument handling. Signed-off-by: JamePeng --- .../workflows/build-wheels-cu130-linux.yml | 132 --------------- .../workflows/build-wheels-cu131-linux.yml | 156 ++++++++++++++++++ 2 files changed, 156 insertions(+), 132 deletions(-) delete mode 100644 .github/workflows/build-wheels-cu130-linux.yml create mode 100644 .github/workflows/build-wheels-cu131-linux.yml diff --git a/.github/workflows/build-wheels-cu130-linux.yml b/.github/workflows/build-wheels-cu130-linux.yml deleted file mode 100644 index 4f4305ad3e..0000000000 --- a/.github/workflows/build-wheels-cu130-linux.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: Build Wheels(CU130) for Linux - -on: - workflow_dispatch: # Manual trigger - -permissions: - contents: write - -jobs: - build_wheels: - name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag == 'wheels' && 'AVX2' || matrix.releasetag }} - runs-on: ubuntu-22.04 - container: nvidia/cuda:13.0.2-cudnn-devel-ubuntu22.04 - strategy: - matrix: # Define the build matrix directly here - os: ["ubuntu-22.04"] - pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] # Python versions - cuda: ["13.0.2"] - releasetag: ["Basic"] # Controls CMAKE_ARGS for CPU features (even in CUDA build) - cudaarch: ["all"] # Controls target CUDA architectures for nvcc - - defaults: - run: - shell: bash - - env: - CUDAVER: ${{ matrix.cuda }} - AVXVER: ${{ matrix.releasetag }} - CUDAARCHVER: ${{ matrix.cudaarch }} - - steps: - - name: Install dependencies - run: | - apt update - apt install -y build-essential ccache cmake curl git libgomp1 libjpeg-dev libssl-dev - - - uses: actions/checkout@v6 # Checkout code - with: - submodules: "recursive" - - # from astral-sh/setup-uv - - name: Install the latest version of uv and set the python version - uses: astral-sh/setup-uv@v7 - with: - python-version: ${{ matrix.pyver }} - activate-environment: true - enable-cache: true - - - run: nvcc -V - - - name: Build Wheel With Cmake # Main build step: configures and builds the wheel - env: - LD_LIBRARY_PATH: "/usr/local/cuda/lib64:/usr/local/cuda/compat:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" - VERBOSE: 1 # Enable verbose build output - CUDA_HOME: "/usr/local/cuda/" # Set CUDA_HOME - CUDA_PATH: "${PATH}" - CUDA_TOOLKIT_ROOT_DIR: "/usr/local/cuda/" # Set CUDA_TOOLKIT_ROOT_DIR - run: | - echo "VERBOSE=1" >> $GITHUB_ENV # Enable verbose build output for troubleshooting - find /usr/ -name 'libcuda.so.*' - find /usr/ -name 'libcudart.so.*' - echo $LD_LIBRARY_PATH - - # Add project-specific and feature flags - CMAKE_ARGS="-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES='75-real;80-real;86-real;87-real;89-real;90-real;100-real;120-real'" - CMAKE_ARGS="-DGGML_CUDA_FORCE_MMQ=on ${CMAKE_ARGS}" - CMAKE_ARGS="${CMAKE_ARGS} -DLLAMA_CURL=off -DLLAMA_OPENSSL=on" - - if [ "${AVXVER}" = "AVX" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" - fi - if [ "${AVXVER}" = "AVX2" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off" - fi - if [ "${AVXVER}" = "AVXVNNI" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX_VNNI=on" - fi - # if [ "${AVXVER}" = "AVX512" ]; then - # CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX512=on" - # fi - # Basic options for compiling without AVX instructions - if [ "${AVXVER}" = "Basic" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX_VNNI=off -DGGML_AVX512=off -DGGML_AVX512_VBMI=off -DGGML_AVX512_VNNI=off -DGGML_AVX512_BF16=off -DGGML_FMA=off -DGGML_F16C=off" - fi - - # Export CMAKE_ARGS environment variable so the python -m build command can use it - echo ${CMAKE_ARGS} - echo "CMAKE_ARGS=${CMAKE_ARGS}" >> $GITHUB_ENV - - # Run the Python build command to generate the wheel - uv pip install build setuptools wheel packaging - CMAKE_ARGS=${CMAKE_ARGS} uv build --wheel - - # --- Post-build steps to get info for rename wheel file and release tag --- - - cuda_ver_short=$(echo "${CUDAVER}" | cut -d'.' -f 1,2 | sed 's/\.//g') - avx_ver=$(echo "${AVXVER}" | tr '[:upper:]' '[:lower:]') - - wheel_path=$(ls dist/*.whl | head -n 1) - filename=$(basename "$wheel_path") - - # Split wheel filename - IFS='-' read -r dist_name version py_tag abi_tag plat_tag <<< "$filename" - - new_version="${version}+cu${cuda_ver_short}.${avx_ver}" - new_filename="${dist_name}-${new_version}-${py_tag}-${abi_tag}-${plat_tag}" - - # Rename wheel file - mv "$wheel_path" "dist/$new_filename" - echo "Renamed wheel to: $new_filename" - - echo "CUDA_VERSION=$cuda_ver_short" >> $GITHUB_ENV # Store short CUDA version in env - echo "TAG_VERSION=$version" >> $GITHUB_ENV # Store version in env for release step - - - name: Get Current Date # Step to get current date for the release tag - id: get-date - run: | - # Get date in YYYYMMDD format using bash date command - currentDate=$(date +%Y%m%d) - # Store the date in environment variable for the release step - echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV - - - uses: softprops/action-gh-release@v3 # Action to create a GitHub Release - with: - files: dist/* # Upload the generated wheel files from the dist directory - # Define the release tag name using the collected environment variables - # Format: v-cu--linux- - tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-${{ env.AVXVER }}-linux-${{ env.BUILD_DATE }} # Release tag format for Linux - # Note: This action will create a new release tag if it doesn't exist, - # or upload assets to an existing tag. Be mindful of potential tag name conflicts. - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use the secret provided by GitHub Actions for authentication \ No newline at end of file diff --git a/.github/workflows/build-wheels-cu131-linux.yml b/.github/workflows/build-wheels-cu131-linux.yml new file mode 100644 index 0000000000..d70f8a01c8 --- /dev/null +++ b/.github/workflows/build-wheels-cu131-linux.yml @@ -0,0 +1,156 @@ +name: Build Wheels (CU131) for Linux + +on: + workflow_dispatch: + +permissions: + contents: write + +jobs: + build_wheels: + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu131 + runs-on: ubuntu-22.04 + container: nvidia/cuda:13.1.2-cudnn-devel-ubuntu22.04 + + strategy: + fail-fast: false + matrix: + os: ["ubuntu-22.04"] + pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] # Python versions + cuda: ["13.1.2"] + cudaarch: ["75-real;80-real;86-real;87-real;89-real;90-real;100-real;120-real;121-real"] + + defaults: + run: + shell: bash + + env: + CUDAVER: ${{ matrix.cuda }} + CUDAARCHVER: ${{ matrix.cudaarch }} + MAX_JOBS: 12 + + steps: + - name: Install dependencies + run: | + apt update + apt install -y \ + build-essential \ + ccache \ + cmake \ + curl \ + git \ + libgomp1 \ + libjpeg-dev \ + libssl-dev \ + ninja-build + + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Install uv and Python ${{ matrix.pyver }} + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.pyver }} + activate-environment: true + enable-cache: true + + - name: Show CUDA version + run: nvcc -V + + - name: Build wheel + env: + LD_LIBRARY_PATH: "/usr/local/cuda/lib64:/usr/local/cuda/compat:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" + VERBOSE: "1" + CUDA_HOME: "/usr/local/cuda" + CUDA_PATH: "/usr/local/cuda" + CUDA_TOOLKIT_ROOT_DIR: "/usr/local/cuda" + run: | + set -euo pipefail + + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" + find /usr/ -name 'libcuda.so.*' || true + find /usr/ -name 'libcudart.so.*' || true + + cuda_ver_short=$(echo "${CUDAVER}" | cut -d'.' -f 1,2 | sed 's/\.//g') + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend shared libraries. + # - GGML_CPU_ALL_VARIANTS builds CPU variant backends when supported. + # - GGML_NATIVE=OFF avoids binding the wheel to the CI runner CPU. + CMAKE_ARGS_ARRAY=( + "-G Ninja" + + # Disable non-wheel targets. + "-DLLAMA_BUILD_EXAMPLES=OFF" + "-DLLAMA_BUILD_TESTS=OFF" + "-DLLAMA_BUILD_TOOLS=OFF" + "-DLLAMA_BUILD_SERVER=OFF" + "-DLLAMA_BUILD_UI=OFF" + "-DLLAMA_USE_PREBUILT_UI=OFF" + "-DLLAMA_CURL=OFF" + "-DLLAMA_OPENSSL=ON" + + # GGML dynamic backend layout. + "-DGGML_CPU=ON" + "-DGGML_CUDA=ON" + "-DGGML_NATIVE=OFF" + "-DGGML_BACKEND_DL=ON" + "-DGGML_CPU_ALL_VARIANTS=ON" + "-DGGML_OPENMP=ON" + + # CUDA backend. + "-DCMAKE_CUDA_ARCHITECTURES=${CUDAARCHVER}" + "-DGGML_CUDA_FORCE_MMQ=ON" + "-DCUDA_SEPARABLE_COMPILATION=ON" + "-DCMAKE_CUDA_FLAGS=--diag-suppress=177,221,550" + + # Build behavior. + "-DCMAKE_BUILD_PARALLEL_LEVEL=${MAX_JOBS}" + "-DGGML_CCACHE=ON" + "-DENABLE_CCACHE=ON" + ) + + CMAKE_ARGS="${CMAKE_ARGS_ARRAY[*]}" + echo "CMAKE_ARGS=${CMAKE_ARGS}" + + uv pip install --upgrade build setuptools wheel packaging + CMAKE_ARGS="${CMAKE_ARGS}" uv build --wheel + + if ! ls dist/*.whl >/dev/null 2>&1; then + echo "No wheel built in dist/ directory" + exit 1 + fi + + wheel_path=$(ls dist/*.whl | head -n 1) + filename=$(basename "$wheel_path") + + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl + IFS='-' read -r dist_name version py_tag abi_tag plat_tag <<< "$filename" + + # CPU all-variants is now an internal runtime layout detail. + new_version="${version}+cu${cuda_ver_short}" + new_filename="${dist_name}-${new_version}-${py_tag}-${abi_tag}-${plat_tag}" + + mv "$wheel_path" "dist/$new_filename" + echo "Renamed wheel to: $new_filename" + + echo "CUDA_VERSION=$cuda_ver_short" >> "$GITHUB_ENV" + echo "TAG_VERSION=$version" >> "$GITHUB_ENV" + + - name: Get current date + id: get-date + run: | + currentDate=$(date +%Y%m%d) + echo "BUILD_DATE=$currentDate" >> "$GITHUB_ENV" + + - name: Create release + if: always() && env.TAG_VERSION != '' + uses: softprops/action-gh-release@v3 + with: + files: dist/* + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-linux-${{ env.BUILD_DATE }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 42891efb70bfd8ac6dd438e99f6c2a1b4119299d Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 19:39:57 +0800 Subject: [PATCH 106/304] ci(cu12+linux): build CU124/126/128 wheels with GGML dynamic backends for linux - Replace the old CPU/AVX release tag matrix with a single CU124/126/128 backend wheel layout. - Enable `GGML_BACKEND_DL` and `GGML_CPU_ALL_VARIANTS` so Linux wheels ship runtime-loadable GGML backend DLLs and CPU variant backends. - Disable non-wheel targets such as examples, tests, tools, server, embedded UI, and curl. - Remove the `.basic` style local version suffix and publish wheels as `+cu124/cu126/cu128`. Signed-off-by: JamePeng --- .../workflows/build-wheels-cu124-linux.yml | 170 ++++++++++-------- .../workflows/build-wheels-cu126-linux.yml | 170 ++++++++++-------- .../workflows/build-wheels-cu128-linux.yml | 170 ++++++++++-------- 3 files changed, 291 insertions(+), 219 deletions(-) diff --git a/.github/workflows/build-wheels-cu124-linux.yml b/.github/workflows/build-wheels-cu124-linux.yml index 889a1679a4..d7a3a90d81 100644 --- a/.github/workflows/build-wheels-cu124-linux.yml +++ b/.github/workflows/build-wheels-cu124-linux.yml @@ -1,23 +1,24 @@ -name: Build Wheels(CU124) for Linux # Workflow name +name: Build Wheels (CU124) for Linux on: - workflow_dispatch: # Manual trigger + workflow_dispatch: permissions: contents: write jobs: build_wheels: - name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag == 'wheels' && 'AVX2' || matrix.releasetag }} + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu124 runs-on: ubuntu-22.04 container: nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 + strategy: - matrix: # Define the build matrix directly here + fail-fast: false + matrix: os: ["ubuntu-22.04"] pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] # Python versions cuda: ["12.4.1"] - releasetag: ["Basic"] # Controls CMAKE_ARGS for CPU features (even in CUDA build) - cudaarch: ["all"] # Controls target CUDA architectures for nvcc + cudaarch: ["70-real;75-real;80-real;86-real;87-real;89-real"] defaults: run: @@ -25,108 +26,131 @@ jobs: env: CUDAVER: ${{ matrix.cuda }} - AVXVER: ${{ matrix.releasetag }} CUDAARCHVER: ${{ matrix.cudaarch }} + MAX_JOBS: 12 steps: - name: Install dependencies run: | - apt update - apt install -y build-essential ccache cmake curl git libgomp1 libjpeg-dev libssl-dev - - - uses: actions/checkout@v6 # Checkout code + apt update + apt install -y \ + build-essential \ + ccache \ + cmake \ + curl \ + git \ + libgomp1 \ + libjpeg-dev \ + libssl-dev \ + ninja-build + + - name: Checkout + uses: actions/checkout@v6 with: - submodules: "recursive" + submodules: recursive - # from astral-sh/setup-uv - - name: Install the latest version of uv and set the python version + - name: Install uv and Python ${{ matrix.pyver }} uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true enable-cache: true - - run: nvcc -V + - name: Show CUDA version + run: nvcc -V - - name: Build Wheel With Cmake # Main build step: configures and builds the wheel + - name: Build wheel env: LD_LIBRARY_PATH: "/usr/local/cuda/lib64:/usr/local/cuda/compat:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" - VERBOSE: 1 # Enable verbose build output - CUDA_HOME: "/usr/local/cuda/" # Set CUDA_HOME - CUDA_PATH: "${PATH}" - CUDA_TOOLKIT_ROOT_DIR: "/usr/local/cuda/" # Set CUDA_TOOLKIT_ROOT_DIR + VERBOSE: "1" + CUDA_HOME: "/usr/local/cuda" + CUDA_PATH: "/usr/local/cuda" + CUDA_TOOLKIT_ROOT_DIR: "/usr/local/cuda" run: | - echo "VERBOSE=1" >> $GITHUB_ENV # Enable verbose build output for troubleshooting - find /usr/ -name 'libcuda.so.*' - find /usr/ -name 'libcudart.so.*' - echo $LD_LIBRARY_PATH - - # Add project-specific and feature flags - CMAKE_ARGS="-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES='70-real;75-real;80-real;86-real;87-real;89-real'" - CMAKE_ARGS="-DGGML_CUDA_FORCE_MMQ=on ${CMAKE_ARGS}" - CMAKE_ARGS="${CMAKE_ARGS} -DLLAMA_CURL=off -DLLAMA_OPENSSL=on" - - if [ "${AVXVER}" = "AVX" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" - fi - if [ "${AVXVER}" = "AVX2" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off" - fi - if [ "${AVXVER}" = "AVXVNNI" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX_VNNI=on" - fi - # if [ "${AVXVER}" = "AVX512" ]; then - # CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX512=on" - # fi - # Basic options for compiling without AVX instructions - if [ "${AVXVER}" = "Basic" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX_VNNI=off -DGGML_AVX512=off -DGGML_AVX512_VBMI=off -DGGML_AVX512_VNNI=off -DGGML_AVX512_BF16=off -DGGML_FMA=off -DGGML_F16C=off" - fi - - # Export CMAKE_ARGS environment variable so the python -m build command can use it - echo ${CMAKE_ARGS} - echo "CMAKE_ARGS=${CMAKE_ARGS}" >> $GITHUB_ENV - - # Run the Python build command to generate the wheel - uv pip install build setuptools wheel packaging - CMAKE_ARGS=${CMAKE_ARGS} uv build --wheel + set -euo pipefail - # --- Post-build steps to get info for rename wheel file and release tag --- + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" + find /usr/ -name 'libcuda.so.*' || true + find /usr/ -name 'libcudart.so.*' || true cuda_ver_short=$(echo "${CUDAVER}" | cut -d'.' -f 1,2 | sed 's/\.//g') - avx_ver=$(echo "${AVXVER}" | tr '[:upper:]' '[:lower:]') + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend shared libraries. + # - GGML_CPU_ALL_VARIANTS builds CPU variant backends when supported. + # - GGML_NATIVE=OFF avoids binding the wheel to the CI runner CPU. + CMAKE_ARGS_ARRAY=( + "-G Ninja" + + # Disable non-wheel targets. + "-DLLAMA_BUILD_EXAMPLES=OFF" + "-DLLAMA_BUILD_TESTS=OFF" + "-DLLAMA_BUILD_TOOLS=OFF" + "-DLLAMA_BUILD_SERVER=OFF" + "-DLLAMA_BUILD_UI=OFF" + "-DLLAMA_USE_PREBUILT_UI=OFF" + "-DLLAMA_CURL=OFF" + "-DLLAMA_OPENSSL=ON" + + # GGML dynamic backend layout. + "-DGGML_CPU=ON" + "-DGGML_CUDA=ON" + "-DGGML_NATIVE=OFF" + "-DGGML_BACKEND_DL=ON" + "-DGGML_CPU_ALL_VARIANTS=ON" + "-DGGML_OPENMP=ON" + + # CUDA backend. + "-DCMAKE_CUDA_ARCHITECTURES=${CUDAARCHVER}" + "-DGGML_CUDA_FORCE_MMQ=ON" + "-DCUDA_SEPARABLE_COMPILATION=ON" + "-DCMAKE_CUDA_FLAGS=--diag-suppress=177,221,550" + + # Build behavior. + "-DCMAKE_BUILD_PARALLEL_LEVEL=${MAX_JOBS}" + "-DGGML_CCACHE=ON" + "-DENABLE_CCACHE=ON" + ) + + CMAKE_ARGS="${CMAKE_ARGS_ARRAY[*]}" + echo "CMAKE_ARGS=${CMAKE_ARGS}" + + uv pip install --upgrade build setuptools wheel packaging + CMAKE_ARGS="${CMAKE_ARGS}" uv build --wheel + + if ! ls dist/*.whl >/dev/null 2>&1; then + echo "No wheel built in dist/ directory" + exit 1 + fi wheel_path=$(ls dist/*.whl | head -n 1) filename=$(basename "$wheel_path") - # Split wheel filename + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl IFS='-' read -r dist_name version py_tag abi_tag plat_tag <<< "$filename" - new_version="${version}+cu${cuda_ver_short}.${avx_ver}" + # CPU all-variants is now an internal runtime layout detail. + new_version="${version}+cu${cuda_ver_short}" new_filename="${dist_name}-${new_version}-${py_tag}-${abi_tag}-${plat_tag}" - # Rename wheel file mv "$wheel_path" "dist/$new_filename" echo "Renamed wheel to: $new_filename" - echo "CUDA_VERSION=$cuda_ver_short" >> $GITHUB_ENV # Store short CUDA version in env - echo "TAG_VERSION=$version" >> $GITHUB_ENV # Store version in env for release step + echo "CUDA_VERSION=$cuda_ver_short" >> "$GITHUB_ENV" + echo "TAG_VERSION=$version" >> "$GITHUB_ENV" - - name: Get Current Date # Step to get current date for the release tag + - name: Get current date id: get-date run: | - # Get date in YYYYMMDD format using bash date command currentDate=$(date +%Y%m%d) - # Store the date in environment variable for the release step - echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV + echo "BUILD_DATE=$currentDate" >> "$GITHUB_ENV" - - uses: softprops/action-gh-release@v3 # Action to create a GitHub Release + - name: Create release + if: always() && env.TAG_VERSION != '' + uses: softprops/action-gh-release@v3 with: - files: dist/* # Upload the generated wheel files from the dist directory - # Define the release tag name using the collected environment variables - # Format: v-cu--linux- - tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-${{ env.AVXVER }}-linux-${{ env.BUILD_DATE }} # Release tag format for Linux - # Note: This action will create a new release tag if it doesn't exist, - # or upload assets to an existing tag. Be mindful of potential tag name conflicts. + files: dist/* + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-linux-${{ env.BUILD_DATE }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use the secret provided by GitHub Actions for authentication \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-wheels-cu126-linux.yml b/.github/workflows/build-wheels-cu126-linux.yml index 568824c642..9f28a57ca2 100644 --- a/.github/workflows/build-wheels-cu126-linux.yml +++ b/.github/workflows/build-wheels-cu126-linux.yml @@ -1,23 +1,24 @@ -name: Build Wheels(CU126) for Linux # Workflow name +name: Build Wheels (CU126) for Linux on: - workflow_dispatch: # Manual trigger + workflow_dispatch: permissions: contents: write jobs: build_wheels: - name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag == 'wheels' && 'AVX2' || matrix.releasetag }} + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu126 runs-on: ubuntu-22.04 container: nvidia/cuda:12.6.3-cudnn-devel-ubuntu22.04 + strategy: - matrix: # Define the build matrix directly here + fail-fast: false + matrix: os: ["ubuntu-22.04"] pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] # Python versions cuda: ["12.6.3"] - releasetag: ["Basic"] # Controls CMAKE_ARGS for CPU features (even in CUDA build) - cudaarch: ["all"] # Controls target CUDA architectures for nvcc + cudaarch: ["70-real;75-real;80-real;86-real;87-real;89-real"] defaults: run: @@ -25,108 +26,131 @@ jobs: env: CUDAVER: ${{ matrix.cuda }} - AVXVER: ${{ matrix.releasetag }} CUDAARCHVER: ${{ matrix.cudaarch }} + MAX_JOBS: 12 steps: - name: Install dependencies run: | - apt update - apt install -y build-essential ccache cmake curl git libgomp1 libjpeg-dev libssl-dev - - - uses: actions/checkout@v6 # Checkout code + apt update + apt install -y \ + build-essential \ + ccache \ + cmake \ + curl \ + git \ + libgomp1 \ + libjpeg-dev \ + libssl-dev \ + ninja-build + + - name: Checkout + uses: actions/checkout@v6 with: - submodules: "recursive" + submodules: recursive - # from astral-sh/setup-uv - - name: Install the latest version of uv and set the python version + - name: Install uv and Python ${{ matrix.pyver }} uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true enable-cache: true - - run: nvcc -V + - name: Show CUDA version + run: nvcc -V - - name: Build Wheel With Cmake # Main build step: configures and builds the wheel + - name: Build wheel env: LD_LIBRARY_PATH: "/usr/local/cuda/lib64:/usr/local/cuda/compat:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" - VERBOSE: 1 # Enable verbose build output - CUDA_HOME: "/usr/local/cuda/" # Set CUDA_HOME - CUDA_PATH: "${PATH}" - CUDA_TOOLKIT_ROOT_DIR: "/usr/local/cuda/" # Set CUDA_TOOLKIT_ROOT_DIR + VERBOSE: "1" + CUDA_HOME: "/usr/local/cuda" + CUDA_PATH: "/usr/local/cuda" + CUDA_TOOLKIT_ROOT_DIR: "/usr/local/cuda" run: | - echo "VERBOSE=1" >> $GITHUB_ENV # Enable verbose build output for troubleshooting - find /usr/ -name 'libcuda.so.*' - find /usr/ -name 'libcudart.so.*' - echo $LD_LIBRARY_PATH - - # Add project-specific and feature flags - CMAKE_ARGS="-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES='70-real;75-real;80-real;86-real;87-real;89-real'" - CMAKE_ARGS="-DGGML_CUDA_FORCE_MMQ=on ${CMAKE_ARGS}" - CMAKE_ARGS="${CMAKE_ARGS} -DLLAMA_CURL=off -DLLAMA_OPENSSL=on" - - if [ "${AVXVER}" = "AVX" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" - fi - if [ "${AVXVER}" = "AVX2" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off" - fi - if [ "${AVXVER}" = "AVXVNNI" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX_VNNI=on" - fi - # if [ "${AVXVER}" = "AVX512" ]; then - # CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX512=on" - # fi - # Basic options for compiling without AVX instructions - if [ "${AVXVER}" = "Basic" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX_VNNI=off -DGGML_AVX512=off -DGGML_AVX512_VBMI=off -DGGML_AVX512_VNNI=off -DGGML_AVX512_BF16=off -DGGML_FMA=off -DGGML_F16C=off" - fi - - # Export CMAKE_ARGS environment variable so the python -m build command can use it - echo ${CMAKE_ARGS} - echo "CMAKE_ARGS=${CMAKE_ARGS}" >> $GITHUB_ENV - - # Run the Python build command to generate the wheel - uv pip install build setuptools wheel packaging - CMAKE_ARGS=${CMAKE_ARGS} uv build --wheel + set -euo pipefail - # --- Post-build steps to get info for rename wheel file and release tag --- + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" + find /usr/ -name 'libcuda.so.*' || true + find /usr/ -name 'libcudart.so.*' || true cuda_ver_short=$(echo "${CUDAVER}" | cut -d'.' -f 1,2 | sed 's/\.//g') - avx_ver=$(echo "${AVXVER}" | tr '[:upper:]' '[:lower:]') + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend shared libraries. + # - GGML_CPU_ALL_VARIANTS builds CPU variant backends when supported. + # - GGML_NATIVE=OFF avoids binding the wheel to the CI runner CPU. + CMAKE_ARGS_ARRAY=( + "-G Ninja" + + # Disable non-wheel targets. + "-DLLAMA_BUILD_EXAMPLES=OFF" + "-DLLAMA_BUILD_TESTS=OFF" + "-DLLAMA_BUILD_TOOLS=OFF" + "-DLLAMA_BUILD_SERVER=OFF" + "-DLLAMA_BUILD_UI=OFF" + "-DLLAMA_USE_PREBUILT_UI=OFF" + "-DLLAMA_CURL=OFF" + "-DLLAMA_OPENSSL=ON" + + # GGML dynamic backend layout. + "-DGGML_CPU=ON" + "-DGGML_CUDA=ON" + "-DGGML_NATIVE=OFF" + "-DGGML_BACKEND_DL=ON" + "-DGGML_CPU_ALL_VARIANTS=ON" + "-DGGML_OPENMP=ON" + + # CUDA backend. + "-DCMAKE_CUDA_ARCHITECTURES=${CUDAARCHVER}" + "-DGGML_CUDA_FORCE_MMQ=ON" + "-DCUDA_SEPARABLE_COMPILATION=ON" + "-DCMAKE_CUDA_FLAGS=--diag-suppress=177,221,550" + + # Build behavior. + "-DCMAKE_BUILD_PARALLEL_LEVEL=${MAX_JOBS}" + "-DGGML_CCACHE=ON" + "-DENABLE_CCACHE=ON" + ) + + CMAKE_ARGS="${CMAKE_ARGS_ARRAY[*]}" + echo "CMAKE_ARGS=${CMAKE_ARGS}" + + uv pip install --upgrade build setuptools wheel packaging + CMAKE_ARGS="${CMAKE_ARGS}" uv build --wheel + + if ! ls dist/*.whl >/dev/null 2>&1; then + echo "No wheel built in dist/ directory" + exit 1 + fi wheel_path=$(ls dist/*.whl | head -n 1) filename=$(basename "$wheel_path") - # Split wheel filename + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl IFS='-' read -r dist_name version py_tag abi_tag plat_tag <<< "$filename" - new_version="${version}+cu${cuda_ver_short}.${avx_ver}" + # CPU all-variants is now an internal runtime layout detail. + new_version="${version}+cu${cuda_ver_short}" new_filename="${dist_name}-${new_version}-${py_tag}-${abi_tag}-${plat_tag}" - # Rename wheel file mv "$wheel_path" "dist/$new_filename" echo "Renamed wheel to: $new_filename" - echo "CUDA_VERSION=$cuda_ver_short" >> $GITHUB_ENV # Store short CUDA version in env - echo "TAG_VERSION=$version" >> $GITHUB_ENV # Store version in env for release step + echo "CUDA_VERSION=$cuda_ver_short" >> "$GITHUB_ENV" + echo "TAG_VERSION=$version" >> "$GITHUB_ENV" - - name: Get Current Date # Step to get current date for the release tag + - name: Get current date id: get-date run: | - # Get date in YYYYMMDD format using bash date command currentDate=$(date +%Y%m%d) - # Store the date in environment variable for the release step - echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV + echo "BUILD_DATE=$currentDate" >> "$GITHUB_ENV" - - uses: softprops/action-gh-release@v3 # Action to create a GitHub Release + - name: Create release + if: always() && env.TAG_VERSION != '' + uses: softprops/action-gh-release@v3 with: - files: dist/* # Upload the generated wheel files from the dist directory - # Define the release tag name using the collected environment variables - # Format: v-cu--linux- - tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-${{ env.AVXVER }}-linux-${{ env.BUILD_DATE }} # Release tag format for Linux - # Note: This action will create a new release tag if it doesn't exist, - # or upload assets to an existing tag. Be mindful of potential tag name conflicts. + files: dist/* + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-linux-${{ env.BUILD_DATE }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use the secret provided by GitHub Actions for authentication \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-wheels-cu128-linux.yml b/.github/workflows/build-wheels-cu128-linux.yml index d1c387c52a..c6b255c9f9 100644 --- a/.github/workflows/build-wheels-cu128-linux.yml +++ b/.github/workflows/build-wheels-cu128-linux.yml @@ -1,23 +1,24 @@ -name: Build Wheels(CU128) for Linux # Workflow name +name: Build Wheels (CU128) for Linux on: - workflow_dispatch: # Manual trigger + workflow_dispatch: permissions: contents: write jobs: build_wheels: - name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag == 'wheels' && 'AVX2' || matrix.releasetag }} + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu128 runs-on: ubuntu-22.04 container: nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 + strategy: - matrix: # Define the build matrix directly here + fail-fast: false + matrix: os: ["ubuntu-22.04"] pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] # Python versions cuda: ["12.8.1"] - releasetag: ["Basic"] # Controls CMAKE_ARGS for CPU features (even in CUDA build) - cudaarch: ["all"] # Controls target CUDA architectures for nvcc + cudaarch: ["75-real;80-real;86-real;87-real;89-real;90-real;100-real;120-real"] defaults: run: @@ -25,108 +26,131 @@ jobs: env: CUDAVER: ${{ matrix.cuda }} - AVXVER: ${{ matrix.releasetag }} CUDAARCHVER: ${{ matrix.cudaarch }} + MAX_JOBS: 12 steps: - name: Install dependencies run: | - apt update - apt install -y build-essential ccache cmake curl git libgomp1 libjpeg-dev libssl-dev - - - uses: actions/checkout@v6 # Checkout code + apt update + apt install -y \ + build-essential \ + ccache \ + cmake \ + curl \ + git \ + libgomp1 \ + libjpeg-dev \ + libssl-dev \ + ninja-build + + - name: Checkout + uses: actions/checkout@v6 with: - submodules: "recursive" + submodules: recursive - # from astral-sh/setup-uv - - name: Install the latest version of uv and set the python version + - name: Install uv and Python ${{ matrix.pyver }} uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.pyver }} activate-environment: true enable-cache: true - - run: nvcc -V + - name: Show CUDA version + run: nvcc -V - - name: Build Wheel With Cmake # Main build step: configures and builds the wheel + - name: Build wheel env: LD_LIBRARY_PATH: "/usr/local/cuda/lib64:/usr/local/cuda/compat:/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}" - VERBOSE: 1 # Enable verbose build output - CUDA_HOME: "/usr/local/cuda/" # Set CUDA_HOME - CUDA_PATH: "${PATH}" - CUDA_TOOLKIT_ROOT_DIR: "/usr/local/cuda/" # Set CUDA_TOOLKIT_ROOT_DIR + VERBOSE: "1" + CUDA_HOME: "/usr/local/cuda" + CUDA_PATH: "/usr/local/cuda" + CUDA_TOOLKIT_ROOT_DIR: "/usr/local/cuda" run: | - echo "VERBOSE=1" >> $GITHUB_ENV # Enable verbose build output for troubleshooting - find /usr/ -name 'libcuda.so.*' - find /usr/ -name 'libcudart.so.*' - echo $LD_LIBRARY_PATH - - # Add project-specific and feature flags - CMAKE_ARGS="-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES='75-real;80-real;86-real;87-real;89-real;90-real;100-real;101-real;120-real'" - CMAKE_ARGS="-DGGML_CUDA_FORCE_MMQ=on ${CMAKE_ARGS}" - CMAKE_ARGS="${CMAKE_ARGS} -DLLAMA_CURL=off -DLLAMA_OPENSSL=on" - - if [ "${AVXVER}" = "AVX" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off" - fi - if [ "${AVXVER}" = "AVX2" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off" - fi - if [ "${AVXVER}" = "AVXVNNI" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX_VNNI=on" - fi - # if [ "${AVXVER}" = "AVX512" ]; then - # CMAKE_ARGS="${CMAKE_ARGS} -DGGML_AVX512=on" - # fi - # Basic options for compiling without AVX instructions - if [ "${AVXVER}" = "Basic" ]; then - CMAKE_ARGS="${CMAKE_ARGS} -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX_VNNI=off -DGGML_AVX512=off -DGGML_AVX512_VBMI=off -DGGML_AVX512_VNNI=off -DGGML_AVX512_BF16=off -DGGML_FMA=off -DGGML_F16C=off" - fi - - # Export CMAKE_ARGS environment variable so the python -m build command can use it - echo ${CMAKE_ARGS} - echo "CMAKE_ARGS=${CMAKE_ARGS}" >> $GITHUB_ENV - - # Run the Python build command to generate the wheel - uv pip install build setuptools wheel packaging - CMAKE_ARGS=${CMAKE_ARGS} uv build --wheel + set -euo pipefail - # --- Post-build steps to get info for rename wheel file and release tag --- + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" + find /usr/ -name 'libcuda.so.*' || true + find /usr/ -name 'libcudart.so.*' || true cuda_ver_short=$(echo "${CUDAVER}" | cut -d'.' -f 1,2 | sed 's/\.//g') - avx_ver=$(echo "${AVXVER}" | tr '[:upper:]' '[:lower:]') + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend shared libraries. + # - GGML_CPU_ALL_VARIANTS builds CPU variant backends when supported. + # - GGML_NATIVE=OFF avoids binding the wheel to the CI runner CPU. + CMAKE_ARGS_ARRAY=( + "-G Ninja" + + # Disable non-wheel targets. + "-DLLAMA_BUILD_EXAMPLES=OFF" + "-DLLAMA_BUILD_TESTS=OFF" + "-DLLAMA_BUILD_TOOLS=OFF" + "-DLLAMA_BUILD_SERVER=OFF" + "-DLLAMA_BUILD_UI=OFF" + "-DLLAMA_USE_PREBUILT_UI=OFF" + "-DLLAMA_CURL=OFF" + "-DLLAMA_OPENSSL=ON" + + # GGML dynamic backend layout. + "-DGGML_CPU=ON" + "-DGGML_CUDA=ON" + "-DGGML_NATIVE=OFF" + "-DGGML_BACKEND_DL=ON" + "-DGGML_CPU_ALL_VARIANTS=ON" + "-DGGML_OPENMP=ON" + + # CUDA backend. + "-DCMAKE_CUDA_ARCHITECTURES=${CUDAARCHVER}" + "-DGGML_CUDA_FORCE_MMQ=ON" + "-DCUDA_SEPARABLE_COMPILATION=ON" + "-DCMAKE_CUDA_FLAGS=--diag-suppress=177,221,550" + + # Build behavior. + "-DCMAKE_BUILD_PARALLEL_LEVEL=${MAX_JOBS}" + "-DGGML_CCACHE=ON" + "-DENABLE_CCACHE=ON" + ) + + CMAKE_ARGS="${CMAKE_ARGS_ARRAY[*]}" + echo "CMAKE_ARGS=${CMAKE_ARGS}" + + uv pip install --upgrade build setuptools wheel packaging + CMAKE_ARGS="${CMAKE_ARGS}" uv build --wheel + + if ! ls dist/*.whl >/dev/null 2>&1; then + echo "No wheel built in dist/ directory" + exit 1 + fi wheel_path=$(ls dist/*.whl | head -n 1) filename=$(basename "$wheel_path") - # Split wheel filename + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl IFS='-' read -r dist_name version py_tag abi_tag plat_tag <<< "$filename" - new_version="${version}+cu${cuda_ver_short}.${avx_ver}" + # CPU all-variants is now an internal runtime layout detail. + new_version="${version}+cu${cuda_ver_short}" new_filename="${dist_name}-${new_version}-${py_tag}-${abi_tag}-${plat_tag}" - # Rename wheel file mv "$wheel_path" "dist/$new_filename" echo "Renamed wheel to: $new_filename" - echo "CUDA_VERSION=$cuda_ver_short" >> $GITHUB_ENV # Store short CUDA version in env - echo "TAG_VERSION=$version" >> $GITHUB_ENV # Store version in env for release step + echo "CUDA_VERSION=$cuda_ver_short" >> "$GITHUB_ENV" + echo "TAG_VERSION=$version" >> "$GITHUB_ENV" - - name: Get Current Date # Step to get current date for the release tag + - name: Get current date id: get-date run: | - # Get date in YYYYMMDD format using bash date command currentDate=$(date +%Y%m%d) - # Store the date in environment variable for the release step - echo "BUILD_DATE=$currentDate" >> $GITHUB_ENV + echo "BUILD_DATE=$currentDate" >> "$GITHUB_ENV" - - uses: softprops/action-gh-release@v3 # Action to create a GitHub Release + - name: Create release + if: always() && env.TAG_VERSION != '' + uses: softprops/action-gh-release@v3 with: - files: dist/* # Upload the generated wheel files from the dist directory - # Define the release tag name using the collected environment variables - # Format: v-cu--linux- - tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-${{ env.AVXVER }}-linux-${{ env.BUILD_DATE }} # Release tag format for Linux - # Note: This action will create a new release tag if it doesn't exist, - # or upload assets to an existing tag. Be mindful of potential tag name conflicts. + files: dist/* + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-linux-${{ env.BUILD_DATE }} env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use the secret provided by GitHub Actions for authentication \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6e32ef0171b075605fa527181b924a816a237b77 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 19:42:56 +0800 Subject: [PATCH 107/304] ci: Remove outdated workflow Signed-off-by: JamePeng --- .github/workflows/build-and-release.yaml | 145 ----------------------- .github/workflows/build-docker.yaml | 50 -------- 2 files changed, 195 deletions(-) delete mode 100644 .github/workflows/build-and-release.yaml delete mode 100644 .github/workflows/build-docker.yaml diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml deleted file mode 100644 index 7eaf017fbc..0000000000 --- a/.github/workflows/build-and-release.yaml +++ /dev/null @@ -1,145 +0,0 @@ -name: Build Release - -on: workflow_dispatch - -permissions: - contents: write - -jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-22.04, windows-2022, macos-14, macos-15] - - steps: - - uses: actions/checkout@v4 - with: - submodules: "recursive" - - # Used to host cibuildwheel - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install uv - RUST_LOG=trace python -m uv pip install -e .[all] --verbose - shell: bash - - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - env: - RUST_LOG: trace - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - shell: cmd - - - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 - env: - # disable repair - CIBW_REPAIR_WHEEL_COMMAND: "" - with: - package-dir: . - output-dir: wheelhouse - - - uses: actions/upload-artifact@v4 - with: - name: wheels-${{ matrix.os }} - path: ./wheelhouse/*.whl - - build_wheels_arm64: - name: Build arm64 wheels - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: linux/arm64 - - - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 - env: - CIBW_SKIP: "*musllinux* pp*" - CIBW_REPAIR_WHEEL_COMMAND: "" - CIBW_ARCHS: "aarch64" - CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DCMAKE_CROSSCOMPILING=ON" - CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*" - with: - output-dir: wheelhouse - - - name: Upload wheels as artifacts - uses: actions/upload-artifact@v4 - with: - name: wheels_arm64 - path: ./wheelhouse/*.whl - - build_sdist: - name: Build source distribution - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - uses: actions/setup-python@v5 - with: - python-version: "3.9" - - - name: Install dependencies (Linux/MacOS) - if: runner.os != 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install uv - RUST_LOG=trace python -m uv pip install -e .[all] --verbose - python -m uv pip install build - shell: bash - - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - env: - RUST_LOG: trace - run: | - python -m pip install --upgrade pip - python -m pip install uv - python -m uv pip install -e .[all] --verbose - python -m uv pip install build - shell: cmd - - - name: Build source distribution - run: | - python -m build --sdist - - - uses: actions/upload-artifact@v4 - with: - name: sdist - path: ./dist/*.tar.gz - - release: - name: Release - needs: [build_wheels, build_wheels_arm64, build_sdist] - runs-on: ubuntu-latest - - steps: - - uses: actions/download-artifact@v4 - with: - merge-multiple: true - path: dist - - - uses: softprops/action-gh-release@v2 - with: - files: dist/* - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml deleted file mode 100644 index b290f6273f..0000000000 --- a/.github/workflows/build-docker.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build Docker - -on: workflow_dispatch - -permissions: - contents: write - packages: write - -jobs: - docker: - name: Build and push Docker image - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v6 - with: - context: . - file: "docker/simple/Dockerfile" - push: ${{ startsWith(github.ref, 'refs/tags/') }} - pull: true - platforms: linux/amd64,linux/arm64 - tags: | - ghcr.io/abetlen/llama-cpp-python:latest - ghcr.io/abetlen/llama-cpp-python:${{ github.ref_name }} - build-args: | - BUILDKIT_INLINE_CACHE=1 - - - name: Publish to GitHub Tag - if: steps.docker_build.outputs.digest && startsWith(github.ref, 'refs/tags/') - run: | - echo "Docker image published for tag: ${{ github.ref_name }}" From b8d69f29e71b74973a7fd295e32d9e9d86908f5d Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 22:11:42 +0800 Subject: [PATCH 108/304] Update .gitmodules submodule git addr Signed-off-by: JamePeng --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 7edf0975dc..f56cca32df 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "vendor/llama.cpp"] path = vendor/llama.cpp - url = https://github.com/ggerganov/llama.cpp.git + url = https://github.com/ggml-org/llama.cpp.git From ec580cb1c3b0ef946523979dca63df5c2d0483cc Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 22:23:49 +0800 Subject: [PATCH 109/304] chore(docs): remove outdated mkdocs workflow - Transition documentation focus to the repository wiki and /docs/wiki. - Clean up and remove all unnecessary mkdocs-related configuration files. Signed-off-by: JamePeng --- .readthedocs.yaml | 24 ------------ docs/api-reference.md | 88 ------------------------------------------- docs/changelog.md | 1 - docs/index.md | 5 --- docs/requirements.txt | 3 -- mkdocs.yml | 74 ------------------------------------ pyproject.toml | 14 ++----- 7 files changed, 3 insertions(+), 206 deletions(-) delete mode 100644 .readthedocs.yaml delete mode 100644 docs/api-reference.md delete mode 100644 docs/changelog.md delete mode 100644 docs/index.md delete mode 100644 docs/requirements.txt delete mode 100644 mkdocs.yml diff --git a/.readthedocs.yaml b/.readthedocs.yaml deleted file mode 100644 index ff3e950cd1..0000000000 --- a/.readthedocs.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Read the Docs configuration file for MkDocs projects -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -# Set the version of Python and other tools you might need -build: - os: ubuntu-22.04 - tools: - python: "3.11" - -mkdocs: - configuration: mkdocs.yml - -python: - install: - - method: pip - path: . - - requirements: docs/requirements.txt - -submodules: - include: all - recursive: true \ No newline at end of file diff --git a/docs/api-reference.md b/docs/api-reference.md deleted file mode 100644 index ab51ef754e..0000000000 --- a/docs/api-reference.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -title: API Reference ---- - -## High Level API - -High-level Python bindings for llama.cpp. - -::: llama_cpp.Llama - options: - members: - - __init__ - - tokenize - - detokenize - - reset - - eval - - sample - - generate - - create_embedding - - embed - - create_completion - - __call__ - - create_chat_completion - - create_chat_completion_openai_v1 - - set_cache - - save_state - - load_state - - token_bos - - token_eos - - from_pretrained - show_root_heading: true - -::: llama_cpp.LlamaGrammar - options: - members: - - from_string - - from_json_schema - -::: llama_cpp.LlamaCache - options: - show_root_heading: true - -::: llama_cpp.LlamaState - options: - show_root_heading: true - -::: llama_cpp.LogitsProcessor - options: - show_root_heading: true - -::: llama_cpp.LogitsProcessorList - options: - show_root_heading: true - -::: llama_cpp.StoppingCriteria - options: - show_root_heading: true - -::: llama_cpp.StoppingCriteriaList - options: - show_root_heading: true - -## Low Level API - -Low-level Python bindings for llama.cpp using Python's ctypes library. - -::: llama_cpp.llama_cpp - options: - show_if_no_docstring: true - # filter only members starting with `llama_` - filters: - - "^llama_" - -::: llama_cpp.llama_cpp - options: - show_if_no_docstring: true - show_root_heading: false - show_root_toc_entry: false - heading_level: 4 - # filter only members starting with `LLAMA_` - filters: - - "^LLAMA_" - -## Misc - -::: llama_cpp.llama_types - options: - show_if_no_docstring: true \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md deleted file mode 100644 index 047bc14424..0000000000 --- a/docs/changelog.md +++ /dev/null @@ -1 +0,0 @@ --8<- "CHANGELOG.md" \ No newline at end of file diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 60bc7aef42..0000000000 --- a/docs/index.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Getting Started ---- - --8<- "README.md" \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 199bd4ffbf..0000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -mkdocs -mkdocs-material -mkdocstrings[python] \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index 79a9e67a1a..0000000000 --- a/mkdocs.yml +++ /dev/null @@ -1,74 +0,0 @@ -site_name: llama-cpp-python -repo_url: https://github.com/abetlen/llama-cpp-python - -theme: - name: material - palette: - - # Palette toggle for light mode - - scheme: default - primary: indigo - toggle: - icon: material/brightness-7 - name: Switch to dark mode - - # Palette toggle for dark mode - - scheme: slate - primary: indigo - toggle: - icon: material/brightness-4 - name: Switch to light mode - -plugins: - - search - - mkdocstrings: - handlers: - python: - options: - members_order: source - group_by_category: false - signature_crossrefs: true - show_signature: true - docstring_section_style: list - show_root_heading: true - heading_level: 3 - preload_modules: - - typing - - typing_extensions - - ctypes - import: - - https://docs.python.org/3/objects.inv - - https://numpy.org/doc/stable/objects.inv - -watch: - - llama_cpp - - README.md - -nav: - - "Getting Started": "index.md" - - "Installation Guides": - - "macOS (Metal)": "install/macos.md" - - "API Reference": "api-reference.md" - - "OpenAI Compatible Web Server": "server.md" - - "Changelog": "changelog.md" - -markdown_extensions: - - attr_list - - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg - - pymdownx.highlight: - anchor_linenums: true - line_spans: __span - pygments_lang_class: true - - pymdownx.inlinehilite - - pymdownx.magiclink: - repo_url_shorthand: true - user: abetlen - repo: llama-cpp-python - - pymdownx.snippets - - pymdownx.superfences - - pymdownx.tabbed: - alternate_style: true - - pymdownx.tilde - - tables diff --git a/pyproject.toml b/pyproject.toml index 2e439c0685..eb4b879dd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,17 +49,9 @@ test = [ "pydantic-settings>=2.0.1", "huggingface-hub>=0.23.0" ] -dev = [ - "black>=23.3.0", - "twine>=4.0.2", - "mkdocs>=1.4.3", - "mkdocstrings[python]>=0.22.0", - "mkdocs-material>=9.1.18", - "pytest>=7.4.0", - "httpx>=0.24.1", -] + all = [ - "llama_cpp_python[server,test,dev]", + "llama_cpp_python[server,test]", ] [tool.scikit-build] @@ -76,7 +68,7 @@ input = "llama_cpp/__init__.py" [project.urls] Homepage = "https://github.com/JamePeng/llama-cpp-python" Issues = "https://github.com/JamePeng/llama-cpp-python/issues" -Documentation = "https://llama-cpp-python.readthedocs.io/en/latest/" +Documentation = "https://github.com/JamePeng/llama-cpp-python/wiki" Changelog = "https://github.com/JamePeng/llama-cpp-python/blob/main/CHANGELOG.md" FAQ = "https://github.com/JamePeng/llama-cpp-python?tab=readme-ov-file#faq" From d33d98806f1e55219a9a9bfde9557b06e8f16b01 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 22:28:23 +0800 Subject: [PATCH 110/304] Update Submodule vendor/llama.cpp b64739e..39cf5d6 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b64739ea39..39cf5d6191 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b64739ea393b3c9d07cc9907e0a611f707838051 +Subproject commit 39cf5d61915769124b7efbbfa69c46f19a6363ee From e87041e4ee6a89798abe9f36315f60f3fb06c5cb Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 22:57:56 +0800 Subject: [PATCH 111/304] docs(readme): update wheel requirements and dynamic CPU backend info - Update supported CUDA versions to include 12.8 and 13.1, while outlining the supported compute architectures (SM70 up to SM120a). - Document the transition to `GGML_BACKEND_DL` and `GGML_CPU_ALL_VARIANTS` starting in `0.3.39-preview`. - Clarify that dynamic CPU backend loading eliminates the need for separate `Basic` and `AVX2` wheel distributions. - Add a technical note in the FAQ recommending LLVM/Clang over MSVC for achieving full x64 CPU variant coverage on Windows. Signed-off-by: JamePeng --- README.md | 54 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index caec7e32e0..4a1550a85c 100644 --- a/README.md +++ b/README.md @@ -162,12 +162,41 @@ pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python **Pre-built Wheel (New)** -It is also possible to install a pre-built wheel with CUDA support. As long as your system meets some requirements: +It is also possible to install a pre-built wheel with CUDA support. Make sure your system meets the following requirements: -- CUDA Version is 12.4, 12.6, 12.8 or 13.0 -- Python Version is 3.10, 3.11, 3.12, 3.13 or 3.14 -- Basic version(Default): A version compiled without using AVX instructions (for compatibility with CPU platforms lacking AVX instructions or with AVX instruction compatibility issues). -- AVX2 version: A version compiled using AVX2 instructions. +- CUDA version: 12.4, 12.6, 12.8, or 13.1 +- Python version: 3.10, 3.11, 3.12, 3.13, or 3.14 +- Starting with `0.3.39-preview`, Windows and Linux x64 wheels are built with `GGML_BACKEND_DL` and `GGML_CPU_ALL_VARIANTS`. + +This means CPU backends are shipped as dynamically loaded runtime libraries under: + +```text +site-packages/llama_cpp/lib +```` + +Supported CPU backend variants may include: + +* `ggml-cpu-x64` +* `ggml-cpu-sse42` +* `ggml-cpu-sandybridge` +* `ggml-cpu-ivybridge` +* `ggml-cpu-piledriver` +* `ggml-cpu-haswell` +* `ggml-cpu-skylakex` +* `ggml-cpu-cannonlake` +* `ggml-cpu-cascadelake` +* `ggml-cpu-cooperlake` +* `ggml-cpu-icelake` +* `ggml-cpu-alderlake` +* `ggml-cpu-sapphirerapids` +* `ggml-cpu-zen4` + +The old `Basic` and `AVX2` wheel variants are no longer required for the new dynamic-backend wheels. GGML can load the compatible CPU backend at runtime, which improves CPU instruction-set compatibility across different x64 machines. + +Before `0.3.39-preview`: + +* `Basic`: compiled without AVX instructions for maximum compatibility. +* `AVX2`: compiled with AVX2 instructions for newer CPUs. Check the releases page: https://github.com/JamePeng/llama-cpp-python/releases @@ -1695,17 +1724,20 @@ This error is primarily caused by the following reasons: 3. **CUDA Version Mismatch:** Regarding `ggml-cuda.dll`, the CUDA version of the pre-compiled library does not match your local CUDA Toolkit version (e.g., a mismatch between CUDA 12.X and CUDA 13.X). It is recommended to fully configure your local CUDA Toolkit environment (ensuring the PATH for dynamic libraries is set and the nvcc compiler is recognized). Then, clone the code and compile it locally. -### Why are libraries compiled by other authors only around 100MB, while your pre-compiled versions range from 300MB to 900MB? +### Why are libraries compiled by other authors only around 100MB, while your pre-compiled versions are 300MB or larger? -My GitHub Actions script is configured to compile against **all supported CUDA compute architectures** for each specific CUDA version I maintain. +My GitHub Actions workflow is configured to compile against multiple supported CUDA compute architectures for each CUDA version I maintain. For example: -* **CUDA 13.0.2:** Currently supports architectures from SM75 (Turing) up to SM120a (Blackwell). -* **CUDA 12.4.1 and 12.6.3:** Support older architectures as well, such as SM70. -* *(Note: The Windows versions are built to support every architecture compatible with the respective CUDA version).* +- **CUDA 13.1 and CUDA 12.8:** currently target architectures from SM75 (Turing) up to SM120a / SM121a (Blackwell generation, depending on CUDA support). +- **CUDA 12.4 and CUDA 12.6:** currently target architectures from SM70 (Volta) up to SM90 (Hopper). + +Libraries from other authors are often smaller because they may only compile for a single architecture, such as RTX 30 series (`SM86`) or RTX 40 series (`SM89`). To maximize compatibility, these wheels include CUDA kernels for a wider range of GPUs. You only need to choose the wheel that matches your installed CUDA version. + + - **Updated 2026-05-16 / 2026-05-17:** Starting with `0.3.39-preview`, Windows wheels support the `GGML_BACKEND_DL` + `GGML_CPU_ALL_VARIANTS` runtime layout. CPU backend libraries such as `ggml-cpu-*.dll` are packaged under `site-packages/llama_cpp/lib` and loaded dynamically at runtime. This allows GGML to select a compatible CPU backend automatically, reducing the need for separate `Basic` / `AVX2` wheel variants. -The reason libraries from other authors are smaller is that they often **only compile for a single architecture** (e.g., targeting only the RTX 30 series [SM86] or the RTX 40 series [SM89]). To maximize convenience, I provide an **integrated compilation** covering a wide range of hardware; you simply need to select the CUDA version that matches your environment to load and run it. + - Note: for full x64 CPU variant coverage on Windows, LLVM/Clang builds are preferred. MSVC may skip some variants such as `zen4`, `cooperlake`, or `sapphirerapids` due to compiler intrinsic support limitations. ### Quick tips for develop/user (continuously updated): From a778c57d73ec7d4f43e2518a513e7d4cf68a0df8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 17 May 2026 23:35:57 +0800 Subject: [PATCH 112/304] Bump version to 0.3.39 Signed-off-by: JamePeng --- CHANGELOG.md | 116 ++++++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 253b2ae4cc..e4c6e4c976 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,122 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.39] Dynamic GGML Backends, Qwen3-ASR/MiniCPM-V-4.6, On-Device Hybrid Checkpoint, and Granular Logging + +- **ci(cu131/128/126/124): build wheels with GGML dynamic backends for windows/Linux** + - Replace the old CPU/AVX release tag matrix with a single backend + wheel layout. + - Enable `GGML_BACKEND_DL` and `GGML_CPU_ALL_VARIANTS` so Windows wheels ship + runtime-loadable GGML backend DLLs and CPU variant backends. + - Use the Windows LLVM toolchain and disable non-wheel targets such as examples, + tests, tools, server, embedded UI, and curl. + - Remove the `.basic` style local version suffix and publish wheels + as `+cu131`. + - Update CUDA architectures to CUDA 13.1 and simplify CMake argument handling. + - Note: for full x64 CPU variant coverage on Windows, LLVM/Clang builds are preferred. MSVC may skip some variants such as zen4, cooperlake, or sapphirerapids due to compiler intrinsic support limitations. + +- **feat(core): support loading GGML_BACKEND_DL dynamic backend libraries from wheel lib** + - Import `ggml_backend_load_all_from_path` and `ggml_backend_reg_count` + from `_ggml`. + - Load dynamic ggml backend libraries from the packaged `llama_cpp/lib` + directory after `llama_backend_init()`. + - Support wheels built with `GGML_BACKEND_DL`, where CPU variants and + accelerator backends such as `ggml-cpu-*` and `ggml-cuda` are shipped as + separate runtime libraries. + - Print the registered backend count in verbose mode to help diagnose backend + discovery issues. + +- **build(cmake): refactor install target lists for new GGML backend layout** + - Categorize build targets into logical groups (`LLAMA_CPP_TARGETS`, + `GGML_CORE_TARGETS`, `GGML_CPU_VARIANT_TARGETS`, and `GGML_BACKEND_TARGETS`) + to improve maintainability and keep the Python package installation in sync + with the updated upstream GGML backend layout. + - Add missing targets such as `llama-common` and the separated + `ggml-cpu-*` CPU variant backends. + - Ensure all grouped targets are passed through `llama_cpp_python_install_target`. + - Update llama build option descriptions to match the current upstream naming style. + - Explicitly disable `LLAMA_BUILD_SERVER` to avoid building the server target for Python package wheels. + - Explicitly disable `LLAMA_BUILD_UI` and `LLAMA_USE_PREBUILT_UI` because the + embedded server Web UI is not needed for wheel builds. + - Keep examples, tests, and curl support disabled for minimal wheel artifacts. + - Add a cleanup function to strip `cmake`, `pkgconfig`, and import libraries from the python wheel runtime directories. + - Ensures Windows builds only package the required runtime DLLs. + +- **Implement Qwen3ASRChatHandler for Qwen3-ASR models.** + - Integrate MTMD multimodal logic to extract and inject `audio_url` and base64 `input_audio` data directly into the `<|audio_start|><|audio_pad|>[DATA]<|audio_end|>` sequence. + - Define a default multilingual transcription system prompt and configure model-specific stop tokens. + - docs(README.md): add Qwen3-ASR documentation and usage example + - Update the supported multi-modal models table to include `qwen3-asr` and the `Qwen3ASRChatHandler`. + - Add a new dedicated section for Speech-to-Text inference with a complete, collapsible Python script. + - Provide a `build_media_payload` helper function to demonstrate proper Base64 encoding of local `.wav` and `.mp3` files into OpenAI-compatible `input_audio` schemas. + - Include a critical warning advising users to use BF16 quantization for the multimodal projector (`mmproj`) to prevent audio degradation. + - Clarify usage mechanics, specifically that all instructions must be placed in the `system` role due to the ASR template's text-dropping behavior. + +- **Implement MiniCPMV46ChatHandler for MiniCPM-V-4.6** + +- **feat(core): integrate fine-grained logging API into Llama class** + - This commit exposes the newly refactored `_logger` configuration system directly through the `Llama` class, providing users with robust, programmatic control over native `llama.cpp` backend logs. + - docs(wiki): document runtime verbosity and log filters for Llama + - docs(Llama.md): update verbose=False vs. verbosity=0 note + - Key changes: + - Expand `Llama.__init__` with `verbosity`, `log_filters`, and `log_filters_case_sensitive` parameters. + - Add instance methods for runtime log management (`set_verbosity`, `get_verbosity`, `set_log_filters`, `add_log_filters`, `clear_log_filters`, etc.). + - Add comprehensive docstrings explaining the 0-5 verbosity scale and explicitly noting the process-global nature of the native backend logger. + - Advantages over the legacy implementation: + - Granular Control: Replaces the restrictive binary `verbose=True/False` flag (which only toggled between ERROR and DEBUG) with a granular 6-tier scale (output, error, warn, info, trace, debug). + - Dynamic Filtering: Empowers users to actively suppress specific noisy C++ logs using custom substring filters, removing the need for hardcoded internal patches. + - Better Discoverability: Attaches logging controls directly to the `Llama` object, making log management much more accessible and intuitive without requiring users to import internal logger modules. + +- **feat(logger): refactor and enhance ggml logging configuration system** + - Introduce a `LoggerConfig` dataclass to provide fine-grained control over native ggml/llama.cpp runtime logging. + - Align `verbosity` levels (0 to 5) with upstream `llama.cpp` conventions (`common/log.h`). + - Implement a dynamic, configurable substring filtering system, replacing the hardcoded "CUDA Graph" patch with `DEFAULT_LOG_FILTERS`. + - Add comprehensive public APIs for log management: `configure_logging`, `set_verbosity`, `set_quiet`, `set_silent`, `set_log_filters`, and `add_log_filters`. + - Maintain backwards compatibility for the existing `set_verbose(bool)` function. + - Improve the `ggml_log_callback` to correctly handle `GGML_LOG_LEVEL_CONT` by inheriting the verbosity of the preceding log message. + - Route `GGML_LOG_LEVEL_NONE` to `stdout` and all other diagnostic logs to `stderr` by default. + - docs(Logger.md): Upload Logger documentation + +- fix(MTMDChatHandler): correct audio_url content type check and improve variable handling + - Changed condition from `content == "audio_url"` to `content_type == "audio_url"` for proper type-based dispatching. + - Extracted `audio_url` variable for better readability. + - Converted `else` to `elif content_type == "input_audio"` to make the control flow explicit and safer. + +- fix(_internals): Remove unnecessary free operations; models should not be released within the context. + +- **feat(cache): add on-device hybrid checkpoint support** + - Introduce `HybridCheckpointCache` with dual-mode behavior (Host/On-Device). + - Device mode utilizes `LLAMA_STATE_SEQ_FLAGS_ON_DEVICE` to keep tensor + payloads in `llama_context` VRAM, reducing host-device copy overhead. + - Host mode remains the default, preserving full Python-owned rollback history. + - Implement safety guards against stale on-device checkpoint restores and + enforce one active device checkpoint per `seq_id`. + - Unify checkpoint management with shared FIFO eviction. + - Expose `checkpoint_on_device` in `Llama.__init__` and reduce default + `ctx_checkpoints` from 32 to 16. + - Enhance verbose logging and docs to clarify host vs. VRAM ownership + semantics and track memory usage accurately. + - Rename internal `_flag_partial` to `_flags` to support multiple state flags. + - Update /docs/wiki/core/Llama.md for on_device option + - Update /docs/wiki/modules/LlamaCache.md for on_device option + +- docs: Update /docs/wiki and README.md file and remove outdated mkdocs workflow + - docs(readme): update wheel requirements and dynamic CPU backend info + - Update supported CUDA versions to include 12.8 and 13.1, while outlining + the supported compute architectures (SM70 up to SM120a). + - Document the transition to `GGML_BACKEND_DL` and `GGML_CPU_ALL_VARIANTS` + starting in `0.3.39-preview`. + - Clarify that dynamic CPU backend loading eliminates the need for separate + `Basic` and `AVX2` wheel distributions. + - Add a technical note in the FAQ recommending LLVM/Clang over MSVC for + achieving full x64 CPU variant coverage on Windows. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/39cf5d61915769124b7efbbfa69c46f19a6363ee](https://github.com/ggml-org/llama.cpp/commit/39cf5d61915769124b7efbbfa69c46f19a6363ee) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260517 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/ef27f333f367fdc53dc1a729ad8bb6c3c9362514...e87041e4ee6a89798abe9f36315f60f3fb06c5cb + ## [0.3.38] Optimized CJK Detokenization, Sync Grammar Parser, and Patched CUDA Graph Logs - perf: Optimize detokenize buffer sizing for CJK-heavy outputs diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index b32fbfd36e..ec28faae66 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.39-preview" +__version__ = "0.3.39" From a96f2807c3be057650b7bc34173274e1cda68128 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 18 May 2026 21:41:47 +0800 Subject: [PATCH 113/304] ci(metal): upgrade actions/download-artifact@v6 ->v7 - actions/download-artifact@v7 now runs on Node.js 24 Signed-off-by: JamePeng --- .github/workflows/build-wheels-metal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 40675b4c26..a809909720 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v6 - name: Download artifacts - uses: actions/download-artifact@v6 + uses: actions/download-artifact@v7 with: merge-multiple: true path: dist2 From b48d57a2b4019bbd248c848eefa1442c9e7890cb Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 18 May 2026 21:43:37 +0800 Subject: [PATCH 114/304] Update Submodule vendor/llama.cpp 39cf5d6..6db1304 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 39cf5d6191..6db130445d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 39cf5d61915769124b7efbbfa69c46f19a6363ee +Subproject commit 6db130445d29b243ee2171efb8cd61b84a1c5322 From f309265b0df3ab2477682db3a959656dcb6d06e6 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 19 May 2026 19:36:28 +0800 Subject: [PATCH 115/304] build(ci+cu131): bundle LLVM OpenMP runtime for Windows CPU backends - Add a PowerShell step to the Windows CI workflow to locate and copy `libomp140.x86_64.dll` from the Visual Studio redistributables. - Place the runtime DLL into the `llama_cpp\lib` package directory. This ensures that the dynamically loaded `ggml-cpu-*.dll` variants (which are built with LLVM OpenMP on Windows) have their required dependencies packaged in the wheel. Without this, `ggml_backend_load_all_from_path()` can silently fail to load the CPU backends at runtime on end-user machines. Signed-off-by: JamePeng --- .github/workflows/build-wheels-cu131-win.yml | 25 ++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/build-wheels-cu131-win.yml b/.github/workflows/build-wheels-cu131-win.yml index 14bea65d19..5f77003a5f 100644 --- a/.github/workflows/build-wheels-cu131-win.yml +++ b/.github/workflows/build-wheels-cu131-win.yml @@ -67,6 +67,31 @@ jobs: echo LIB=%LIB%>>%GITHUB_ENV% echo LIBPATH=%LIBPATH%>>%GITHUB_ENV% + - name: Copy LLVM OpenMP runtime + shell: pwsh + run: | + # GGML CPU all-variant backends are built with LLVM OpenMP on Windows. + # The dynamically loaded ggml-cpu-*.dll files depend on this runtime. + # If it is missing from the wheel, ggml_backend_load_all_from_path() + # may fail to load CPU backend DLLs at runtime. + $packageLibDir = Join-Path $env:GITHUB_WORKSPACE "llama_cpp\lib" + New-Item -ItemType Directory -Force $packageLibDir | Out-Null + + $omp = Get-ChildItem "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Redist\MSVC" ` + -Recurse ` + -Filter "libomp140.x86_64.dll" ` + -ErrorAction SilentlyContinue | + Where-Object { $_.FullName -match "OpenMP\.LLVM" } | + Select-Object -First 1 + + if (!$omp) { + Write-Error "Could not find libomp140.x86_64.dll in Visual Studio LLVM OpenMP redistributables." + exit 1 + } + + Copy-Item $omp.FullName (Join-Path $packageLibDir "libomp140.x86_64.dll") -Force + Write-Output "Copied LLVM OpenMP runtime: $($omp.FullName)" + - name: Build wheel run: | $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.', '') From dd61687fc6cbabb0885e45d708cc9562d1bd2d53 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 18 May 2026 21:43:37 +0800 Subject: [PATCH 116/304] Update Submodule vendor/llama.cpp 39cf5d6..6db1304 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 39cf5d6191..d14ce3dab4 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 39cf5d61915769124b7efbbfa69c46f19a6363ee +Subproject commit d14ce3dab4de197adec5166faa54ac5db8262f26 From 2bc3cdded9285b591454e11e50a4b1524afa32ff Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 19 May 2026 22:54:36 +0800 Subject: [PATCH 117/304] build(cmake): package LLVM OpenMP runtime DLL for Windows wheels Dynamically loaded GGML CPU backends compiled with LLVM/Clang and OpenMP require `libomp140.x86_64.dll` at runtime. - Add `llama_cpp_python_install_windows_runtime_file` to handle installing arbitrary extra DLLs with proper CMake path normalization. - Add `llama_cpp_python_install_windows_openmp_runtime` to automatically locate the OpenMP DLL in common Visual Studio 2022 directories, with an override available via `LLAMA_CPP_OPENMP_RUNTIME_DLL`. - Execute the OpenMP runtime installation before the dev-file cleanup step to ensure the DLL is correctly packaged in the final wheel. Signed-off-by: JamePeng --- CMakeLists.txt | 105 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e5d583d90..f6dfb7c136 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,106 @@ function(llama_cpp_python_install_target target) endfunction() +# Install an extra Windows runtime DLL into the Python package runtime directory. +# +# Some dynamically loaded backend libraries depend on runtime DLLs that are not +# always discoverable through $. One important example +# is libomp140.x86_64.dll, required by LLVM OpenMP CPU backend variants. +function(llama_cpp_python_install_windows_runtime_file runtime_file) + if(NOT WIN32) + return() + endif() + + if(NOT runtime_file) + return() + endif() + + if(NOT EXISTS "${runtime_file}") + message(WARNING "Windows runtime file does not exist and will not be installed: ${runtime_file}") + return() + endif() + + # Normalize Windows paths for generated cmake_install.cmake. + # Without this, paths like C:\Program Files (...) may produce invalid + # CMake escape sequences such as \P during install. + file(TO_CMAKE_PATH "${runtime_file}" runtime_file_cmake) + + set(INSTALL_DIRS + "${CMAKE_CURRENT_SOURCE_DIR}/llama_cpp/lib" + "${SKBUILD_PLATLIB_DIR}/llama_cpp/lib" + ) + + foreach(DIR ${INSTALL_DIRS}) + file(TO_CMAKE_PATH "${DIR}" DIR_CMAKE) + + install( + FILES "${runtime_file_cmake}" + DESTINATION "${DIR_CMAKE}" + ) + endforeach() +endfunction() + + +# Locate and install the Windows LLVM OpenMP runtime when available. +# +# GGML CPU all-variant backends built with LLVM/Clang + OpenMP depend on +# libomp140.x86_64.dll. Since ggml-cpu-*.dll files are loaded dynamically via +# ggml_backend_load_all_from_path(), the OpenMP runtime must be packaged next to +# them under llama_cpp/lib. +# +# CI may pass LLAMA_CPP_OPENMP_RUNTIME_DLL explicitly. Local builds can rely on +# fallback search paths for Visual Studio Enterprise / BuildTools. +function(llama_cpp_python_install_windows_openmp_runtime) + if(NOT WIN32) + return() + endif() + + set(OPENMP_RUNTIME_DLL "") + + if(DEFINED LLAMA_CPP_OPENMP_RUNTIME_DLL AND EXISTS "${LLAMA_CPP_OPENMP_RUNTIME_DLL}") + set(OPENMP_RUNTIME_DLL "${LLAMA_CPP_OPENMP_RUNTIME_DLL}") + else() + file(TO_CMAKE_PATH "$ENV{ProgramFiles}" PROGRAMFILES_CMAKE) + file(TO_CMAKE_PATH "$ENV{ProgramFiles\(x86\)}" PROGRAMFILES_X86_CMAKE) + + set(VS_OPENMP_SEARCH_ROOTS + "${PROGRAMFILES_CMAKE}/Microsoft Visual Studio/2022/Enterprise/VC/Redist/MSVC" + "${PROGRAMFILES_CMAKE}/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" + "${PROGRAMFILES_X86_CMAKE}/Microsoft Visual Studio/2022/Enterprise/VC/Redist/MSVC" + "${PROGRAMFILES_X86_CMAKE}/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" + ) + + foreach(ROOT ${VS_OPENMP_SEARCH_ROOTS}) + if(EXISTS "${ROOT}") + file( + GLOB_RECURSE FOUND_OPENMP_DLLS + "${ROOT}/*/debug_nonredist/x64/Microsoft.VC*.OpenMP.LLVM/libomp140.x86_64.dll" + "${ROOT}/**/libomp140.x86_64.dll" + ) + + if(FOUND_OPENMP_DLLS) + list(GET FOUND_OPENMP_DLLS 0 OPENMP_RUNTIME_DLL) + break() + endif() + endif() + endforeach() + endif() + + if(OPENMP_RUNTIME_DLL) + message(STATUS "Installing Windows LLVM OpenMP runtime: ${OPENMP_RUNTIME_DLL}") + llama_cpp_python_install_windows_runtime_file("${OPENMP_RUNTIME_DLL}") + else() + message(WARNING + "Could not find libomp140.x86_64.dll. " + "If GGML_OPENMP=ON and GGML CPU backend DLLs are built with LLVM OpenMP, " + "the packaged ggml-cpu-*.dll files may fail to load at runtime. " + "Set LLAMA_CPP_OPENMP_RUNTIME_DLL to the full path of libomp140.x86_64.dll " + "to package it explicitly." + ) + endif() +endfunction() + + # Remove development-only artifacts from Python wheel runtime directories. # # Upstream install rules may place CMake package files, pkg-config files, and @@ -241,6 +341,11 @@ if (LLAMA_BUILD) llama_cpp_python_install_target(mtmd) endif() + # Install Windows LLVM OpenMP runtime when available. + # This must run before cleanup so the final wheel keeps runtime DLLs but + # removes development-only files such as .lib, cmake/, and pkgconfig/. + llama_cpp_python_install_windows_openmp_runtime() + # Run after all runtime targets are installed, including mtmd. llama_cpp_python_cleanup_dev_files() From fa36f70421815f3e050f1538e37f40feb5a7005a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 19 May 2026 23:00:09 +0800 Subject: [PATCH 118/304] Update CHANGELOG.md upstream version link Signed-off-by: JamePeng --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4c6e4c976..e8ebb5cd3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,7 +117,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add a technical note in the FAQ recommending LLVM/Clang over MSVC for achieving full x64 CPU variant coverage on Windows. -- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/39cf5d61915769124b7efbbfa69c46f19a6363ee](https://github.com/ggml-org/llama.cpp/commit/39cf5d61915769124b7efbbfa69c46f19a6363ee) +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/d14ce3dab4de197adec5166faa54ac5db8262f26](https://github.com/ggml-org/llama.cpp/commit/d14ce3dab4de197adec5166faa54ac5db8262f26) - feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260517 From d37951799450b6461ac73160630371c9d1d36065 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 19 May 2026 23:07:12 +0800 Subject: [PATCH 119/304] ci: pin windows runner and streamline python test matrix - Pin the Windows CI runner from `windows-latest` to `windows-2022` to ensure build environment stability and prevent unexpected breakages from runner updates. - Remove Python 3.13 from the test matrix to reduce CI runtime and resource consumption. - Retain Python 3.9 (oldest supported) and 3.14 (latest) to ensure compatibility boundaries are still properly tested across Ubuntu, Windows, and macOS (Metal / Non-Metal). Signed-off-by: JamePeng --- .github/workflows/test.yaml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 335b0f0ac3..420c5e9495 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -24,18 +24,14 @@ jobs: # Don't cancel other jobs in the matrix if one fails fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] - python-version: ["3.9", "3.13", "3.14"] + os: [ubuntu-latest, windows-2022] + python-version: ["3.9", "3.14"] include: # macOS Non-Metal - os: macos-14 python-version: "3.9" cmake_args: "-DLLAMA_METAL=off" metal_status: "(No Metal)" - - os: macos-14 - python-version: "3.13" - cmake_args: "-DLLAMA_METAL=off" - metal_status: "(No Metal)" - os: macos-14 python-version: "3.14" cmake_args: "-DLLAMA_METAL=off" @@ -46,10 +42,6 @@ jobs: python-version: "3.9" cmake_args: "-DLLAMA_METAL=on -DGGML_METAL_USE_BF16=on -DGGML_METAL_EMBED_LIBRARY=on" metal_status: "(Metal)" - - os: macos-14 - python-version: "3.13" - cmake_args: "-DLLAMA_METAL=on -DGGML_METAL_USE_BF16=on -DGGML_METAL_EMBED_LIBRARY=on" - metal_status: "(Metal)" - os: macos-14 python-version: "3.14" cmake_args: "-DLLAMA_METAL=on -DGGML_METAL_USE_BF16=on -DGGML_METAL_EMBED_LIBRARY=on" From c7668b150fb1f8f36ecca0100f294829dfef7e66 Mon Sep 17 00:00:00 2001 From: DELUXA Date: Wed, 20 May 2026 15:39:41 +0300 Subject: [PATCH 120/304] Add Windows ROCm build instructions --- README.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a1550a85c..14148562bf 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,9 @@ https://github.com/JamePeng/llama-cpp-python/releases
HIP (ROCm) +
+Linux ROCm + This provides GPU acceleration on HIP-supported AMD GPUs. Make sure to have ROCm installed. You can download it from your Linux distro's package manager or from here: [ROCm Quick Start (Linux)](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/tutorial/quick-start.html#rocm-install-quick). @@ -303,6 +306,40 @@ More details see here: https://github.com/ggml-org/llama.cpp/blob/master/docs/bu
+
+Windows ROCm + +> **Note:** Install TheRock ROCm, activate your venv, then run in PowerShell. Replace `gfx1200` with your GPU architecture. + +```powershell +cmd /c '"C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat" >nul 2>&1 && set' | ForEach-Object { if ($_ -match '^([^=]+)=(.*)$') { [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2], 'Process') } } + +rocm-sdk init + +$ROCM_DEVEL = "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_devel" +$ROCM_CORE = "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_core" +$ROCM_GFX = (Get-Item "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_libraries_gfx*").FullName + +$env:HIP_PATH = $ROCM_DEVEL +$env:ROCM_PATH = $ROCM_DEVEL +$env:HIP_DEVICE_LIB_PATH = "$ROCM_CORE\lib\llvm\amdgcn\bitcode" +$env:PATH = "$ROCM_DEVEL\bin;$ROCM_DEVEL\lib\llvm\bin;$ROCM_GFX\bin;$env:PATH" +$env:CMAKE_GENERATOR = "Ninja" +$env:HIP_PLATFORM = "amd" +$env:CC = "$ROCM_DEVEL\lib\llvm\bin\clang.exe" +$env:CXX = "$ROCM_DEVEL\lib\llvm\bin\clang++.exe" +$env:HIP_CLANG_PATH = "$ROCM_DEVEL\lib\llvm\bin" + +$R = $ROCM_DEVEL -replace '\\', '/' +$env:CMAKE_ARGS = "-DGGML_HIP=ON -DGGML_HIPBLAS=on -DGPU_TARGETS=gfx1200 -DCMAKE_HIP_ARCHITECTURES=gfx1200 -DCMAKE_C_COMPILER=`"$R/lib/llvm/bin/clang.exe`" -DCMAKE_CXX_COMPILER=`"$R/lib/llvm/bin/clang++.exe`" -DHIP_LIBRARIES=`"$R/lib/amdhip64.lib`" -DCMAKE_PREFIX_PATH=`"$R`"" + +pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" --no-cache-dir +``` + +
+ +
+
Vulkan @@ -1743,7 +1780,7 @@ Libraries from other authors are often smaller because they may only compile for * 1. I've determined that `llama_cpp.server` is currently in a semi-deprecated state (meaning it won't be maintained unless absolutely necessary, and I might even consider deleting or separating it to reduce the library size). I highly recommend using the `llama-server` program maintained by the upstream `llama.cpp` project, which offers a lower-level implementation, more frequent maintenance and optimization, and more reliable API calls. -* 2. Regarding AMD and Intel graphics cards, AMD can certainly use ROCm as the primary backend (but the drawback is that it's basically only stable on Linux platforms), and Intel's Sycl will also encounter some compilation difficulties. I consistently recommend using the Vulkan backend for these two types of graphics cards for greater efficiency and stability, because the upstream `llama.cpp` Vulkan backend is actively maintained by many developers, generally allowing you to enjoy new feature optimizations and bug fixes earlier and faster. +* 2. Regarding AMD and Intel graphics cards, AMD can use ROCm as the primary backend, while Intel's Sycl will encounter some compilation difficulties. I consistently recommend using the Vulkan backend for these two types of graphics cards for greater efficiency and stability, because the upstream `llama.cpp` Vulkan backend is actively maintained by many developers, generally allowing you to enjoy new feature optimizations and bug fixes earlier and faster. * 3. If you are using hybrid multimodal model for building ComfyUI nodes or running single-turn API wrappers where you do not need multi-turn state rollbacks, simply initialize your Llama instance with `ctx_checkpoints=0`: From a4080a4a7e1f4550fd534fa3f798d39946089c7a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 20 May 2026 22:00:36 +0800 Subject: [PATCH 121/304] docs: Optimize the formatting of the ROCm section in README.md. Signed-off-by: JamePeng --- README.md | 68 +++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 14148562bf..07d7635cbc 100644 --- a/README.md +++ b/README.md @@ -288,55 +288,55 @@ https://github.com/JamePeng/llama-cpp-python/releases
HIP (ROCm) -
-Linux ROCm + -
+ Linux ROCm -This provides GPU acceleration on HIP-supported AMD GPUs. Make sure to have ROCm installed. + This provides GPU acceleration on HIP-supported AMD GPUs. Make sure to have ROCm installed. -You can download it from your Linux distro's package manager or from here: [ROCm Quick Start (Linux)](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/tutorial/quick-start.html#rocm-install-quick). + You can download it from your Linux distro's package manager or from here: [ROCm Quick Start (Linux)](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/tutorial/quick-start.html#rocm-install-quick). -To install with HIP / ROCm support for AMD cards, set the `GGML_HIP=ON` environment variable before installing: + To install with HIP / ROCm support for AMD cards, set the `GGML_HIP=ON` environment variable before installing: -```bash -CMAKE_ARGS="-DGGML_HIP=ON -DGPU_TARGETS=gfx1030" pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" -``` -Note: `GPU_TARGETS` is optional, omitting it will build the code for all GPUs in the current system. + ```bash + CMAKE_ARGS="-DGGML_HIP=ON -DGPU_TARGETS=gfx1030" pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" + ``` + Note: `GPU_TARGETS` is optional, omitting it will build the code for all GPUs in the current system. -More details see here: https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md#hip + More details see here: https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md#hip -
+
-
-Windows ROCm + -
+ Windows ROCm -> **Note:** Install TheRock ROCm, activate your venv, then run in PowerShell. Replace `gfx1200` with your GPU architecture. + > **Note:** Install TheRock ROCm, activate your venv, then run in PowerShell. Replace `gfx1200` with your GPU architecture. -```powershell -cmd /c '"C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat" >nul 2>&1 && set' | ForEach-Object { if ($_ -match '^([^=]+)=(.*)$') { [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2], 'Process') } } + ```powershell + cmd /c '"C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat" >nul 2>&1 && set' | ForEach-Object { if ($_ -match '^([^=]+)=(.*)$') { [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2], 'Process') } } -rocm-sdk init + rocm-sdk init -$ROCM_DEVEL = "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_devel" -$ROCM_CORE = "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_core" -$ROCM_GFX = (Get-Item "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_libraries_gfx*").FullName + $ROCM_DEVEL = "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_devel" + $ROCM_CORE = "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_core" + $ROCM_GFX = (Get-Item "$env:VIRTUAL_ENV\Lib\site-packages\_rocm_sdk_libraries_gfx*").FullName -$env:HIP_PATH = $ROCM_DEVEL -$env:ROCM_PATH = $ROCM_DEVEL -$env:HIP_DEVICE_LIB_PATH = "$ROCM_CORE\lib\llvm\amdgcn\bitcode" -$env:PATH = "$ROCM_DEVEL\bin;$ROCM_DEVEL\lib\llvm\bin;$ROCM_GFX\bin;$env:PATH" -$env:CMAKE_GENERATOR = "Ninja" -$env:HIP_PLATFORM = "amd" -$env:CC = "$ROCM_DEVEL\lib\llvm\bin\clang.exe" -$env:CXX = "$ROCM_DEVEL\lib\llvm\bin\clang++.exe" -$env:HIP_CLANG_PATH = "$ROCM_DEVEL\lib\llvm\bin" + $env:HIP_PATH = $ROCM_DEVEL + $env:ROCM_PATH = $ROCM_DEVEL + $env:HIP_DEVICE_LIB_PATH = "$ROCM_CORE\lib\llvm\amdgcn\bitcode" + $env:PATH = "$ROCM_DEVEL\bin;$ROCM_DEVEL\lib\llvm\bin;$ROCM_GFX\bin;$env:PATH" + $env:CMAKE_GENERATOR = "Ninja" + $env:HIP_PLATFORM = "amd" + $env:CC = "$ROCM_DEVEL\lib\llvm\bin\clang.exe" + $env:CXX = "$ROCM_DEVEL\lib\llvm\bin\clang++.exe" + $env:HIP_CLANG_PATH = "$ROCM_DEVEL\lib\llvm\bin" -$R = $ROCM_DEVEL -replace '\\', '/' -$env:CMAKE_ARGS = "-DGGML_HIP=ON -DGGML_HIPBLAS=on -DGPU_TARGETS=gfx1200 -DCMAKE_HIP_ARCHITECTURES=gfx1200 -DCMAKE_C_COMPILER=`"$R/lib/llvm/bin/clang.exe`" -DCMAKE_CXX_COMPILER=`"$R/lib/llvm/bin/clang++.exe`" -DHIP_LIBRARIES=`"$R/lib/amdhip64.lib`" -DCMAKE_PREFIX_PATH=`"$R`"" + $R = $ROCM_DEVEL -replace '\\', '/' + $env:CMAKE_ARGS = "-DGGML_HIP=ON -DGGML_HIPBLAS=on -DGPU_TARGETS=gfx1200 -DCMAKE_HIP_ARCHITECTURES=gfx1200 -DCMAKE_C_COMPILER=`"$R/lib/llvm/bin/clang.exe`" -DCMAKE_CXX_COMPILER=`"$R/lib/llvm/bin/clang++.exe`" -DHIP_LIBRARIES=`"$R/lib/amdhip64.lib`" -DCMAKE_PREFIX_PATH=`"$R`"" -pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" --no-cache-dir -``` + pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" --no-cache-dir + ``` -
+
From 89927d4633e3ff6dde3aa903c3cc84d454e040ad Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 20 May 2026 22:00:49 +0800 Subject: [PATCH 122/304] Update Submodule vendor/llama.cpp d14ce3d..e947228 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index d14ce3dab4..e947228222 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit d14ce3dab4de197adec5166faa54ac5db8262f26 +Subproject commit e947228222147356bc7e64154d3439e142481632 From 023780091755724b9e41d62d3df9f9ffcbafda09 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 21 May 2026 08:27:48 +0800 Subject: [PATCH 123/304] docs: Removed outdated macOS installation guides and added the latest installation notes. Signed-off-by: JamePeng --- README.md | 84 +++++++++++++++++++++++++++++++------------ docs/install/macos.md | 59 ------------------------------ 2 files changed, 62 insertions(+), 81 deletions(-) delete mode 100644 docs/install/macos.md diff --git a/README.md b/README.md index 07d7635cbc..6c56c034c3 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,8 @@ On MacOS, Metal is enabled by default(`GGML_METAL=ON`). Using Metal makes the co To disable the Metal build at compile time use the `CMAKE_ARGS="-DGGML_METAL=OFF"` cmake option. +When built with Metal support, you can explicitly disable GPU inference with the `n-gpu-layers=0` parameter. + ```bash pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" ``` @@ -277,6 +279,7 @@ pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python It is also possible to install a pre-built wheel with Metal support. As long as your system meets some requirements: +- CPU Arch: arm64 - MacOS Version is 11.0 or later - Python Version is 3.10, 3.11, 3.12, 3.13 or 3.14 @@ -415,46 +418,83 @@ CMAKE_ARGS="-DGGML_RPC=on" pip install "llama-cpp-python @ git+https://github.co
-### Windows Notes - +### Install Notes
-Error: Can't find 'nmake' or 'CMAKE_C_COMPILER' + Optimization Options (Optional) -If you run into issues where it complains it can't find `'nmake'` `'?'` or CMAKE_C_COMPILER, you can extract w64devkit as [mentioned in llama.cpp repo](https://github.com/ggerganov/llama.cpp#openblas) and add those manually to CMAKE_ARGS before running `pip` install: +> **šŸ’” Tip:** If you want to save compilation time, you can skip building of llama.cpp with the standalone examples, tools, tests, and server by adding the following flags, as they are not required for Python bindings: -```ps -$env:CMAKE_GENERATOR = "MinGW Makefiles" -$env:CMAKE_ARGS = "-DGGML_OPENBLAS=on -DCMAKE_C_COMPILER=C:/w64devkit/bin/gcc.exe -DCMAKE_CXX_COMPILER=C:/w64devkit/bin/g++.exe" +```bash +-DLLAMA_BUILD_EXAMPLES=OFF \ +-DLLAMA_BUILD_TOOLS=OFF \ +-DLLAMA_BUILD_TESTS=OFF \ +-DLLAMA_BUILD_SERVER=OFF ``` - -See the above instructions and set `CMAKE_ARGS` to the BLAS backend you want to use.
-### MacOS Notes +
+ CUDA compiler warning suppression is optional +CUDA nvcc compiler may print many template-related warnings from ggml-cuda, such as: -Detailed MacOS Metal GPU install documentation is available at [docs/install/macos.md](https://llama-cpp-python.readthedocs.io/en/latest/install/macos/) +```bash +warning #177-D +warning #221-D +warning #550-D +``` -
-M1 Mac Performance Issue +These usually generate a huge amount of noisy diagnostics rather than build blockers. They constantly flood logs and consume CPU printing performance. -Note: If you are using Apple Silicon (M1) Mac, make sure you have installed a version of Python that supports arm64 architecture. For example: +For cleaner CI/local logs, you can pass: ```bash -wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh -bash Miniforge3-MacOSX-arm64.sh +-DCMAKE_CUDA_FLAGS="--diag-suppress=177 --diag-suppress=221 --diag-suppress=550" ``` - -Otherwise, while installing it will build the llama.cpp x86 version which will be 10x slower on Apple Silicon (M1) Mac.
-M Series Mac Error: `(mach-o file, but is an incompatible architecture (have 'x86_64', need 'arm64'))` + Notes for `GGML_BACKEND_DL` + `GGML_CPU_ALL_VARIANTS` builds +When building wheels with `GGML_BACKEND_DL=ON` and `GGML_CPU_ALL_VARIANTS=ON`, +GGML CPU backends are built as separate dynamic libraries, such as: -Try installing with +```text +ggml-cpu-x64.dll +ggml-cpu-haswell.dll +ggml-cpu-alderlake.dll +ggml-cpu-zen4.dll +``` +These backend libraries must be packaged together under: -```bash -CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DGGML_METAL=on" pip install --upgrade --verbose --force-reinstall --no-cache-dir "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +```text +site-packages/llama_cpp/lib ``` + +The runtime must also explicitly load them with: + +```text +ggml_backend_load_all_from_path() +``` + +### Windows notes + +For full x64 CPU variant coverage, `LLVM/Clang` is recommended. `MSVC` may skip some variants such as `zen4`, `cooperlake`, or `sapphirerapids`. + +If `GGML_OPENMP=ON` is used, the LLVM OpenMP runtime must also be packaged next to the backend DLLs: + +```text +libomp140.x86_64.dll +``` + +Without this file, `ggml-cpu-*.dll` may fail to load dynamically at runtime. + +### Wheel packaging checklist + +* Enable `GGML_BACKEND_DL=ON` +* Enable `GGML_CPU_ALL_VARIANTS=ON` +* Use `GGML_NATIVE=OFF` for portable wheels +* Install all `ggml-cpu-*` backend libraries into `llama_cpp/lib` +* Package required runtime dependencies such as `libomp140.x86_64.dll` +* Remove development-only files such as `.lib`, `cmake/`, and `pkgconfig/` +
### Upgrading and Reinstalling diff --git a/docs/install/macos.md b/docs/install/macos.md deleted file mode 100644 index e006fc0a3c..0000000000 --- a/docs/install/macos.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: MacOS Install with Metal GPU ---- - -**(1) Make sure you have xcode installed... at least the command line parts** -``` -# check the path of your xcode install -xcode-select -p - -# xcode installed returns -# /Applications/Xcode-beta.app/Contents/Developer - -# if xcode is missing then install it... it takes ages; -xcode-select --install -``` - -**(2) Install the conda version for MacOS that supports Metal GPU** -``` -wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-arm64.sh -bash Miniforge3-MacOSX-arm64.sh -``` - -**(3) Make a conda environment** -``` -conda create -n llama python=3.9.16 -conda activate llama -``` - -**(4) Install the LATEST llama-cpp-python...which happily supports MacOS Metal GPU as of version 0.1.62** - *(you needed xcode installed in order pip to build/compile the C++ code)* -``` -pip uninstall llama-cpp-python -y -CMAKE_ARGS="-DGGML_METAL=on" pip install -U llama-cpp-python --no-cache-dir -pip install 'llama-cpp-python[server]' - -# you should now have llama-cpp-python v0.1.62 or higher installed -llama-cpp-python Ā  Ā  Ā  Ā  0.1.68 - -``` - -**(5) Download a v3 gguf v2 model** - - **ggufv2** - - file name ends with **Q4_0.gguf** - indicating it is 4bit quantized, with quantisation method 0 - -https://huggingface.co/TheBloke/CodeLlama-7B-GGUF - - -**(6) run the llama-cpp-python API server with MacOS Metal GPU support** -``` -# config your ggml model path -# make sure it is gguf v2 -# make sure it is q4_0 -export MODEL=[path to your llama.cpp ggml models]]/[ggml-model-name]]Q4_0.gguf -python3 -m llama_cpp.server --model $MODEL --n_gpu_layers 1 -``` - -***Note:** If you omit the `--n_gpu_layers 1` then CPU will be used* - - From 14b98ae81802d8c89a55609ec2bf64349aac58f6 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 21 May 2026 20:33:15 +0800 Subject: [PATCH 124/304] Update Submodule vendor/llama.cpp e947228..40d5358 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e947228222..40d5358d3c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e947228222147356bc7e64154d3439e142481632 +Subproject commit 40d5358d3c730b81729ba81cd5c44ed596d02510 From b2f09bb42c0242ae9fcc8a24f0456365891b28de Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 07:14:34 +0800 Subject: [PATCH 125/304] Update Submodule vendor/llama.cpp 40d5358..1acee6b Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 40d5358d3c..1acee6bf89 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 40d5358d3c730b81729ba81cd5c44ed596d02510 +Subproject commit 1acee6bf8939948f9bcbf4b14034e4b475f06069 From 78fa55bd5f8129ebbbf11a4fd6f7fef046707b85 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 08:00:23 +0800 Subject: [PATCH 126/304] feat(speculative): upgrade ngram map decoder with k/k4v modes Enhance `LlamaNGramMapDecoding` to align with the upstream llama.cpp ngram-map algorithm, offering better memory management and draft quality. - Introduce `mode` selection ("k" and "k4v"): "k" stores only historical positions for memory efficiency, while "k4v" caches continuation values directly for faster lookups. - Add `min_hits` threshold to filter out low-confidence drafts. - Implement `max_entries_per_key` to cap dictionary growth and prevent memory bloat during long-context generations. - Improve state synchronization (`_sync_and_index`) using `sync_check_tokens` to safely verify incremental history appends. - Add explicit lifecycle management methods (`clear`, `close`, `accept`) for better API symmetry and resource cleanup. Signed-off-by: JamePeng --- llama_cpp/llama_speculative.py | 313 ++++++++++++++++++++++++++------- 1 file changed, 252 insertions(+), 61 deletions(-) diff --git a/llama_cpp/llama_speculative.py b/llama_cpp/llama_speculative.py index c3814aaf42..c4289d0797 100644 --- a/llama_cpp/llama_speculative.py +++ b/llama_cpp/llama_speculative.py @@ -1,7 +1,7 @@ import abc import collections -from typing import Any, Dict, List, Tuple +from typing import Any, DefaultDict, Dict, List, Literal, Optional, Tuple import numpy as np import numpy.typing as npt @@ -17,102 +17,293 @@ def __call__( class LlamaNGramMapDecoding(LlamaDraftModel): """ - Ultra-fast speculative decoder based on hash inverted index and incremental updates. - O(1) time complexity, aligned with llama.cpp's underlying ngram-map algorithm. + Fast model-free speculative decoder based on prompt n-gram lookup. + + It supports two modes: + + - "k": + Key-only mode. Stores n-gram key -> history positions. + This is memory-efficient and similar to llama.cpp's ngram-map-k behavior. + + - "k4v": + Key-to-value mode. Stores n-gram key -> continuation tokens. + This uses more memory, but can return cached continuations directly. + + This class does not use a draft model. It only speculates from already verified + token history. Therefore, rejected tokens are handled naturally when the next + `input_ids` is passed in. + + Aligned with llama.cpp's underlying ngram-map k/k4v algorithm. """ - def __init__(self, ngram_size: int = 3, num_pred_tokens: int = 10): + def __init__( + self, + ngram_size: int = 3, + num_pred_tokens: int = 10, + mode: Literal["k", "k4v"] = "k", + min_hits: int = 2, + max_entries_per_key: Optional[int] = None, + sync_check_tokens: int = 16, + ) -> None: """ - Initializes the N-Gram Map speculative decoder. - Args: - ngram_size (int): The length of the token sequence used as the search key. - Larger values provide strictly accurate context matching but may result - in fewer cache hits. Defaults to 3. - num_pred_tokens (int): The maximum number of future tokens to draft (predict) - and return once a match is found in the history. Defaults to 10. + ngram_size: + Number of tokens used as the lookup key. + + num_pred_tokens: + Maximum number of draft tokens to return. + + mode: + "k" stores only matched positions. + "k4v" stores matched continuation values directly. + + min_hits: + Minimum number of historical matches required before returning a draft. + Use 1 for maximum recall. Use >1 to reduce low-confidence drafts. + + max_entries_per_key: + Optional memory cap per n-gram key. + When set, only the most recent entries are kept. + For k4v mode, setting max_entries_per_key is strongly recommended. + + sync_check_tokens: + Number of trailing tokens used to verify whether the new input is an + incremental append of the previous input. This avoids expensive full + prefix comparison while still detecting most rollback/prompt-switch cases. """ - self.ngram_size = ngram_size - self.num_pred_tokens = num_pred_tokens + if ngram_size <= 0: + raise ValueError("ngram_size must be greater than 0") + if num_pred_tokens <= 0: + raise ValueError("num_pred_tokens must be greater than 0") + if min_hits <= 0: + raise ValueError("min_hits must be greater than 0") + if max_entries_per_key is not None and max_entries_per_key <= 0: + raise ValueError("max_entries_per_key must be None or greater than 0") + if sync_check_tokens <= 0: + raise ValueError("sync_check_tokens must be greater than 0") + + mode = mode.lower() + if mode not in ("k", "k4v"): + raise ValueError("mode must be either 'k' or 'k4v'") + + self.ngram_size = int(ngram_size) + self.num_pred_tokens = int(num_pred_tokens) + self.mode = mode + self.min_hits = int(min_hits) + self.sync_check_tokens = int(sync_check_tokens) + + if mode == "k4v" and max_entries_per_key is None: + max_entries_per_key = 8 + self.max_entries_per_key = max_entries_per_key - # Core state cache - # Mapping format: (token_1, ..., token_N) -> [index_1, index_2, ...] - self._ngram_map: Dict[Tuple[int, ...], List[int]] = collections.defaultdict(list) self._history: List[int] = [] - def _update_cache(self, input_ids: npt.NDArray[np.intc]) -> None: + # In "k" mode: + # key -> [position, position, ...] + self._map_k: DefaultDict[Tuple[int, ...], List[int]] = collections.defaultdict(list) + + # In "k4v" mode: + # key -> {position: continuation} + # + # A dict is used so that recent entries can be refreshed when more continuation + # tokens become available. + self._map_k4v: DefaultDict[ + Tuple[int, ...], Dict[int, Tuple[int, ...]] + ] = collections.defaultdict(dict) + + self._closed = False + self._last_draft_len = 0 + + def clear(self) -> None: """ - Smart state synchronization and incremental build (Extreme O(1) optimization). + Clear token history and indexes. - Args: - input_ids (npt.NDArray[np.intc]): The complete sequence of current token IDs - generated or processed so far. + Use this when starting a completely unrelated generation while keeping the + decoder instance reusable. + """ + self._history.clear() + self._map_k.clear() + self._map_k4v.clear() + self._last_draft_len = 0 + + def close(self) -> None: + """ + Release internal memory. + + This class does not own native memory, but clearing large Python containers + explicitly is still useful for long-running applications. + """ + self.clear() + self._closed = True + + def __del__(self) -> None: + # Best-effort cleanup. Program correctness must not depend on __del__. + try: + self.close() + except Exception: + pass + + def accept(self, n_accepted: int) -> None: """ - new_len = len(input_ids) + Notify how many draft tokens were accepted by the target model. + + This implementation does not need to update internal state here, because the + next call receives the verified token history through `input_ids`. + + The method is kept for API symmetry and future extensions, such as acceptance + statistics, adaptive reset, or low-acceptance fallback. + """ + return + + def _sync_and_index(self, input_ids: npt.NDArray[np.intc]) -> None: + """ + Synchronize internal history with input_ids and update the n-gram index. + + The index intentionally stores only n-grams that have at least one continuation + token. This prevents the current tail n-gram from matching itself and returning + an empty draft. + """ + if self._closed: + raise RuntimeError("LlamaNGramMapDecoding is closed") + + tokens = np.asarray(input_ids, dtype=np.intc).reshape(-1).tolist() + old_len = len(self._history) + new_len = len(tokens) + + if new_len == 0: + self.clear() + return + + # Fast path: identical input, no update needed. + if new_len == old_len: + if self._history == tokens: + return + + # Incremental append path. + is_append = False + if old_len > 0 and new_len > old_len: + check_len = min(old_len, max(self.ngram_size, self.sync_check_tokens)) + is_append = self._history[old_len - check_len : old_len] == tokens[ + old_len - check_len : old_len + ] + + if is_append: + # Append only new tokens. + self._history.extend(tokens[old_len:]) + + if self.mode == "k": + # Only newly-valid keys need to be added. + start = max(0, old_len - self.ngram_size) + else: + # K4V must also refresh recent keys because their continuation values + # can grow as new tokens are appended. + start = max(0, old_len - self.ngram_size - self.num_pred_tokens + 1) + else: + # Rollback, prompt switch, truncation, or unsafe mutation. + self.clear() + self._history.extend(tokens) + start = 0 + + # Only index keys that have at least one token after the key. + # Valid pos satisfies: + # pos + ngram_size < len(history) + end = max(0, len(self._history) - self.ngram_size) + + if start >= end: + return + + if self.mode == "k": + for pos in range(start, end): + key = tuple(self._history[pos : pos + self.ngram_size]) + bucket = self._map_k[key] + + if not bucket or bucket[-1] != pos: + bucket.append(pos) + + if ( + self.max_entries_per_key is not None + and len(bucket) > self.max_entries_per_key + ): + del bucket[: len(bucket) - self.max_entries_per_key] - # Check if it's a perfect incremental append (verify if the previous token matches) - is_incremental = False - if new_len > old_len and old_len > 0: - if self._history[-1] == input_ids[old_len - 1]: - is_incremental = True - - if is_incremental: - # Only extract, convert, and append new tokens. - # Never copy or touch the entire historical array! - new_tokens = input_ids[old_len:].tolist() - self._history.extend(new_tokens) - start_idx = max(0, old_len - self.ngram_size) else: - # Rollback occurred (wrong prediction) or a completely new Prompt. Trigger full rebuild. - self._ngram_map.clear() - self._history = input_ids.tolist() - start_idx = 0 + for pos in range(start, end): + key_start = pos + value_start = pos + self.ngram_size + value_end = min(value_start + self.num_pred_tokens, len(self._history)) + + if value_start >= value_end: + continue + + key = tuple(self._history[key_start:value_start]) + value = tuple(self._history[value_start:value_end]) - # Build/update the hash inverted index - for i in range(start_idx, new_len - self.ngram_size): - key = tuple(self._history[i : i + self.ngram_size]) - self._ngram_map[key].append(i) + bucket = self._map_k4v[key] + bucket[pos] = value + + if ( + self.max_entries_per_key is not None + and len(bucket) > self.max_entries_per_key + ): + # Keep the most recent positions. + for old_pos in sorted(bucket)[: len(bucket) - self.max_entries_per_key]: + del bucket[old_pos] def __call__( self, input_ids: npt.NDArray[np.intc], /, **kwargs: Any ) -> npt.NDArray[np.intc]: """ - Generates draft tokens based on historical N-Gram frequency. + Generate draft tokens from verified token history. Args: - input_ids (npt.NDArray[np.intc]): The current sequence of token IDs. - **kwargs: Additional generation arguments (ignored in this implementation). + input_ids: + Complete verified token sequence so far. Returns: - npt.NDArray[np.intc]: An array of predicted draft tokens. Returns an empty - array if no matching context is found. + np.ndarray[np.intc]: + Predicted draft tokens. Empty array means no reliable match was found. """ - # 1. Ultra-fast state synchronization - self._update_cache(input_ids) + _ = kwargs + + self._sync_and_index(input_ids) + self._last_draft_len = 0 - # 2. Cannot speculate if the history is too short if len(self._history) < self.ngram_size: return np.array([], dtype=np.intc) - # 3. Extract the Search Key (the last N tokens) - search_key = tuple(self._history[-self.ngram_size:]) + search_key = tuple(self._history[-self.ngram_size :]) - # 4. O(1) instant lookup - match_indices = self._ngram_map.get(search_key) + if self.mode == "k": + positions = self._map_k.get(search_key) + if not positions or len(positions) < self.min_hits: + return np.array([], dtype=np.intc) - if not match_indices: - return np.array([], dtype=np.intc) + # Use the latest valid match with an available continuation. + draft: List[int] = [] + for pos in reversed(positions): + start = pos + self.ngram_size + if start < len(self._history): + end = min(start + self.num_pred_tokens, len(self._history)) + draft = self._history[start:end] + break + + else: + values = self._map_k4v.get(search_key) + if not values or len(values) < self.min_hits: + return np.array([], dtype=np.intc) - # 5. Get the context of the last match and extract draft tokens - best_match_idx = match_indices[-1] - draft_start = best_match_idx + self.ngram_size - draft_end = min(draft_start + self.num_pred_tokens, len(self._history)) + # Use the continuation from the latest historical position. + latest_pos = max(values) + draft = list(values[latest_pos]) - return np.array(self._history[draft_start:draft_end], dtype=np.intc) + self._last_draft_len = len(draft) + return np.asarray(draft, dtype=np.intc) # Legacy Numpy sliding window implementation +# Fast in some cases, but may degrade output quality. +# Not recommended for production. class LlamaPromptLookupDecoding(LlamaDraftModel): """ Stateless speculative decoding based on Numpy sliding window From 91627a0c6b713858ce5a102253d9c694e36c511b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 08:11:26 +0800 Subject: [PATCH 127/304] examples: add benchmark script for speculative decoding - Add `benchmark_speculative.py` to the `examples/benchmark` directory. - Test `LlamaPromptLookupDecoding` and `LlamaNGramMapDecoding` (k/k4v). - Include diverse test scenarios (code, JSON logs, tables, essays) to measure tokens-per-second (TPS) speedup compared to baseline generation. Signed-off-by: JamePeng --- examples/benchmark/benchmark_speculative.py | 466 ++++++++++++++++++++ 1 file changed, 466 insertions(+) create mode 100644 examples/benchmark/benchmark_speculative.py diff --git a/examples/benchmark/benchmark_speculative.py b/examples/benchmark/benchmark_speculative.py new file mode 100644 index 0000000000..73e7c203a2 --- /dev/null +++ b/examples/benchmark/benchmark_speculative.py @@ -0,0 +1,466 @@ +import csv +import gc +import random +import statistics +import time +from dataclasses import dataclass +from typing import Callable, Dict, List, Optional + +from llama_cpp import Llama +from llama_cpp.llama_speculative import ( + LlamaPromptLookupDecoding, + LlamaNGramMapDecoding, +) + + +# ============================================================ +# Model Configuration +# ============================================================ + +MODEL_PATH = r"/path/to/your/model.GGUF" + +N_CTX = 4096 +MAX_TOKENS = 1024 +REPEATS = 2 +CSV_OUTPUT = "speculative_benchmark_results.csv" + +RANDOMIZE_ENGINE_ORDER = False + + +# ============================================================ +# Benchmark Scenario Definition +# ============================================================ + +@dataclass(frozen=True) +class Scenario: + name: str + category: str + prompt: str + expected_behavior: str + + +TEST_SCENARIOS: List[Scenario] = [ + Scenario( + name="A1. Medium-High Repetition - CRUD Boilerplate Code", + category="code_boilerplate", + expected_behavior="Should benefit from n-gram lookup because class and method structures repeat.", + prompt="""<|im_start|>system +You are a senior backend developer. Write highly structured and consistent boilerplate code.<|im_end|> +<|im_start|>user +Write a Python script using `sqlite3` to define CRUD operations for a core banking system database. + +Create 6 separate classes: +- Account +- Transaction +- Customer +- Loan +- Portfolio +- AuditLog + +Each class MUST use the same internal method structure: +- create +- get +- update +- delete +- list_all + +Do not add extra explanations. Output only code.<|im_end|> +<|im_start|>assistant +""", + ), + Scenario( + name="A2. Extreme Repetition - JSONL Trading Logs", + category="structured_logs", + expected_behavior="Should strongly favor n-gram methods, especially K/K4V.", + prompt="""<|im_start|>system +You are a deterministic data generation script. Output only raw JSON lines.<|im_end|> +<|im_start|>user +Continue this algorithmic trading execution log for 40 more lines. +Only change timestamp seconds, symbol, quantity, price, and execution_time_ms. + +{"timestamp":"2026-05-23T09:30:01Z","level":"INFO","module":"exec_engine","event":"trade_filled","symbol":"AAPL","side":"BUY","quantity":100,"price":175.50,"execution_time_ms":12} +{"timestamp":"2026-05-23T09:30:02Z","level":"INFO","module":"exec_engine","event":"trade_filled","symbol":"MSFT","side":"SELL","quantity":50,"price":410.25,"execution_time_ms":15} +{"timestamp":"2026-05-23T09:30:03Z","level":"INFO","module":"exec_engine","event":"trade_filled","symbol":"TSLA","side":"BUY","quantity":200,"price":180.10,"execution_time_ms":11}<|im_end|> +<|im_start|>assistant +""", + ), + Scenario( + name="A3. Markdown Table - Repetitive Course Catalog", + category="markdown_table", + expected_behavior="Repeated table columns and row structure should benefit from speculative lookup.", + prompt="""<|im_start|>system +You generate clean Markdown tables with consistent formatting.<|im_end|> +<|im_start|>user +Create a Markdown comparison table for 30 university postgraduate courses. + +Columns: +| Course ID | Course Title | Department | Credits | Prerequisites | Grading Basis | Core Objective | + +The row format must stay consistent. +Use concise but realistic academic descriptions. +Do not add explanation outside the table.<|im_end|> +<|im_start|>assistant +| Course ID | Course Title | Department | Credits | Prerequisites | Grading Basis | Core Objective | +|---:|---|---|---:|---|---|---| +""", + ), + Scenario( + name="A4. Structured Financial Market Report", + category="structured_report", + expected_behavior="Heading and bullet patterns repeat; n-gram lookup should help moderately.", + prompt="""<|im_start|>system +You are a quantitative macroeconomic analyst. Output structured, clear, and professional financial reports.<|im_end|> +<|im_start|>user +Write a Q3 Macroeconomic & Equity Strategy Outlook Report for institutional investors. + +Requirements: +1. Divide the report into exactly 8 sections. +2. Each section MUST contain exactly one heading and 3 bullet points. +3. Repeatedly emphasize the following themes across the sections: interest rate trajectory, inflation stickiness, equity market volatility, supply chain realignment, and fixed-income duration strategies. +4. Keep the tone highly professional and analytical.<|im_end|> +<|im_start|>assistant +""", + ), + Scenario( + name="B1. Low Repetition - Macroeconomic Historical Essay", + category="low_repetition_creative", + expected_behavior="Should show limited or no speedup; useful as a negative control.", + prompt="""<|im_start|>system +You are an academic historian of economics. Write with varied sentence structures, rich vocabulary, and analytical depth.<|im_end|> +<|im_start|>user +Write a comprehensive essay exploring the psychological and sociological impacts of hyperinflation on institutional trust during the Weimar Republic in the 1920s. + +Requirements: +- Use highly academic and varied language. +- Do NOT use repetitive paragraph structures. +- Do NOT use bullet points or lists. +- Avoid parallel phrasing; favor complex, flowing narrative analysis. +- Make it a long, continuous essay.<|im_end|> +<|im_start|>assistant +The catastrophic devaluation of the Papiermark in the early 1920s fundamentally fractured the psychological bedrock of the Weimar Republic. """, + ), + Scenario( + name="B2. Reasoning-Like Explanation - Quantitative Finance", + category="reasoning_explanation", + expected_behavior="May show smaller speedup because content is less template-like.", + prompt="""<|im_start|>system +You are a careful technical explainer. Avoid repetitive phrasing.<|im_end|> +<|im_start|>user +Explain the foundational assumptions and inherent limitations of the Black-Scholes option pricing model. + +Discuss the following concepts contextually: +- Log-normal distribution of asset prices +- The assumption of constant volatility and risk-free rates +- Frictionless markets (no transaction costs or taxes) +- The difference in applicability between European and American options + +Write in clear, academic paragraphs. Do not use bullet points or lists.<|im_end|> +<|im_start|>assistant +""", + ), + Scenario( + name="C1. Long Context Copy-Edit - High Local Reuse", + category="copy_edit", + expected_behavior="Prompt contains repeated phrases; n-gram lookup should exploit local reuse.", + prompt="""<|im_start|>system +You are a precise academic editing assistant. Preserve the structure while improving the wording.<|im_end|> +<|im_start|>user +Rewrite the following academic grant proposal abstract in a cleaner professional style. +Keep the same repetitive sentence layout but fix the grammar and flow. + +Draft Proposal: +The proposed research will investigate the efficiency of machine learning in high-frequency trading. +The proposed research will demonstrate the risk vectors of automated market making. +The methodology will utilize massive historical limit order book datasets. +The methodology will require significant computational cluster resources. +The expected outcomes will provide a new framework for liquidity provisioning. +The expected outcomes will establish a baseline for regulatory compliance monitoring. +The budget will allocate funds for data acquisition from major exchanges. +The budget will allocate funds for two postdoctoral researchers. +The timeline will span twenty-four months of continuous data analysis. +The timeline will include three major peer-reviewed journal submissions. +The significance will address the growing instability in algorithmic flash crashes. +The significance will ensure safer automated trading environments.<|im_end|> +<|im_start|>assistant +""", + ), +] + + +# ============================================================ +# Engine Definition +# ============================================================ + +@dataclass(frozen=True) +class EngineConfig: + name: str + draft_factory: Callable[[], Optional[object]] + note: str + + +ENGINE_CONFIGS: List[EngineConfig] = [ + EngineConfig( + name="Baseline", + draft_factory=lambda: None, + note="No speculative decoding.", + ), + EngineConfig( + name="PromptLookup-Numpy-n10", + draft_factory=lambda: LlamaPromptLookupDecoding( + max_ngram_size=3, + num_pred_tokens=10, + ), + note="Legacy sliding-window prompt lookup.", + ), + EngineConfig( + name="NGramMap-K-n6", + draft_factory=lambda: LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=6, + mode="k", + min_hits=1, + ), + note="Key-only n-gram map, shorter draft.", + ), + EngineConfig( + name="NGramMap-K-n10", + draft_factory=lambda: LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10, + mode="k", + min_hits=1, + ), + note="Key-only n-gram map, default draft length.", + ), + EngineConfig( + name="NGramMap-K4V-n10-cap8", + draft_factory=lambda: LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10, + mode="k4v", + min_hits=1, + max_entries_per_key=8, + ), + note="K4V with bounded per-key memory.", + ), + EngineConfig( + name="NGramMap-K4V-n16-cap8", + draft_factory=lambda: LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=16, + mode="k4v", + min_hits=1, + max_entries_per_key=8, + ), + note="Longer K4V draft; can be faster on highly repetitive outputs.", + ), + EngineConfig( + name="NGramMap-K-minhits2-n10", + draft_factory=lambda: LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10, + mode="k", + min_hits=2, + ), + note="More conservative K mode.", + ), +] + + +# ============================================================ +# Measurement Helpers +# ============================================================ + +def cleanup_model(llm: Optional[Llama]) -> None: + if llm is not None: + del llm + gc.collect() + + +def create_llama(draft_model: Optional[object]) -> Llama: + return Llama( + model_path=MODEL_PATH, + n_ctx=N_CTX, + n_gpu_layers=-1, + draft_model=draft_model, + verbose=False, + ) + + +def measure_once( + scenario: Scenario, + engine: EngineConfig, + repeat_idx: int, +) -> Dict[str, object]: + draft_model = engine.draft_factory() + + print(f"\nā³ [{scenario.name}] Engine={engine.name} | Repeat={repeat_idx + 1}") + print(f" Note: {engine.note}") + + llm: Optional[Llama] = None + + try: + llm = create_llama(draft_model) + + # Warmup: force backend initialization and first-token path. + llm.create_completion( + prompt=scenario.prompt, + max_tokens=1, + temperature=0.0, + echo=False, + ) + + start = time.perf_counter() + + response = llm.create_completion( + prompt=scenario.prompt, + max_tokens=MAX_TOKENS, + temperature=0.0, + top_p=1.0, + top_k=1, + repeat_penalty=1.0, + echo=False, + ) + + end = time.perf_counter() + + duration = end - start + usage = response.get("usage", {}) + completion_tokens = int(usage.get("completion_tokens", 0)) + total_tokens = int(usage.get("total_tokens", 0)) + prompt_tokens = int(usage.get("prompt_tokens", 0)) + + text = response["choices"][0]["text"] + tps = completion_tokens / duration if duration > 0 else 0.0 + + print( + f"āœ… {engine.name:<28} " + f"{tps:8.2f} tok/s | " + f"time={duration:7.2f}s | " + f"gen={completion_tokens:4d} | " + f"prompt={prompt_tokens:4d}" + ) + print(f" Snippet: {text[:120].replace(chr(10), ' ')}...") + + return { + "scenario": scenario.name, + "category": scenario.category, + "expected_behavior": scenario.expected_behavior, + "engine": engine.name, + "engine_note": engine.note, + "repeat": repeat_idx + 1, + "duration_sec": duration, + "completion_tokens": completion_tokens, + "prompt_tokens": prompt_tokens, + "total_tokens": total_tokens, + "tokens_per_sec": tps, + "snippet": text[:160].replace("\n", "\\n"), + } + + finally: + if hasattr(draft_model, "close"): + draft_model.close() + cleanup_model(llm) + + +# ============================================================ +# Reporting +# ============================================================ + +def summarize_results(rows: List[Dict[str, object]]) -> None: + print("\n\n" + "=" * 90) + print("šŸ“Š Benchmark Summary") + print("=" * 90) + + by_scenario: Dict[str, List[Dict[str, object]]] = {} + for row in rows: + by_scenario.setdefault(str(row["scenario"]), []).append(row) + + for scenario_name, scenario_rows in by_scenario.items(): + print(f"\nšŸ“‚ {scenario_name}") + print("-" * 90) + + grouped: Dict[str, List[float]] = {} + for row in scenario_rows: + grouped.setdefault(str(row["engine"]), []).append(float(row["tokens_per_sec"])) + + baseline_avg = statistics.mean(grouped.get("Baseline", [0.0])) + + print( + f"{'Engine':<32} | {'Avg tok/s':>10} | {'Best':>10} | " + f"{'Worst':>10} | {'Speedup':>8}" + ) + print("-" * 90) + + for engine_name, speeds in grouped.items(): + avg = statistics.mean(speeds) + best = max(speeds) + worst = min(speeds) + speedup = avg / baseline_avg if baseline_avg > 0 else 1.0 + + print( + f"{engine_name:<32} | " + f"{avg:10.2f} | " + f"{best:10.2f} | " + f"{worst:10.2f} | " + f"{speedup:8.2f}x" + ) + + +def save_csv(rows: List[Dict[str, object]], path: str) -> None: + if not rows: + return + + fieldnames = list(rows[0].keys()) + + with open(path, "w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + print(f"\nšŸ’¾ CSV saved to: {path}") + + +# ============================================================ +# Main Benchmark Flow +# ============================================================ + +def run_benchmark() -> None: + print("=" * 90) + print("šŸ† llama-cpp-python Speculative Decoding Benchmark") + print("=" * 90) + print(f"Model: {MODEL_PATH}") + print(f"n_ctx={N_CTX}, max_tokens={MAX_TOKENS}, repeats={REPEATS}") + print("=" * 90) + + rows: List[Dict[str, object]] = [] + + for scenario in TEST_SCENARIOS: + print("\n\n" + "#" * 90) + print(f"šŸ“‚ Scenario: {scenario.name}") + print(f"šŸ“Œ Category: {scenario.category}") + print(f"šŸ§  Expected: {scenario.expected_behavior}") + print("#" * 90) + + engines = list(ENGINE_CONFIGS) + if RANDOMIZE_ENGINE_ORDER: + baseline = [e for e in engines if e.name == "Baseline"] + others = [e for e in engines if e.name != "Baseline"] + random.shuffle(others) + engines = baseline + others + + for engine in engines: + for repeat_idx in range(REPEATS): + row = measure_once( + scenario=scenario, + engine=engine, + repeat_idx=repeat_idx, + ) + rows.append(row) + + summarize_results(rows) + save_csv(rows, CSV_OUTPUT) + + +if __name__ == "__main__": + run_benchmark() \ No newline at end of file From 969f5be484ab9f9d602fd73c129906f4ca2ed63e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 08:31:19 +0800 Subject: [PATCH 128/304] docs(speculative): update wiki for NGramMap k/k4v modes and lifecycle APIs Reflect the recent architectural upgrades to `LlamaNGramMapDecoding` in the official documentation. - Document the new `__init__` parameters (`mode`, `min_hits`, `max_entries_per_key`, `sync_check_tokens`) and their validation rules. - Add a detailed comparison table explaining the memory and behavior differences between the `"k"` and `"k4v"` lookup modes. - Document the newly exposed lifecycle methods (`clear`, `close`, `accept`). - Add comprehensive usage examples demonstrating `k4v` mode with memory caps. - Update internal state descriptions (replacing `_ngram_map` with `_map_k` and `_map_k4v`). - Add a strong production warning against the legacy `LlamaPromptLookupDecoding` and cross-link the new `benchmark_speculative.py` script. Signed-off-by: JamePeng --- docs/wiki/modules/LlamaSpeculative.md | 343 +++++++++++++++++++------- 1 file changed, 260 insertions(+), 83 deletions(-) diff --git a/docs/wiki/modules/LlamaSpeculative.md b/docs/wiki/modules/LlamaSpeculative.md index 0c0ad099fb..9255d01496 100644 --- a/docs/wiki/modules/LlamaSpeculative.md +++ b/docs/wiki/modules/LlamaSpeculative.md @@ -2,7 +2,7 @@ title: Llama Speculative Decoding module_name: llama_cpp.llama_speculative source_file: llama_cpp/llama_speculative.py -last_updated: 2026-05-02 +last_updated: 2026-05-23 version_target: "latest" --- @@ -10,30 +10,37 @@ version_target: "latest" ## Overview -`llama_speculative.py` provides draft model interfaces and prompt-based speculative decoding helpers for `llama-cpp-python`. +`llama_speculative.py` defines draft-model interfaces and prompt-based speculative decoding helpers for `llama-cpp-python`. -Speculative decoding uses a lightweight draft model to propose candidate tokens before the main model verifies them. In this module, the draft model does not need to be a neural model. It can also be a prompt lookup decoder that predicts future tokens by finding repeated token patterns in the existing context. +Speculative decoding lets a draft model propose candidate tokens before the main `Llama` model verifies them. In this module, the draft model does not have to be a neural network. It can also be a model-free prompt lookup decoder that predicts future tokens from repeated token patterns in the already verified context. This module currently defines: | Class | Status | Description | |---|---|---| -| `LlamaDraftModel` | public interface | Abstract base class for draft models used by speculative decoding. | -| `LlamaNGramMapDecoding` | public | Fast stateful n-gram map based speculative decoder. | +| `LlamaDraftModel` | public interface | Abstract base class for speculative draft models. | +| `LlamaNGramMapDecoding` | public | Stateful model-free n-gram lookup decoder with `k` and `k4v` modes. | | `LlamaPromptLookupDecoding` | legacy public | Stateless NumPy sliding-window prompt lookup decoder. | ## Role in the Library This module defines the draft-model side of speculative decoding. -A draft model receives the current token sequence and returns predicted draft tokens. These draft tokens can then be verified by the main `Llama` model during generation. +A draft model receives the verified token sequence so far and returns predicted draft token IDs. These tokens are later verified by the main `Llama` model during generation. The module provides two prompt-based implementations: -- `LlamaNGramMapDecoding`: optimized, stateful, hash-map based lookup. -- `LlamaPromptLookupDecoding`: older stateless NumPy sliding-window implementation. +- `LlamaNGramMapDecoding`: optimized, stateful, hash-map based n-gram lookup. +- `LlamaPromptLookupDecoding`: older stateless NumPy sliding-window lookup. -For new usage, prefer `LlamaNGramMapDecoding` because it incrementally maintains an n-gram index instead of scanning the full token history on every call. +For new usage, prefer `LlamaNGramMapDecoding`. It incrementally maintains an n-gram index, supports memory-oriented lookup modes, and avoids scanning the full token history on every call. + +## Choosing Between Related APIs + +| API | Recommended Use | Notes | +|---|---|---| +| `LlamaNGramMapDecoding` | Default prompt lookup decoder for new usage. | Uses stateful n-gram maps and supports `k` / `k4v` modes. | +| `LlamaPromptLookupDecoding` | Compatibility with older prompt lookup behavior. | Stateless and simple, but scans token history with NumPy sliding windows. | ## Classes @@ -41,7 +48,7 @@ For new usage, prefer `LlamaNGramMapDecoding` because it incrementally maintains ```python class LlamaDraftModel(abc.ABC) -```` +``` Abstract base class for speculative draft models. @@ -58,15 +65,15 @@ def __call__( ) -> npt.NDArray[np.intc] ``` -| Parameter | Type | Description | -| ----------- | ---------------------- | ----------------------------------------------------------------- | -| `input_ids` | `npt.NDArray[np.intc]` | Current token sequence. | -| `**kwargs` | `Any` | Additional generation arguments. Implementations may ignore them. | +| Parameter | Type | Description | +|---|---|---| +| `input_ids` | `npt.NDArray[np.intc]` | Complete verified token sequence so far. | +| `**kwargs` | `Any` | Additional generation arguments. Implementations may ignore them. | Returns: -| Type | Description | -| ---------------------- | -------------------------------------------- | +| Type | Description | +|---|---| | `npt.NDArray[np.intc]` | Draft token IDs proposed by the draft model. | ## `LlamaNGramMapDecoding` @@ -75,9 +82,11 @@ Returns: class LlamaNGramMapDecoding(LlamaDraftModel) ``` -Fast speculative decoder based on an n-gram hash map. +Fast model-free speculative decoder based on prompt n-gram lookup. + +This decoder maintains internal indexes from historical n-grams to either previous positions or cached continuation tokens. When called with the current verified token sequence, it searches for the final n-gram in the already verified history and returns a continuation from the most recent valid historical match. -This decoder maintains an internal inverted index from historical n-grams to their positions. When called with the current token sequence, it looks up the final n-gram in the history and returns the following tokens from the most recent matching context. +It does not own or run a separate draft model. Rejected draft tokens do not require manual rollback inside this class, because the next call receives the verified token history through `input_ids`. ### Constructor @@ -86,52 +95,207 @@ def __init__( self, ngram_size: int = 3, num_pred_tokens: int = 10, -) + mode: Literal["k", "k4v"] = "k", + min_hits: int = 2, + max_entries_per_key: Optional[int] = None, + sync_check_tokens: int = 16, +) -> None ``` -| Parameter | Type | Default | Description | -| ----------------- | ----- | ------- | ------------------------------------------------------------------------------------------------------------------------------- | -| `ngram_size` | `int` | `3` | Length of the token sequence used as the lookup key. Larger values require stricter context matches but may produce fewer hits. | -| `num_pred_tokens` | `int` | `10` | Maximum number of draft tokens to return after a matching n-gram is found. | +| Parameter | Type | Default | Source | Description | +|---|---|---|---|---| +| `ngram_size` | `int` | `3` | `__init__` signature | Number of tokens used as the lookup key. Larger values require stricter matches and may reduce hit rate. | +| `num_pred_tokens` | `int` | `10` | `__init__` signature | Maximum number of draft tokens to return. | +| `mode` | `Literal["k", "k4v"]` | `"k"` | `__init__` signature | Lookup storage mode. `"k"` stores key-to-position mappings. `"k4v"` stores key-to-continuation mappings. | +| `min_hits` | `int` | `2` | `__init__` signature | Minimum number of historical matches required before returning a draft. Use `1` for maximum recall; use values greater than `1` to reduce low-confidence drafts. | +| `max_entries_per_key` | `Optional[int]` | `None` | `__init__` signature and initialization logic | Optional memory cap per n-gram key. If `mode="k4v"` and this is `None`, it is automatically set to `8`. | +| `sync_check_tokens` | `int` | `16` | `__init__` signature | Number of trailing tokens used to detect whether new input is an incremental append without doing a full prefix comparison. | + +### Parameter Validation + +The constructor raises `ValueError` when: + +| Condition | Error Meaning | +|---|---| +| `ngram_size <= 0` | `ngram_size` must be positive. | +| `num_pred_tokens <= 0` | `num_pred_tokens` must be positive. | +| `min_hits <= 0` | `min_hits` must be positive. | +| `max_entries_per_key is not None and max_entries_per_key <= 0` | The memory cap must be `None` or positive. | +| `sync_check_tokens <= 0` | `sync_check_tokens` must be positive. | +| `mode` is not `"k"` or `"k4v"` after lowercasing | Only the two supported lookup modes are valid. | + +### Lookup Modes + +| Mode | Internal Storage | Memory Use | Behavior | +|---|---|---|---| +| `"k"` | `key -> [position, position, ...]` | Lower | Stores historical positions and slices continuations from `_history` during lookup. | +| `"k4v"` | `key -> {position: continuation}` | Higher | Stores continuation tokens directly and returns the latest cached continuation. | + +Use `"k"` as the general-purpose default. Use `"k4v"` when faster continuation retrieval is preferred and the extra memory use is acceptable. For `"k4v"`, `max_entries_per_key` defaults to `8` when not specified. ### Important Attributes / State -| Attribute | Type | Source | Description | -| ----------------- | ---------------------------------- | -------------- | -------------------------------------------------------------------------------- | -| `ngram_size` | `int` | constructor | Number of tokens used as the n-gram lookup key. | -| `num_pred_tokens` | `int` | constructor | Maximum number of predicted draft tokens to return. | -| `_ngram_map` | `Dict[Tuple[int, ...], List[int]]` | internal cache | Internal inverted index mapping n-gram tuples to positions in the token history. | -| `_history` | `List[int]` | internal cache | Internal token history used to maintain the n-gram map. | +| Attribute | Type | Source | Description | +|---|---|---|---| +| `ngram_size` | `int` | constructor | Number of tokens used as the n-gram lookup key. | +| `num_pred_tokens` | `int` | constructor | Maximum number of predicted draft tokens to return. | +| `mode` | `str` | constructor | Active lookup mode: `"k"` or `"k4v"`. | +| `min_hits` | `int` | constructor | Required number of historical matches before returning a draft. | +| `max_entries_per_key` | `Optional[int]` | constructor / initialization logic | Optional per-key memory cap. Automatically becomes `8` for `k4v` mode when not provided. | +| `sync_check_tokens` | `int` | constructor | Trailing-token window used for incremental append detection. | +| `_history` | `List[int]` | internal state | Verified token history mirrored from `input_ids`. | +| `_map_k` | `DefaultDict[Tuple[int, ...], List[int]]` | internal state | Key-to-position index used in `"k"` mode. | +| `_map_k4v` | `DefaultDict[Tuple[int, ...], Dict[int, Tuple[int, ...]]]` | internal state | Key-to-continuation index used in `"k4v"` mode. | +| `_closed` | `bool` | internal state | Marks the decoder as closed. Calling the decoder after `close()` raises `RuntimeError`. | +| `_last_draft_len` | `int` | internal state | Length of the most recent returned draft. Currently internal diagnostic state. | + +Internal state should not be mutated directly. + +### Core Methods + +#### `__call__` + +```python +def __call__( + self, + input_ids: npt.NDArray[np.intc], + /, + **kwargs: Any, +) -> npt.NDArray[np.intc] +``` + +Generates draft tokens from verified token history. + +| Parameter | Type | Description | +|---|---|---| +| `input_ids` | `npt.NDArray[np.intc]` | Complete verified token sequence so far. | +| `**kwargs` | `Any` | Accepted for interface compatibility and ignored by this implementation. | + +Returns: + +| Type | Description | +|---|---| +| `npt.NDArray[np.intc]` | Predicted draft tokens. Returns an empty array when no reliable match is found. | + +Raises: + +| Exception | Condition | +|---|---| +| `RuntimeError` | The decoder has been closed with `close()` and is called again. | + +#### `clear` + +```python +def clear(self) -> None +``` + +Clears token history and internal indexes while keeping the decoder reusable. + +Use this when starting a completely unrelated generation with the same decoder instance. + +#### `close` + +```python +def close(self) -> None +``` + +Clears internal containers and marks the decoder as closed. + +This class does not own native memory, but explicit cleanup can be useful in long-running applications that may otherwise keep large Python containers alive. + +#### `accept` + +```python +def accept(self, n_accepted: int) -> None +``` + +Compatibility hook for speculative decoding loops. -`_ngram_map` and `_history` are internal state and should not be modified directly. +This implementation is intentionally a no-op. Accepted tokens are reflected by the next `input_ids` passed to `__call__`, so no separate rollback or acceptance state update is required. ### Behavior When called, `LlamaNGramMapDecoding`: -1. Synchronizes its internal history with the provided `input_ids`. -2. Incrementally updates the n-gram map when tokens are appended. -3. Rebuilds the map if the input sequence is no longer a simple continuation, such as after rollback or a new prompt. -4. Uses the last `ngram_size` tokens as the search key. -5. Returns up to `num_pred_tokens` tokens following the most recent historical match. -6. Returns an empty NumPy array if no match is found. +1. Converts `input_ids` to a flat `np.intc` token list. +2. Synchronizes internal history with the verified token sequence. +3. Uses a fast path when the new input is identical to the stored history. +4. Uses an incremental append path when the trailing tokens indicate that the new input extends the previous input. +5. Rebuilds the index after rollback, prompt switch, truncation, or unsafe mutation. +6. Indexes only n-grams with at least one available continuation token, so the current tail n-gram does not match itself. +7. Looks up the final `ngram_size` tokens as the search key. +8. Requires at least `min_hits` historical matches before returning a draft. +9. Returns up to `num_pred_tokens` tokens from the latest valid historical match. +10. Returns an empty NumPy array if no reliable match is available. -### Example +### Example: Direct Prompt Lookup + +Use `min_hits=1` in a small standalone example so that one historical match is enough to return a draft. ```python import numpy as np + from llama_cpp.llama_speculative import LlamaNGramMapDecoding draft_model = LlamaNGramMapDecoding( ngram_size=3, - num_pred_tokens=5, + num_pred_tokens=2, + min_hits=1, ) -input_ids = np.array([1, 2, 3, 4, 1, 2, 3], dtype=np.intc) - +input_ids = np.array([1, 2, 3, 4, 5, 1, 2, 3], dtype=np.intc) draft_tokens = draft_model(input_ids) print(draft_tokens) +# Expected output: +# [4 5] +``` + +### Example: Use with `Llama` + +```python +from llama_cpp import Llama +from llama_cpp.llama_speculative import LlamaNGramMapDecoding + +llm = Llama( + model_path="path/to/model.gguf", + n_ctx=4096, + n_gpu_layers=-1, + draft_model=LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10, + mode="k", + min_hits=2, + ), +) + +response = llm.create_chat_completion( + messages=[ + { + "role": "user", + "content": ( + "Write five short Python classes with the same CRUD method layout: " + "User, Product, Order, Review, and Category." + ), + } + ] +) + +print(response["choices"][0]["message"]["content"]) +``` + +### Example: Use `k4v` Mode with a Memory Cap + +```python +from llama_cpp.llama_speculative import LlamaNGramMapDecoding + +draft_model = LlamaNGramMapDecoding( + ngram_size=4, + num_pred_tokens=8, + mode="k4v", + min_hits=2, + max_entries_per_key=8, +) ``` ## `LlamaPromptLookupDecoding` @@ -144,7 +308,7 @@ Legacy speculative decoder based on NumPy sliding-window lookup. This implementation is stateless. Each call scans the input token sequence to find previous occurrences of the current n-gram and returns the following tokens as draft predictions. -> Warning: This implementation may have high computational overhead for long contexts. Prefer `LlamaNGramMapDecoding` for new usage. +> Warning: This implementation is not recommended for production. It may have high computational overhead for long contexts and may degrade output quality. Prefer `LlamaNGramMapDecoding` for new usage. ### Constructor @@ -156,16 +320,16 @@ def __init__( ) ``` -| Parameter | Type | Default | Description | -| ----------------- | ----- | ------- | -------------------------------------------------------------------------- | -| `max_ngram_size` | `int` | `3` | Maximum n-gram size to search for. The decoder tries larger n-grams first. | -| `num_pred_tokens` | `int` | `10` | Maximum number of draft tokens to return. | +| Parameter | Type | Default | Source | Description | +|---|---|---|---|---| +| `max_ngram_size` | `int` | `3` | `__init__` signature | Maximum n-gram size to search for. The decoder tries larger n-grams first. | +| `num_pred_tokens` | `int` | `10` | `__init__` signature | Maximum number of draft tokens to return. | ### Important Attributes / State -| Attribute | Type | Source | Description | -| ----------------- | ----- | ----------- | --------------------------------------------------- | -| `max_ngram_size` | `int` | constructor | Maximum n-gram window size used during lookup. | +| Attribute | Type | Source | Description | +|---|---|---|---| +| `max_ngram_size` | `int` | constructor | Maximum n-gram window size used during lookup. | | `num_pred_tokens` | `int` | constructor | Maximum number of predicted draft tokens to return. | ### Static Method @@ -181,58 +345,71 @@ def find_candidate_pred_tokens( Linearly scans `input_ids` using NumPy sliding windows to find matching n-grams. -| Parameter | Type | Description | -| ----------------- | ---------------------- | ----------------------------------------- | -| `input_ids` | `npt.NDArray[np.intc]` | Complete token sequence. | -| `max_ngram_size` | `int` | Maximum n-gram size to search for. | -| `num_pred_tokens` | `int` | Maximum number of draft tokens to return. | +| Parameter | Type | Description | +|---|---|---| +| `input_ids` | `npt.NDArray[np.intc]` | Complete token sequence. | +| `max_ngram_size` | `int` | Maximum n-gram size to search for. | +| `num_pred_tokens` | `int` | Maximum number of draft tokens to return. | Returns: -| Type | Description | -| ---------------------- | --------------------------------------------------------------- | +| Type | Description | +|---|---| | `npt.NDArray[np.intc]` | Candidate draft tokens, or an empty array if no match is found. | -### Example +### Method ```python -from llama_cpp import Llama -from llama_cpp.llama_speculative import LlamaNGramMapDecoding - -llama = Llama( - model_path="path/to/qwen-3.6-27b.gguf", - n_ctx=4096, - n_gpu_layers=-1, - draft_model=LlamaNGramMapDecoding( - ngram_size=3, - num_pred_tokens=10 - ) -) - -response = llama.create_chat_completion( - messages=[{"role": "user", "content": """ - Write a Python script using `sqlite3` to define CRUD (Create, Read, Update, Delete) operations for an e-commerce database. -You need to create 5 separate classes for the following entities: `User`, `Product`, `Order`, `Review`, and `Category`. -Each class MUST have exactly the same internal structure and method names (create, get, update, delete). Do not add extra logic, just the standard boilerplate. - """}] -) +def __call__( + self, + input_ids: npt.NDArray[np.intc], + /, + **kwargs: Any, +) -> npt.NDArray[np.intc] ``` +Calls `find_candidate_pred_tokens` with the instance's `max_ngram_size` and `num_pred_tokens`. + ## Best Practices & Common Patterns -* Prefer `LlamaNGramMapDecoding` for new usage. -* Use `LlamaPromptLookupDecoding` only when compatibility with the older stateless prompt lookup behavior is needed. -* Increase `ngram_size` or `max_ngram_size` for stricter context matching. -* Increase `num_pred_tokens` when you want longer draft proposals, but keep in mind that speculative decoding still depends on later verification by the main model. -* Do not mutate `_ngram_map` or `_history` directly. -* If input token history rolls back or changes unexpectedly, `LlamaNGramMapDecoding` automatically rebuilds its internal cache. +- Prefer `LlamaNGramMapDecoding` for new usage. +- Use `mode="k"` as the default memory-efficient mode. +- Use `mode="k4v"` when cached continuations are useful and the additional memory use is acceptable. +- Keep `max_entries_per_key` set for `k4v` mode unless you intentionally want an unbounded per-key cache. +- Use `min_hits=1` for maximum recall in repetitive prompts or benchmarks. +- Use `min_hits > 1` to reduce low-confidence drafts. +- Increase `ngram_size` for stricter pattern matching. +- Increase `num_pred_tokens` to allow longer draft proposals, but remember that the target model still verifies the tokens. +- Call `clear()` before reusing the same decoder for an unrelated prompt or generation session. +- Do not call the decoder again after `close()` unless you create a new instance. +- Do not mutate `_history`, `_map_k`, `_map_k4v`, or other internal state directly. + +## Limitations + +- Prompt lookup only predicts tokens that are already implied by repeated patterns in the verified context. +- It is most useful for repetitive, structured, or boilerplate-heavy output. +- It may return an empty draft when the context has too few repeated n-grams or when `min_hits` is too strict. +- It does not replace target-model verification. +- `LlamaPromptLookupDecoding` is kept for compatibility and is not recommended for production use. ## Deprecated / Changed APIs -`LlamaPromptLookupDecoding` is marked as a legacy NumPy sliding-window implementation in the source code. It is still available, but `LlamaNGramMapDecoding` is the preferred implementation for faster repeated calls over long contexts. +`LlamaPromptLookupDecoding` is the legacy NumPy sliding-window implementation. It remains available, but `LlamaNGramMapDecoding` is the preferred prompt lookup implementation for new code. + +Compared with the older `LlamaNGramMapDecoding` documentation, the current implementation adds: + +- `mode` +- `min_hits` +- `max_entries_per_key` +- `sync_check_tokens` +- `clear()` +- `close()` +- `accept()` +- Separate internal indexes for `k` and `k4v` modes ## Related Links * [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] * [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] +* [[Benchmark_Speculative](https://github.com/JamePeng/llama-cpp-python/blob/main/examples/benchmark/benchmark_speculative.py)] From d90895d33c7868d9c949d9c1648d33ad3ebc7f8e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 11:02:33 +0800 Subject: [PATCH 129/304] docs(readme): revamp speculative decoding documentation Expand the Speculative Decoding section to fully document the new `LlamaNGramMapDecoding` capabilities and configuration options. - Clarify that `LlamaNGramMapDecoding` is a model-free prompt lookup decoder that does not require a secondary GGUF draft model. - Add a detailed parameter table explaining `mode` (k vs. k4v), `min_hits`, memory caps, and sync thresholds. - Provide usage examples and tuning recommendations for different hardware (e.g., lowering `num_pred_tokens` for CPU setups). - Demote the older `LlamaPromptLookupDecoding` to a legacy section, warning about its sliding-window overhead on long contexts. - Add practical notes on performance and state management (`clear()`). Signed-off-by: JamePeng --- README.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 85 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6c56c034c3..1986a6ca54 100644 --- a/README.md +++ b/README.md @@ -1592,44 +1592,116 @@ emb = llm.create_embedding("text") --- -### Speculative Decoding +## Speculative Decoding -`llama-cpp-python` supports speculative decoding which allows the model to generate completions based on a draft model. +`llama-cpp-python` supports speculative decoding through a `draft_model` passed to the `Llama` class. -The fastest way to use speculative decoding is through the `LlamaNGramMapDecoding`(**Recommend**) or `LlamaPromptLookupDecoding` class. +Speculative decoding lets a draft decoder propose candidate tokens before the main model verifies them. This can improve generation speed, especially for repetitive or structured outputs such as code, JSON, boilerplate text, templates, and long-form responses with repeated patterns. -Just pass this as a draft model to the `Llama` class during initialization. +The recommended built-in draft decoder is `LlamaNGramMapDecoding`. + +Unlike neural draft-model speculative decoding, `LlamaNGramMapDecoding` does not require a second GGUF model. It is a model-free prompt n-gram lookup decoder that predicts draft tokens from already verified token history. ```python from llama_cpp import Llama from llama_cpp.llama_speculative import LlamaNGramMapDecoding llama = Llama( - model_path="path/to/qwen-3.6-27b.gguf", + model_path="path/to/model.gguf", n_ctx=4096, n_gpu_layers=-1, draft_model=LlamaNGramMapDecoding( ngram_size=3, - num_pred_tokens=10 - ) + num_pred_tokens=10, + ), ) response = llama.create_chat_completion( - messages=[{"role": "user", "content": "Write a python script..."}] + messages=[ + { + "role": "user", + "content": "Write a Python script using sqlite3 with repeated CRUD classes.", + } + ] +) +```` + +`LlamaNGramMapDecoding` maintains an internal n-gram index and can reuse repeated token patterns from the current prompt and generated context. Compared with the legacy sliding-window prompt lookup decoder, it avoids scanning the full token history on every call, making draft generation much cheaper for long contexts. + +#### Advanced configuration + +```python +from llama_cpp.llama_speculative import LlamaNGramMapDecoding + +draft_model = LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10, + mode="k", + min_hits=2, + max_entries_per_key=None, + sync_check_tokens=16, +) +``` + +| Parameter | Default | Description | +| --------------------- | ----------------------------------------: | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `ngram_size` | `3` | Number of tokens used as the lookup key. Larger values require stricter matches. | +| `num_pred_tokens` | `10` | Maximum number of draft tokens to propose. | +| `mode` | `"k"` | N-gram map mode. `"k"` stores key-to-position mappings. `"k4v"` stores key-to-continuation mappings. | +| `min_hits` | `2` | Minimum number of historical matches required before returning draft tokens. Use `1` for higher recall, or `2+` to reduce low-confidence drafts. | +| `max_entries_per_key` | `None` in `"k"` mode, `8` in `"k4v"` mode | Optional memory cap per n-gram key. Strongly recommended for `"k4v"` mode. | +| `sync_check_tokens` | `16` | Number of trailing tokens used to detect whether the new input is an incremental append or requires rebuilding the internal index. | + +#### Choosing a mode + +`LlamaNGramMapDecoding` supports two modes: + +* `mode="k"`: stores n-gram keys mapped to historical positions. This is the default and is usually the best starting point. +* `mode="k4v"`: stores n-gram keys mapped directly to continuation tokens. This can make continuation lookup cheaper, but uses more memory. When using `"k4v"`, keeping `max_entries_per_key` enabled is recommended. + +For most users, the default configuration is enough: + +```python +draft_model=LlamaNGramMapDecoding() +``` + +For higher recall, especially when the prompt has fewer repeated patterns, you can lower `min_hits`: + +```python +draft_model=LlamaNGramMapDecoding( + ngram_size=3, + num_pred_tokens=10, + min_hits=1, ) ``` -Note: `LlamaPromptLookupDecoding.num_pred_tokens` is the number of tokens to predict 10 is the default and generally good for gpu, 2 performs better for cpu-only machines. Now, `LlamaNGramMapDecoding` with the new Hash Map algorithm, draft generation becomes instantaneous $O(1)$, and the time consumption is almost 0 regardless of whether you set the prediction to 2 or 10 words. -### Adjusting the Context Window +For CPU-only machines, smaller draft lengths such as `num_pred_tokens=2` may still be a better tradeoff. For GPU inference, larger values such as `num_pred_tokens=10` are often reasonable, but the best value depends on model size, prompt structure, backend, and acceptance rate. -The context window of the Llama models determines the maximum number of tokens that can be processed at once. By default, this is set to 512 tokens, but can be adjusted based on your requirements. +#### Legacy prompt lookup decoder -For instance, if you want to work with larger contexts, you can expand the context window by setting the n_ctx parameter when initializing the Llama object: +`LlamaPromptLookupDecoding` is still available for compatibility: ```python -llm = Llama(model_path="./models/llama-model.gguf", n_ctx=2048) +from llama_cpp.llama_speculative import LlamaPromptLookupDecoding + +draft_model = LlamaPromptLookupDecoding( + max_ngram_size=3, + num_pred_tokens=10, +) ``` +However, it uses a legacy NumPy sliding-window lookup and may have higher overhead on long contexts. For new usage, prefer `LlamaNGramMapDecoding`. + +#### Notes + +* Speculative decoding still requires the main model to verify proposed draft tokens. +* Speedup depends on how many draft tokens are accepted. +* Prompt n-gram speculative decoding works best when the current context contains repeated patterns. +* It is especially useful for code generation, structured text, repeated templates, and boilerplate-heavy completions. +* `LlamaNGramMapDecoding` stores internal Python-side history and indexes. If you want to reuse the same decoder instance for an unrelated generation, call `draft_model.clear()`. + +--- + ## Docker image See here: https://github.com/JamePeng/llama-cpp-python/tree/main/docker#cuda_simple From 5364cf914b590065690eacaaf94ecb2453766a67 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 12:58:02 +0800 Subject: [PATCH 130/304] feat(LlamaContext): add safety checks and docstrings to logits retrieval - Add explicit null pointer validation to `get_logits` and `get_logits_ith`. These methods now raise a `RuntimeError` instead of silently returning invalid pointers when logits are unavailable or the index is out of bounds. - Add comprehensive docstrings to both methods, detailing the underlying buffer shape and memory layout. - Include a performance warning in `get_logits_ith` about the internal synchronization/reordering overhead to discourage its use on the hot path. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index c026440a2d..fda9187855 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -755,12 +755,36 @@ def synchronize(self): llama_cpp.llama_synchronize(self.ctx) def get_logits(self): + """ + Token logits obtained from the last call to llama_decode() + The logits for which llama_batch.logits[i] != 0 are stored contiguously + in the order they have appeared in the batch. + Rows: number of tokens for which llama_batch.logits[i] != 0 + Cols: n_vocab + + Returns: + Pointer to the logits buffer of shape (n_tokens, n_vocab) + """ self._assert_ctx() - return llama_cpp.llama_get_logits(self.ctx) + logits = llama_cpp.llama_get_logits(self.ctx) + if not logits: + raise RuntimeError(f"LlamaContext.get_logits: failed to get logits") + return logits def get_logits_ith(self, i: int): + """ + Return logits for the ith output row from the last llama_decode call. + + Note: + This calls llama_get_logits_ith(), which may reorder/synchronize + the output buffer internally. Avoid calling it on the hot path unless + Python-side logits are required. + """ self._assert_ctx() - return llama_cpp.llama_get_logits_ith(self.ctx, i) + logits = llama_cpp.llama_get_logits_ith(self.ctx, i) + if not logits: + raise RuntimeError(f"LlamaContext.get_logits_ith: invalid logits index {i}") + return logits def set_embeddings(self, embeddings: bool): self._assert_ctx() From 7e0cd122d0af2f9971ebdfc40fb177366c394280 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 13:02:09 +0800 Subject: [PATCH 131/304] build(cmake): disable building of upstream unified binary Set `LLAMA_BUILD_APP` to `OFF` to prevent the compilation of the new unified `llama` binary introduced in upstream llama.cpp. Since the Python package only requires the underlying shared libraries and specific targets, explicitly disabling the standalone application reduces build times and prevents unnecessary executable artifacts from being compiled. Signed-off-by: JamePeng --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f6dfb7c136..6f09cdb783 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -222,6 +222,9 @@ if (LLAMA_BUILD) # Disable building of server set(LLAMA_BUILD_SERVER OFF CACHE BOOL "llama: build server example" FORCE) + # Disable building of unified binary + set(LLAMA_BUILD_APP OFF CACHE BOOL "llama: build the unified binary" FORCE) + # Disable build the embedded Web UI for server set(LLAMA_BUILD_UI OFF CACHE BOOL "llama: build the embedded Web UI for server" FORCE) set(LLAMA_USE_PREBUILT_UI OFF CACHE BOOL "llama: use prebuilt UI from HF Bucket when available (requires LLAMA_BUILD_UI=ON)" FORCE) From 615e45a47f47387e741c12eeca397339fee0e74b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 13:20:06 +0800 Subject: [PATCH 132/304] perf(eval): skip unnecessary logit array copies during native sampling - Introduce the `copy_logits` parameter to `Llama.eval()` to control whether C-level logits are copied into the Python `self.scores` array. - Automatically disable `copy_logits` during the generation loop unless Python-side hooks (`logits_processor`, `stopping_criteria`) or `logits_all` explicitly require them. - Skip logit copies entirely for intermediate prompt evaluations (e.g., before hybrid checkpoints). - Update logit retrieval to use `get_logits_ith(-1)` to accurately fetch the final token's logits when copying is required. In a PDF-reading summarization workload, this reduced the end-to-end completion time from 41.32s to 25.93s, a ~37.2% improvement. The main generation hot path also improved noticeably: - `_create_completion`: 41.32s -> 25.93s - `generate`: 37.82s -> below the top sampled entries - `eval`: 35.14s -> 21.96s - logits retrieval/copy path: 29.89s `get_logits()` -> 18.68s `get_logits_ith()` - `decode`: 3.89s -> 2.25s - `detokenize`: 2.60s -> 1.33s - `sample`: 2.35s -> 2.03s This significantly reduces CPU overhead and memory bandwidth during generation, as the native `llama.cpp` sampler reads directly from the C context without needing to expose the `n_vocab` array to Python on every token. Signed-off-by: JamePeng --- llama_cpp/llama.py | 52 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 734485802e..e9d16438e5 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1035,11 +1035,20 @@ def eval( tokens: Sequence[int], active_loras: Optional[List[Dict[str, Union[str, float]]]] = None, control_vector: Optional[Dict[str, Any]] = None, + copy_logits: bool = True, ): """Evaluate a list of tokens. Args: - tokens: The list of tokens to evaluate. + tokens: The token ids to evaluate. + active_loras: Optional LoRA adapters to apply for this evaluation. + Each item should contain a ``name`` and an optional ``scale``. + control_vector: Optional control vector configuration to apply during + this evaluation. + copy_logits: Whether to copy the final logits into ``self.scores`` when + ``logits_all`` is disabled. Set to ``False`` for native sampler paths + that sample directly from the llama context and do not need + Python-side logits. """ n_eval = len(tokens) if n_eval == 0: @@ -1246,9 +1255,11 @@ def eval( if self.verbose: print(f"Llama.eval: [Periodic Checkpoint] HybridCheckpoint save failed at pos {current_pos}, skipping update", file=sys.stderr) - # Save the final logit if not in _logits_all mode - if not self._logits_all: - logits_ptr = self._ctx.get_logits() + # Save the final logits only when Python-side logits are required. + # Native sampler can sample directly from ctx, so normal generation does not + # need to copy n_vocab floats into self.scores on every token. + if not self._logits_all and copy_logits: + logits_ptr = self._ctx.get_logits_ith(-1) logits_view = np.ctypeslib.as_array(logits_ptr, shape=(self._n_vocab,)) self.scores[0, :] = logits_view @@ -1666,6 +1677,14 @@ def adapter(token_data_array: llama_cpp_lib.llama_token_data_array): self._sampling_ctx = LlamaSamplingContext(params, self._model) + # Native sampler samples directly from ctx. Python-side logits are only needed + # for compatibility hooks that explicitly consume self._scores. + copy_logits = ( + self._logits_all + or logits_processor is not None + or stopping_criteria is not None + ) + sample_idx = self.n_tokens + len(tokens) - 1 tokens = list(tokens) @@ -1685,8 +1704,13 @@ def adapter(token_data_array: llama_cpp_lib.llama_token_data_array): body_tokens = tokens[:-1] last_token = [tokens[-1]] - # 1. Evaluate up to N-1 - self.eval(body_tokens, active_loras=active_loras, control_vector=control_vector) + # 1. Evaluate up to N-1 without copying logits. + self.eval( + body_tokens, + active_loras=active_loras, + control_vector=control_vector, + copy_logits=False, + ) # 2. Save the N-1 state snapshot current_history = self._input_ids[:self.n_tokens].tolist() @@ -1695,11 +1719,21 @@ def adapter(token_data_array: llama_cpp_lib.llama_token_data_array): tokens=current_history, seq_id=0 ) - # 3. Evaluate the final token to refresh logits - self.eval(last_token, active_loras=active_loras, control_vector=control_vector) + # 3. Evaluate final token. Copy logits only if Python-side hooks need them. + self.eval( + last_token, + active_loras=active_loras, + control_vector=control_vector, + copy_logits=copy_logits, + ) else: # Standard evaluation or single-token generation step - self.eval(tokens, active_loras=active_loras, control_vector=control_vector) + self.eval( + tokens, + active_loras=active_loras, + control_vector=control_vector, + copy_logits=copy_logits, + ) # Sample loop while sample_idx < self.n_tokens: From 4d50e5860798ac4e1706e0de250e01c298a0f126 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 16:34:00 +0800 Subject: [PATCH 133/304] =?UTF-8?q?docs(CUDA):=20Add=20note=20about=20PDL?= =?UTF-8?q?=20optimization=20for=20newer=20NVIDIA=20GPUs=20(CC=20=E2=89=A5?= =?UTF-8?q?=2090)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: JamePeng --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 1986a6ca54..cc83e9814c 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,9 @@ $env:CMAKE_ARGS = "-DGGML_CUDA=on" pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" ``` +Note: **Programmatic Dependent Launch (PDL)** is a CUDA optimization for newer NVIDIA GPUs (CC >= 90; does not include Ada). +It enables stream-level dependency-driven concurrent execution of CUDA kernels within the same stream, achieving similar kernel launch overhead reduction as CUDA Graphs. If you have a newer NVIDIA GPU (e.g. `Hoppper`, `Blackwell` and above), you can achieve significant speedups and latency reduction in token generation across nearly all models when compiling with ` -DGGML_CUDA_PDL=ON`. + **Pre-built Wheel (New)** It is also possible to install a pre-built wheel with CUDA support. Make sure your system meets the following requirements: From 8a107375f0e4e2d482ce64ad1e886ffb6ac5df37 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 16:51:25 +0800 Subject: [PATCH 134/304] docs(development): add AI agent prompt for git commit generation Introduce `git-commit-generation-agent.md` to the development wiki to standardize the creation of high-quality git commit messages using LLM assistants. - Define the system persona, core principles (Conventional Commits, DCO), and strict formatting rules for generating commits. - Provide concrete template examples for build, performance, and documentation updates. - Ensure future maintainers and contributors can easily generate consistent, maintainer-level commits that explicitly explain the "Why" and "How" of code changes. Signed-off-by: JamePeng --- .../git-commit-generation-agent.md | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 docs/wiki/development/git-commit-generation-agent.md diff --git a/docs/wiki/development/git-commit-generation-agent.md b/docs/wiki/development/git-commit-generation-agent.md new file mode 100644 index 0000000000..4cce635154 --- /dev/null +++ b/docs/wiki/development/git-commit-generation-agent.md @@ -0,0 +1,214 @@ +--- +title: Git Commit Generation Agent +page_type: development-helper +source_file: docs/wiki/development/git-commit-generation-agent.md +last_updated: 2026-05-23 +version_target: "latest" +author: JamePeng +audience: maintainers +--- + +# Git Commit Generation Agent for `llama-cpp-python` + +## Overview + +This page defines a maintainer-facing LLM helper workflow for generating +high-quality, descriptive, and standardized Git commit messages for +`llama-cpp-python`. + +## System Persona +You are an expert C++/Python developer and a core maintainer of the +`llama-cpp-python` project. Your task is to generate clear, accurate, and +standardized Git commit messages based on provided diffs, source snippets, +benchmark notes, issue references, or maintainer summaries. + +## Core Principles + +The project follows the **Conventional Commits** specification and requires a +**Developer Certificate of Origin (DCO) Sign-off**. + +Generated commit messages must prioritize: + +- **Why** the change was needed. +- **How** the change was implemented. +- **What** user-visible, runtime, build, packaging, or documentation behavior + changed. +- **What** future maintainers need to know when reading the project history. + +## Input Requirements + +The agent may receive: + +- A full Git diff +- A changed file list +- Source snippets +- Benchmark results +- Maintainer notes +- Issue or PR references +- A natural-language summary of changes + +When the input is incomplete, generate the best possible commit message from the +provided information, but do not invent implementation details. + +## Formatting Rules + +### 1. Header Line (Subject) +Use the following format: + +```text +(): +```` + +Allowed types: + +| Type | Use for | +| ---------- | ----------------------------------------------------------- | +| `feat` | New features or user-facing capabilities | +| `fix` | Bug fixes | +| `docs` | Documentation-only changes | +| `build` | CMake, build scripts, compiler flags, packaging build logic | +| `perf` | Performance optimizations | +| `ci` | GitHub Actions or other workflow changes | +| `chore` | Maintenance, cleanup, or non-user-facing changes | +| `refactor` | Internal restructuring without behavior change | +| `test` | Test additions or updates | + +Recommended scopes: + +* `llama` +* `core` +* `bindings` +* `sampling` +* `speculative` +* `cache` +* `chat` +* `multimodal` +* `embedding` +* `types` +* `cmake` +* `windows` +* `cuda` +* `metal` +* `ci` +* `docs` +* `readme` +* `packaging` + +Subject rules: + +* Use imperative mood, such as `add`, `fix`, `update`, `skip`, `expose`. +* Do not use past tense, such as `added`, `fixed`, or `updated`. +* Keep the subject under 72 characters when possible. +* Use lowercase unless a proper noun, symbol, or API name requires otherwise. +* Do not end the subject with a period. + +### 2. Body +Leave one blank line between the header and the body. +The body should: +* Start with a short paragraph explaining the motivation or problem. +* Use bullets when the diff contains multiple logical changes. +* Mention important files, classes, functions, flags, or APIs using Markdown + backticks. +* Keep lines wrapped at around 72-80 characters. +* Mention user-visible behavior changes when relevant. +* Mention performance impact only when supported by the input. + +### 3. Footer (Sign-off) +* Leave one blank line after the body. +* You MUST append a generic DCO sign-off line at the very end. +* **Format:** `Signed-off-by: Developer Name ` + +--- + +## Accuracy Rules + +* Do not invent changed files, functions, APIs, benchmarks, flags, or behavior. +* Do not claim performance improvements unless benchmark data is provided or the + diff clearly supports the optimization. +* Do not mention issue or PR numbers unless provided by the user. +* Do not include migration notes unless the change affects user-facing APIs. +* If the change is documentation-only, do not imply runtime behavior changed. +* If the change is internal-only, do not overstate it as a user-facing feature. +* Prefer specific technical descriptions over generic wording. + +## Output Rules + +When the user provides a code diff or a summary of changes, analyze the intent +and output only the raw Git commit message. + +Do not: + +* Wrap the commit message in Markdown code fences. +* Add explanations before or after the commit message. +* Add headings such as `Commit message:`. +* Include alternative versions unless explicitly requested. + +## Output Examples + +### Example 1: Build System Change +```text +build(cmake): package LLVM OpenMP runtime DLL for Windows wheels + +Dynamically loaded GGML CPU backends compiled with LLVM/Clang and OpenMP +require `libomp140.x86_64.dll` at runtime. Since this dependency is not +always caught by `$`, it must be packaged manually. + +- Add `llama_cpp_python_install_windows_runtime_file` to handle installing + arbitrary extra DLLs with proper CMake path normalization. +- Add fallback search logic to locate the OpenMP DLL in common Visual Studio + directories. +- Execute the installation before the dev-file cleanup step to ensure the + DLL is correctly packaged in the final Python wheel. + +Signed-off-by: Developer Name + +``` + +### Example 2: Performance Optimization + +```text +perf(eval): skip unnecessary logit array copies during native sampling + +Introduce a `copy_logits` flag to `Llama.eval()` to control whether C-level +logits are copied into the Python `self.scores` array. + +- Automatically disable `copy_logits` during the generation loop unless + Python-side hooks (`logits_processor`, `stopping_criteria`) explicitly + require them. +- Update logit retrieval to use `get_logits_ith(-1)` to accurately fetch + the final token's logits when copying is required. + +This significantly reduces CPU overhead and memory bandwidth during generation, +as the native `llama.cpp` sampler reads directly from the C context without +needing to expose the `n_vocab` array to Python on every token. + +Signed-off-by: Developer Name + +``` + +### Example 3: Documentation Update + +```text +docs(speculative): document n-gram map k/k4v modes and new parameters + +Reflect the recent architectural upgrades to `LlamaNGramMapDecoding` in +the official documentation. + +- Document the new `__init__` parameters (`mode`, `min_hits`, + `max_entries_per_key`) and their validation rules. +- Add a detailed comparison table explaining the memory and behavior + differences between the `"k"` and `"k4v"` lookup modes. +- Add a strong production warning against the legacy `LlamaPromptLookupDecoding` + implementation. + +Signed-off-by: Developer Name + +``` + +## Execution + +When the user provides a code diff or a summary of changes, analyze the intent and output ONLY the raw Git commit message following the exact structure and tone demonstrated above. + +## Related Links + +* [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] From 1b0ae7097688a0c328f5c4149afa7b9f519318fd Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 23 May 2026 17:14:35 +0800 Subject: [PATCH 135/304] docs(wiki): add development helper to index Introduce the development section in the wiki index so maintainer-facing workflows and LLM-assisted helper tools are discoverable from the main navigation. - Add a Development section with a link to the Git commit generation agent. Include the helper in the recommended reading order for new wiki users. - Add development/git-commit-generation-agent.md to the available pages list. Signed-off-by: JamePeng jame_peng@sina.com --- docs/wiki/index.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/wiki/index.md b/docs/wiki/index.md index 143d6e629b..c721fc4e89 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -34,6 +34,18 @@ These pages document major source modules and related classes. --- +### Development + +This section contains maintainer-facing development notes, workflows, and LLM-assisted helper tools for working on `llama-cpp-python`. + +#### Pages + +| Page | Description | +|---|---| +| [[development/Git Commit Generation Agent]] | Helper workflow for generating clear, structured, and source-aware Git commit messages. | + +--- + ### Wiki Maintenance These pages define how the wiki should be written, updated, and reviewed. @@ -55,9 +67,9 @@ If you are new to this wiki, read the pages in this order: 4. [[modules/LlamaGrammar|Llama Grammar](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaGrammar.md)] 5. [[modules/LlamaSpeculative|Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] 6. [[modules/Logger\|Logger](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/Logger.md)] +7. [[development/Git Commit Generation Agent](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/development/git-commit-generation-agent.md)] If you are contributing documentation, start with: - 1. [[SCHEMA|Wiki Schema](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/SCHEMA.md)] 2. [[contributing-to-wiki|Contributing to the Wiki](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/contributing-to-wiki.md)] @@ -75,6 +87,7 @@ Currently available pages: - `modules/LlamaGrammar.md` - `modules/LlamaSpeculative.md` - `modules/Logger.md` +- `development/git-commit-generation-agent.md` - `SCHEMA.md` - `contributing-to-wiki.md` From 0239328f3f22ba87fd74351d96c5c65f6c95f95a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 25 May 2026 19:55:26 +0800 Subject: [PATCH 136/304] Update Submodule vendor/llama.cpp 1acee6b..328874d Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 1 + vendor/llama.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index ec2b665a16..238a1a4fe1 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -2804,6 +2804,7 @@ def llama_state_seq_load_file( LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY = 1 # // keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load) +# // Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag. LLAMA_STATE_SEQ_FLAGS_ON_DEVICE = 2 llama_state_seq_flags = ctypes.c_uint32 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1acee6bf89..328874d054 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1acee6bf8939948f9bcbf4b14034e4b475f06069 +Subproject commit 328874d054e0eb44591202a23c209cf02c18e3cb From a32daf797a5e1aea527c8957da1f25f631ba98e9 Mon Sep 17 00:00:00 2001 From: Jay0360 Date: Wed, 27 May 2026 21:12:15 +0800 Subject: [PATCH 137/304] fix: wire LFM VL chat handlers into server loader --- llama_cpp/server/model.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 37c5195687..6b3fd1dd15 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -199,6 +199,34 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = llama_cpp.llama_chat_format.Qwen25VLChatHandler( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) + elif settings.chat_format == "lfm2-vl": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.LFM2VLChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.LFM2VLChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) + elif settings.chat_format == "lfm2.5-vl": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.LFM25VLChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.LFM25VLChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) elif settings.chat_format == "hf-autotokenizer": assert ( settings.hf_pretrained_model_name_or_path is not None From d9cc25bcb4e563eed910454f6fb5faa5b736124a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 27 May 2026 22:10:47 +0800 Subject: [PATCH 138/304] Update Submodule vendor/llama.cpp 328874d..617255d Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 328874d054..617255d437 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 328874d054e0eb44591202a23c209cf02c18e3cb +Subproject commit 617255d437898fcef6c3d80d4994b307454da850 From 4a6c311364ca3463619c107d37e0ae8a4c0cd98b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 28 May 2026 00:55:17 +0800 Subject: [PATCH 139/304] refactor(internals): align model metadata wrappers with llama.cpp API - Use `llama_vocab_n_tokens()` instead of the old vocab size helper. - Add Python wrappers for model description, size, chat template, and trained RoPE frequency scaling. - Clarify model capability helpers with docstrings matching llama.cpp semantics. - Rename `desc()` and `size()` to `model_desc()` and `model_size()` to make their scope explicit. - Drop the unused `get_tensor()` stub since llama.cpp does not expose it. - Route rerank template lookup through `LlamaModel.model_chat_template()` for consistency with the internal model abstraction. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 65 +++++++++++++++++++++++++++--------- llama_cpp/llama.py | 9 ++++- llama_cpp/llama_embedding.py | 4 +-- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index fda9187855..5416ce2416 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -102,7 +102,7 @@ def vocab_type(self) -> int: return llama_cpp.llama_vocab_type(self.model) def n_vocab(self) -> int: - return llama_cpp.llama_n_vocab(self.vocab) + return llama_cpp.llama_vocab_n_tokens(self.vocab) def n_ctx_train(self) -> int: return llama_cpp.llama_model_n_ctx_train(self.model) @@ -131,41 +131,76 @@ def n_head_kv(self) -> int: def n_swa(self) -> int: return llama_cpp.llama_model_n_swa(self.model) + def rope_freq_scale_train(self) -> float: + """ + Get the model's RoPE frequency scaling factor + """ + return llama_cpp.llama_model_rope_freq_scale_train(self.model) + + def model_desc(self) -> str: + """ + Get a string describing the model type + """ + buf = ctypes.create_string_buffer(256) + llama_cpp.llama_model_desc(self.model, buf, 256) + return buf.value.decode("utf-8") + + def model_size(self) -> int: + """ + Returns the total size of all the tensors in the model in bytes + """ + return llama_cpp.llama_model_size(self.model) + + def model_chat_template(self, name: bytes) -> str: + """ + Get the default chat template. Returns nullptr if not available + If name is NULL, returns the default chat template + """ + return llama_cpp.llama_model_chat_template(self.model, name).decode("utf-8") + def n_params(self) -> int: + """ + Returns the total number of parameters in the model + """ return llama_cpp.llama_model_n_params(self.model) def has_encoder(self) -> bool: + """ + Returns true if the model contains an encoder that requires llama_encode() call + """ return llama_cpp.llama_model_has_encoder(self.model) def has_decoder(self) -> bool: + """ + Returns true if the model contains a decoder that requires llama_decode() call + """ return llama_cpp.llama_model_has_decoder(self.model) def decoder_start_token(self) -> int: + """ + For encoder-decoder models, this function returns id of the token that must be provided + to the decoder to start generating output sequence. For other models, it returns -1. + """ return llama_cpp.llama_model_decoder_start_token(self.model) def is_recurrent(self) -> bool: + """ + Returns true if the model is recurrent (like Mamba, RWKV, etc.) + """ return llama_cpp.llama_model_is_recurrent(self.model) def is_hybrid(self) -> bool: + """ + Returns true if the model is hybrid (like Jamba, Granite, etc.) + """ return llama_cpp.llama_model_is_hybrid(self.model) def is_diffusion(self) -> bool: + """ + Returns true if the model is diffusion-based (like LLaDA, Dream, etc.) + """ return llama_cpp.llama_model_is_diffusion(self.model) - def rope_freq_scale_train(self) -> float: - return llama_cpp.llama_model_rope_freq_scale_train(self.model) - - def desc(self) -> str: - buf = ctypes.create_string_buffer(1024) - llama_cpp.llama_model_desc(self.model, buf, 1024) - return buf.value.decode("utf-8") - - def size(self) -> int: - return llama_cpp.llama_model_size(self.model) - - def get_tensor(self, name: str) -> ctypes.c_void_p: - raise NotImplementedError("get_tensor is not implemented in llama.cpp") - # Vocab def token_get_text(self, token: int) -> str: diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index e9d16438e5..c2d2757e13 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -696,13 +696,20 @@ def __init__( try: self.metadata = self._model.metadata() + self.model_desc = self._model.model_desc() + # The total size of all the tensors in the model in bytes + self.model_size = self._model.model_size() + except Exception as e: self.metadata = {} if self.verbose: print(f"Failed to load metadata: {e}", file=sys.stderr) if self.verbose: - print(f"Model metadata: {self.metadata}", file=sys.stderr) + print(f"Model desc: {self.model_desc}, " + f"Model size: {self.model_size / (1024 * 1024):.2f} MB, " + f"Model metadata: {self.metadata}", + file=sys.stderr) eos_token_id = self.token_eos() bos_token_id = self.token_bos() diff --git a/llama_cpp/llama_embedding.py b/llama_cpp/llama_embedding.py index 7c8ad1e90f..0c1df339ce 100644 --- a/llama_cpp/llama_embedding.py +++ b/llama_cpp/llama_embedding.py @@ -303,9 +303,7 @@ def rank(self, query: str, documents: List[str]) -> List[float]: # 1. Attempt to retrieve the built-in 'rerank' chat template from model metadata. # Modern GGUF models often include a template for formatting query/document pairs. - rerank_template = llama_cpp.llama_model_chat_template(self._model.model, b"rerank") - if rerank_template: - rerank_template = rerank_template.decode("utf-8") + rerank_template = self._model.model_chat_template(b"rerank") batch_inputs: List[List[int]] = [] From 677db7b0d5b834ae3d3831af4702ec21986ab335 Mon Sep 17 00:00:00 2001 From: Alcoft Date: Thu, 28 May 2026 00:12:35 +0200 Subject: [PATCH 140/304] Resolve file conflicts. --- .github/workflows/build-wheels-cu131-win.yml | 25 -------------------- 1 file changed, 25 deletions(-) diff --git a/.github/workflows/build-wheels-cu131-win.yml b/.github/workflows/build-wheels-cu131-win.yml index 5f77003a5f..14bea65d19 100644 --- a/.github/workflows/build-wheels-cu131-win.yml +++ b/.github/workflows/build-wheels-cu131-win.yml @@ -67,31 +67,6 @@ jobs: echo LIB=%LIB%>>%GITHUB_ENV% echo LIBPATH=%LIBPATH%>>%GITHUB_ENV% - - name: Copy LLVM OpenMP runtime - shell: pwsh - run: | - # GGML CPU all-variant backends are built with LLVM OpenMP on Windows. - # The dynamically loaded ggml-cpu-*.dll files depend on this runtime. - # If it is missing from the wheel, ggml_backend_load_all_from_path() - # may fail to load CPU backend DLLs at runtime. - $packageLibDir = Join-Path $env:GITHUB_WORKSPACE "llama_cpp\lib" - New-Item -ItemType Directory -Force $packageLibDir | Out-Null - - $omp = Get-ChildItem "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Redist\MSVC" ` - -Recurse ` - -Filter "libomp140.x86_64.dll" ` - -ErrorAction SilentlyContinue | - Where-Object { $_.FullName -match "OpenMP\.LLVM" } | - Select-Object -First 1 - - if (!$omp) { - Write-Error "Could not find libomp140.x86_64.dll in Visual Studio LLVM OpenMP redistributables." - exit 1 - } - - Copy-Item $omp.FullName (Join-Path $packageLibDir "libomp140.x86_64.dll") -Force - Write-Output "Copied LLVM OpenMP runtime: $($omp.FullName)" - - name: Build wheel run: | $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.', '') From 4794c8c20ee731838cbc2c8d601ccb2c245d6893 Mon Sep 17 00:00:00 2001 From: Alcoft Date: Thu, 28 May 2026 01:52:48 +0200 Subject: [PATCH 141/304] Added support when using the keyword 'audio' instead of 'audio_url'. --- llama_cpp/llama_chat_format.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index f9b9d52367..254195f95a 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2996,13 +2996,13 @@ def _get_media_items(self, messages: List[llama_types.ChatCompletionRequestMessa media_items.append({"url": url, "type": "image"}) # 2. Audio Processing - elif content_type in ["audio_url", "input_audio"]: + elif content_type in ["audio", "audio_url", "input_audio"]: if not self.is_support_audio: raise ValueError(f"{self.log_prefix}: This mmproj model instance does not support audio inputs.") # Case A: Handle custom/forward-compatible audio_url format - if content_type == "audio_url": - audio_url = content["audio_url"] + if content_type == "audio_url" or content_type == "audio": + audio_url = content[content_type] url = audio_url if isinstance(audio_url, str) else audio_url["url"] media_items.append({"url": url, "type": "audio"}) # Case B: Handle OpenAI standard input_audio format From 103639ce04b72d09e09ce895f3c8d8cfba518e13 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 28 May 2026 21:48:01 +0800 Subject: [PATCH 142/304] Update Submodule vendor/llama.cpp 617255d..6ed481e Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 617255d437..6ed481eea4 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 617255d437898fcef6c3d80d4994b307454da850 +Subproject commit 6ed481eea4cf4ed40777db2fa29e8d08eb712b3b From 6c9e7bf92c346806f91ef06f2522b0def7611f10 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 29 May 2026 22:51:33 +0800 Subject: [PATCH 143/304] feat(chat_handler): update multimodal handlers for Qwen2.5-VL, Qwen3-VL, and PaddleOCR - Update PaddleOCRChatHandler to support version 1.6 - Add token configuration and stop sequences for Qwen2.5-VL and Qwen3-VL - Standardize input_ids initialization in __call__ methods for Qwen2.5-VL, Qwen3-ASR, and Qwen3-VL handlers Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 0365d8f871..cf5dca2492 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -5324,7 +5324,7 @@ def __call__(self, **kwargs): class PaddleOCRChatHandler(MTMDChatHandler): """ - Handler for PaddleOCR 1.5 multimodal models. + Handler for PaddleOCR 1.5/1.6 multimodal models. """ PADDLEOCR_CLS_TOKEN = "<|begin_of_sentence|>" @@ -5431,6 +5431,11 @@ def __call__(self, **kwargs): class Qwen25VLChatHandler(MTMDChatHandler): + + QWEN25_VL_BOS_TOKEN = "<|endoftext|>" + QWEN25_VL_PAD_TOKEN = "<|endoftext|>" + QWEN25_VL_EOS_TOKEN = "<|im_end|>" + CHAT_FORMAT = ( "{% set image_count = namespace(value=0) %}" "{% for message in messages %}" @@ -5462,6 +5467,8 @@ class Qwen25VLChatHandler(MTMDChatHandler): ) def __call__(self, **kwargs): + kwargs['stop'] = [self.QWEN25_VL_EOS_TOKEN, self.QWEN25_VL_PAD_TOKEN] + llama = kwargs['llama'] if hasattr(llama, 'input_ids'): @@ -5547,12 +5554,22 @@ def __call__(self, **kwargs): # Qwen3 models universally use `<|endoftext|>` and `<|im_end|>` as the stop token kwargs['stop'] = [self.QWEN3_ASR_AUDIO_PAD_TOKEN, self.QWEN3_ASR_AUDIO_EOS_TOKEN] + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + if self.verbose: print(f"{self.log_prefix} - Start processing Qwen3-ASR (Audio Only)") return super().__call__(**kwargs) class Qwen3VLChatHandler(MTMDChatHandler): + + QWEN3_VL_BOS_TOKEN = "<|endoftext|>" + QWEN3_VL_PAD_TOKEN = "<|endoftext|>" + QWEN3_VL_EOS_TOKEN = "<|im_end|>" + CHAT_FORMAT = ( "{{- '<|im_start|>system\n' -}}" "{%- if messages[0].content is string and messages[0].role == 'system' -%}" @@ -5661,6 +5678,8 @@ def __init__( self.extra_template_arguments["add_vision_id"] = add_vision_id def __call__(self, **kwargs): + kwargs['stop'] = [self.QWEN3_VL_EOS_TOKEN, self.QWEN3_VL_PAD_TOKEN] + llama = kwargs['llama'] if hasattr(llama, 'input_ids'): From 69e740ce51b064be36fa5e28214839429f89c94e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 30 May 2026 01:21:11 +0800 Subject: [PATCH 144/304] Update Submodule vendor/llama.cpp 6ed481e..06d26df Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6ed481eea4..06d26dfdff 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6ed481eea4cf4ed40777db2fa29e8d08eb712b3b +Subproject commit 06d26dfdff4097dc51eac20155371a9cfd53e094 From e7976f42b23ce29491d1b48bd044682ce4f261a2 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 30 May 2026 01:24:56 +0800 Subject: [PATCH 145/304] feat(mtmd): improve fallback chat template for multimodal models - Add BOS/EOS token handling to the default MTMD chat format. - Use a clearer role-based template with explicit USER and ASSISTANT prefixes. - Append a newline after each message to keep generated prompts readable. - Treat EOS as the end marker for the serialized conversation history before the optional generation prompt. - Improve fallback behavior for multimodal GGUF models that do not provide a chat template, such as OCR-oriented models like DeepSeek-OCR 1/2. - Make the default system prompt a single normalized string while preserving its original meaning. - Clean up minor formatting around MTMD context parameter initialization. This improves prompt compatibility for multimodal models that either lack a GGUF chat template or are not yet covered by a complete custom chat handler. Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index cf5dca2492..71228d0627 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2811,21 +2811,20 @@ def generate_streaming(tools, functions, function_call, prompt): class MTMDChatHandler: DEFAULT_SYSTEM_MESSAGE: Optional[str] = ( -"""You are an exceptionally capable, precise, and helpful multimodal AI assistant that excels at deeply understanding and richly describing images, charts, diagrams, text in images, scenes, and any visual content, -while also answering every question accurately, clearly, and step-by-step when appropriate ā€” always responding in the same language as the user's question, remaining polite, professional, and maximally helpful.""" +"You are an exceptionally capable, precise, and helpful multimodal AI assistant that excels at deeply understanding and richly describing images, charts, diagrams, text in images, scenes, and any visual content, " +"while also answering every question accurately, clearly, and step-by-step when appropriate ā€” always responding in the same language as the user's question, remaining polite, professional, and maximally helpful." ) CHAT_FORMAT = ( + "{{ bos_token if bos_token is defined else '' }}" "{% for message in messages %}" "{% if message.role == 'system' %}" "{{ message.content }}" - "{% endif %}" - - "{% if message.role == 'user' %}" + "{% elif message.role == 'user' %}" + "USER: " "{% if message.content is string %}" - "\nUSER: {{ message.content }}" + "{{ message.content }}" "{% elif message.content is iterable %}" - "\nUSER: " "{% for content in message.content %}" "{% if content.type == 'image_url' %}" "{{ content.image_url if content.image_url is string else content.image_url.url }}" @@ -2842,15 +2841,19 @@ class MTMDChatHandler: "{% endif %}" "{% endfor %}" "{% endif %}" - "{% endif %}" - "{% if message.role == 'assistant' and message.content is not none %}" - "\nASSISTANT: {{ message.content }}" + "{% elif message.role == 'assistant' and message.content is not none %}" + "ASSISTANT: {{ message.content }}" "{% endif %}" + "{{ \"\n\" }}" "{% endfor %}" + "{% if eos_token is defined %}" + "{{ eos_token }}" + "{% endif %}" + "{% if add_generation_prompt %}" - "\nASSISTANT: " + "ASSISTANT: " "{% endif %}" ) @@ -2906,7 +2909,7 @@ def _init_mtmd_context(self, llama_model: llama_core.Llama): self.mctx_params.use_gpu = self.use_gpu self.mctx_params.print_timings = self.verbose self.mctx_params.n_threads = llama_model.n_threads - self.mctx_params.flash_attn_type = self._mtmd_cpp.clip_flash_attn_type.CLIP_FLASH_ATTN_TYPE_AUTO + self.mctx_params.flash_attn_type = self._mtmd_cpp.clip_flash_attn_type.CLIP_FLASH_ATTN_TYPE_AUTO self.mctx_params.warmup = True if self.image_min_tokens > 0: self.mctx_params.image_min_tokens = self.image_min_tokens From 1df7ffc07b7a8f52000614d9f63a90f8b80f0d6f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 30 May 2026 02:16:12 +0800 Subject: [PATCH 146/304] docs(Readme): Update Deepseek-OCR-2-GGUF Link Signed-off-by: JamePeng --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index cc83e9814c..c39df4abd7 100644 --- a/README.md +++ b/README.md @@ -953,6 +953,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [granite-docling](https://huggingface.co/ibm-granite/granite-docling-258M-GGUF) | `GraniteDoclingChatHandler` | `granite-docling` | | [lfm2-vl](https://huggingface.co/LiquidAI/LFM2-VL-3B-GGUF) | `LFM2VLChatHandler` | `lfm2-vl` | | [lfm2.5-vl](https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B-GGUF) | `LFM25VLChatHandler` | `lfm2.5-vl` | +| [deepseek-ocr](https://huggingface.co/JamePeng2023/DeepSeek-OCR-2-GGUF) | `MTMDChatHandler` | `None` | | [paddleocr-vl-1.5](https://huggingface.co/JamePeng2023/PaddleOCR-VL-1.5-GGUF) | `PaddleOCRChatHandler` | `paddleocr` | | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | | [qwen3-asr](https://huggingface.co/JamePeng2023/Qwen3-ASR-1.7B-GGUF) | `Qwen3ASRChatHandler` | `qwen3-asr` | From c4efcff5c1534e0a3946809bec6d0e97e374bf4a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 31 May 2026 19:13:39 +0800 Subject: [PATCH 147/304] Update Submodule vendor/llama.cpp 06d26df..d4c8e2c Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 06d26dfdff..d4c8e2c29c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 06d26dfdff4097dc51eac20155371a9cfd53e094 +Subproject commit d4c8e2c29ce2fb9a251a0a4a16d6c857b4f70f8c From 6a7fde40a2d96bee1da4c004bf3ac0c31b2432d4 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 31 May 2026 19:27:11 +0800 Subject: [PATCH 148/304] ci : update metal build/test job to macos-26/macos-15-intel - Build on the Tahoe runners in order to enable the tensor API for M5 and A19. Signed-off-by: JamePeng --- .github/workflows/build-wheels-metal.yaml | 7 +++---- .github/workflows/test.yaml | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index a809909720..2b00d1abaa 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -8,8 +8,8 @@ permissions: jobs: build_wheels: - name: Build wheels (Metal macos) - runs-on: macos-latest + name: Build wheels (Metal macos-26) + runs-on: macos-26 outputs: version: ${{steps.get_version.outputs.version}} @@ -53,8 +53,7 @@ jobs: -DCMAKE_CROSSCOMPILING=on -DGGML_METAL=on -DGGML_METAL_USE_BF16=on - -DGGML_METAL_EMBED_LIBRARY=off - -DGGML_METAL_SHADER_DEBUG=on" + -DGGML_METAL_EMBED_LIBRARY=on" with: package-dir: . output-dir: wheelhouse2 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 420c5e9495..a9f359d1cd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -28,21 +28,21 @@ jobs: python-version: ["3.9", "3.14"] include: # macOS Non-Metal - - os: macos-14 + - os: macos-15-intel python-version: "3.9" - cmake_args: "-DLLAMA_METAL=off" + cmake_args: "-DLLAMA_METAL=off -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3" metal_status: "(No Metal)" - - os: macos-14 + - os: macos-15-intel python-version: "3.14" - cmake_args: "-DLLAMA_METAL=off" + cmake_args: "-DLLAMA_METAL=off -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3" metal_status: "(No Metal)" # macOS Metal - - os: macos-14 + - os: macos-26 python-version: "3.9" cmake_args: "-DLLAMA_METAL=on -DGGML_METAL_USE_BF16=on -DGGML_METAL_EMBED_LIBRARY=on" metal_status: "(Metal)" - - os: macos-14 + - os: macos-26 python-version: "3.14" cmake_args: "-DLLAMA_METAL=on -DGGML_METAL_USE_BF16=on -DGGML_METAL_EMBED_LIBRARY=on" metal_status: "(Metal)" From ea0907d3870aabbeaf669f42bd1b484a2d7e7c83 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 31 May 2026 20:06:49 +0800 Subject: [PATCH 149/304] refactor(llama_cpp): wrap llama constants into enum.IntEnum - Group global `LLAMA_*` constants into `enum.IntEnum` classes (`llama_vocab_type`, `llama_vocab_pre_type`, `llama_rope_type`, etc.) for better type safety and organization. - Sync new values for `llama_vocab_pre_type` (`SARVAM_MOE`, `MINICPM5`, `WHITESPACE`). Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 245 +++++++++++++++++++++-------------------- 1 file changed, 128 insertions(+), 117 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 238a1a4fe1..62c4c81ef9 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -122,20 +122,21 @@ # LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization # LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming # }; -LLAMA_VOCAB_TYPE_NONE = 0 -"""For models without vocab""" -LLAMA_VOCAB_TYPE_SPM = 1 -"""LLaMA tokenizer based on byte-level BPE with byte fallback""" -LLAMA_VOCAB_TYPE_BPE = 2 -"""GPT-2 tokenizer based on byte-level BPE""" -LLAMA_VOCAB_TYPE_WPM = 3 -"""BERT tokenizer based on WordPiece""" -LLAMA_VOCAB_TYPE_UGM = 4 -"""T5 tokenizer based on Unigram""" -LLAMA_VOCAB_TYPE_RWKV = 5 -"""RWKV tokenizer based on greedy tokenization""" -LLAMA_VOCAB_TYPE_PLAMO2 = 6 -"""PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming""" +class llama_vocab_type(enum.IntEnum): + LLAMA_VOCAB_TYPE_NONE = 0 + """For models without vocab""" + LLAMA_VOCAB_TYPE_SPM = 1 + """LLaMA tokenizer based on byte-level BPE with byte fallback""" + LLAMA_VOCAB_TYPE_BPE = 2 + """GPT-2 tokenizer based on byte-level BPE""" + LLAMA_VOCAB_TYPE_WPM = 3 + """BERT tokenizer based on WordPiece""" + LLAMA_VOCAB_TYPE_UGM = 4 + """T5 tokenizer based on Unigram""" + LLAMA_VOCAB_TYPE_RWKV = 5 + """RWKV tokenizer based on greedy tokenization""" + LLAMA_VOCAB_TYPE_PLAMO2 = 6 + """PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming""" # NOTE: Deprecated and will be removed in the future. (already gone in llama.cpp) @@ -193,58 +194,65 @@ # LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, # LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, # LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, +# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, +# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, +# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, # }; -LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 -LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 -LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2 -LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3 -LLAMA_VOCAB_PRE_TYPE_FALCON = 4 -LLAMA_VOCAB_PRE_TYPE_MPT = 5 -LLAMA_VOCAB_PRE_TYPE_STARCODER = 6 -LLAMA_VOCAB_PRE_TYPE_GPT2 = 7 -LLAMA_VOCAB_PRE_TYPE_REFACT = 8 -LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9 -LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10 -LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11 -LLAMA_VOCAB_PRE_TYPE_OLMO = 12 -LLAMA_VOCAB_PRE_TYPE_DBRX = 13 -LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 -LLAMA_VOCAB_PRE_TYPE_PORO = 15 -LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16 -LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17 -LLAMA_VOCAB_PRE_TYPE_VIKING = 18 -LLAMA_VOCAB_PRE_TYPE_JAIS = 19 -LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20 -LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21 -LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22 -LLAMA_VOCAB_PRE_TYPE_BLOOM = 23 -LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 -LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 -LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 -LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 -LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28 -LLAMA_VOCAB_PRE_TYPE_GPT4O = 29 -LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30 -LLAMA_VOCAB_PRE_TYPE_TRILLION = 31 -LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32 -LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33 -LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34 -LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35 -LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36 -LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37 -LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38 -LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39 -LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40 -LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41 -LLAMA_VOCAB_PRE_TYPE_AFMOE = 42 -LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43 -LLAMA_VOCAB_PRE_TYPE_YOUTU = 44 -LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45 -LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46 -LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47 -LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48 -LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49 -LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50 +class llama_vocab_pre_type(enum.IntEnum): + LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 + LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1 + LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2 + LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3 + LLAMA_VOCAB_PRE_TYPE_FALCON = 4 + LLAMA_VOCAB_PRE_TYPE_MPT = 5 + LLAMA_VOCAB_PRE_TYPE_STARCODER = 6 + LLAMA_VOCAB_PRE_TYPE_GPT2 = 7 + LLAMA_VOCAB_PRE_TYPE_REFACT = 8 + LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9 + LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10 + LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11 + LLAMA_VOCAB_PRE_TYPE_OLMO = 12 + LLAMA_VOCAB_PRE_TYPE_DBRX = 13 + LLAMA_VOCAB_PRE_TYPE_SMAUG = 14 + LLAMA_VOCAB_PRE_TYPE_PORO = 15 + LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16 + LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17 + LLAMA_VOCAB_PRE_TYPE_VIKING = 18 + LLAMA_VOCAB_PRE_TYPE_JAIS = 19 + LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20 + LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21 + LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22 + LLAMA_VOCAB_PRE_TYPE_BLOOM = 23 + LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24 + LLAMA_VOCAB_PRE_TYPE_EXAONE = 25 + LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26 + LLAMA_VOCAB_PRE_TYPE_MINERVA = 27 + LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28 + LLAMA_VOCAB_PRE_TYPE_GPT4O = 29 + LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30 + LLAMA_VOCAB_PRE_TYPE_TRILLION = 31 + LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32 + LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33 + LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34 + LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35 + LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36 + LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37 + LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38 + LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39 + LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40 + LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41 + LLAMA_VOCAB_PRE_TYPE_AFMOE = 42 + LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43 + LLAMA_VOCAB_PRE_TYPE_YOUTU = 44 + LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45 + LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46 + LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47 + LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48 + LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49 + LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50 + LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51 + LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52 + LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53 # // note: these values should be synchronized with ggml_rope @@ -257,12 +265,13 @@ # LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE, # LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION, # }; -LLAMA_ROPE_TYPE_NONE = -1 -LLAMA_ROPE_TYPE_NORM = 0 -LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2 -LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8 -LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE = 40 -LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24 +class llama_rope_type(enum.IntEnum): + LLAMA_ROPE_TYPE_NONE = -1 + LLAMA_ROPE_TYPE_NORM = 0 + LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2 + LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8 + LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24 + LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE = 40 # enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file @@ -274,13 +283,14 @@ # LLAMA_TOKEN_TYPE_UNUSED = 5, # LLAMA_TOKEN_TYPE_BYTE = 6, # }; -LLAMA_TOKEN_TYPE_UNDEFINED = 0 -LLAMA_TOKEN_TYPE_NORMAL = 1 -LLAMA_TOKEN_TYPE_UNKNOWN = 2 -LLAMA_TOKEN_TYPE_CONTROL = 3 -LLAMA_TOKEN_TYPE_USER_DEFINED = 4 -LLAMA_TOKEN_TYPE_UNUSED = 5 -LLAMA_TOKEN_TYPE_BYTE = 6 +class llama_token_type(enum.IntEnum): + LLAMA_TOKEN_TYPE_UNDEFINED = 0 + LLAMA_TOKEN_TYPE_NORMAL = 1 + LLAMA_TOKEN_TYPE_UNKNOWN = 2 + LLAMA_TOKEN_TYPE_CONTROL = 3 + LLAMA_TOKEN_TYPE_USER_DEFINED = 4 + LLAMA_TOKEN_TYPE_UNUSED = 5 + LLAMA_TOKEN_TYPE_BYTE = 6 # enum llama_token_attr { @@ -355,45 +365,46 @@ # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; -LLAMA_FTYPE_ALL_F32 = 0 -LLAMA_FTYPE_MOSTLY_F16 = 1 -LLAMA_FTYPE_MOSTLY_Q4_0 = 2 -LLAMA_FTYPE_MOSTLY_Q4_1 = 3 -LLAMA_FTYPE_MOSTLY_Q8_0 = 7 -LLAMA_FTYPE_MOSTLY_Q5_0 = 8 -LLAMA_FTYPE_MOSTLY_Q5_1 = 9 -LLAMA_FTYPE_MOSTLY_Q2_K = 10 -LLAMA_FTYPE_MOSTLY_Q3_K_S = 11 -LLAMA_FTYPE_MOSTLY_Q3_K_M = 12 -LLAMA_FTYPE_MOSTLY_Q3_K_L = 13 -LLAMA_FTYPE_MOSTLY_Q4_K_S = 14 -LLAMA_FTYPE_MOSTLY_Q4_K_M = 15 -LLAMA_FTYPE_MOSTLY_Q5_K_S = 16 -LLAMA_FTYPE_MOSTLY_Q5_K_M = 17 -LLAMA_FTYPE_MOSTLY_Q6_K = 18 -LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19 -LLAMA_FTYPE_MOSTLY_IQ2_XS = 20 -LLAMA_FTYPE_MOSTLY_Q2_K_S = 21 -LLAMA_FTYPE_MOSTLY_IQ3_XS = 22 -LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23 -LLAMA_FTYPE_MOSTLY_IQ1_S = 24 -LLAMA_FTYPE_MOSTLY_IQ4_NL = 25 -LLAMA_FTYPE_MOSTLY_IQ3_S = 26 -LLAMA_FTYPE_MOSTLY_IQ3_M = 27 -LLAMA_FTYPE_MOSTLY_IQ2_S = 28 -LLAMA_FTYPE_MOSTLY_IQ2_M = 29 -LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 -LLAMA_FTYPE_MOSTLY_IQ1_M = 31 -LLAMA_FTYPE_MOSTLY_BF16 = 32 -# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 -# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 -# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 -LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 -LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 -LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 -LLAMA_FTYPE_MOSTLY_NVFP4 = 39 -LLAMA_FTYPE_MOSTLY_Q1_0 = 40 -LLAMA_FTYPE_GUESSED = 1024 +class llama_ftype(enum.IntEnum): + LLAMA_FTYPE_ALL_F32 = 0 + LLAMA_FTYPE_MOSTLY_F16 = 1 + LLAMA_FTYPE_MOSTLY_Q4_0 = 2 + LLAMA_FTYPE_MOSTLY_Q4_1 = 3 + LLAMA_FTYPE_MOSTLY_Q8_0 = 7 + LLAMA_FTYPE_MOSTLY_Q5_0 = 8 + LLAMA_FTYPE_MOSTLY_Q5_1 = 9 + LLAMA_FTYPE_MOSTLY_Q2_K = 10 + LLAMA_FTYPE_MOSTLY_Q3_K_S = 11 + LLAMA_FTYPE_MOSTLY_Q3_K_M = 12 + LLAMA_FTYPE_MOSTLY_Q3_K_L = 13 + LLAMA_FTYPE_MOSTLY_Q4_K_S = 14 + LLAMA_FTYPE_MOSTLY_Q4_K_M = 15 + LLAMA_FTYPE_MOSTLY_Q5_K_S = 16 + LLAMA_FTYPE_MOSTLY_Q5_K_M = 17 + LLAMA_FTYPE_MOSTLY_Q6_K = 18 + LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19 + LLAMA_FTYPE_MOSTLY_IQ2_XS = 20 + LLAMA_FTYPE_MOSTLY_Q2_K_S = 21 + LLAMA_FTYPE_MOSTLY_IQ3_XS = 22 + LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23 + LLAMA_FTYPE_MOSTLY_IQ1_S = 24 + LLAMA_FTYPE_MOSTLY_IQ4_NL = 25 + LLAMA_FTYPE_MOSTLY_IQ3_S = 26 + LLAMA_FTYPE_MOSTLY_IQ3_M = 27 + LLAMA_FTYPE_MOSTLY_IQ2_S = 28 + LLAMA_FTYPE_MOSTLY_IQ2_M = 29 + LLAMA_FTYPE_MOSTLY_IQ4_XS = 30 + LLAMA_FTYPE_MOSTLY_IQ1_M = 31 + LLAMA_FTYPE_MOSTLY_BF16 = 32 + # LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33 + # LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34 + # LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35 + LLAMA_FTYPE_MOSTLY_TQ1_0 = 36 + LLAMA_FTYPE_MOSTLY_TQ2_0 = 37 + LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 + LLAMA_FTYPE_MOSTLY_NVFP4 = 39 + LLAMA_FTYPE_MOSTLY_Q1_0 = 40 + LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { # LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, From ca81fd457969bba20d183d27962e28b45d9207ea Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 31 May 2026 20:33:38 +0800 Subject: [PATCH 150/304] feat: add `ReasoningBudgetState` enum and `TokenMatcher` helper class to _internals.py Introduce `ReasoningBudgetState` enum and `TokenMatcher` helper class to `_internals.py`. This lays the groundwork for the upcoming `ReasoningBudgetSampler`, mirroring the state machine defined in `common/reasoning-budget.h`. - `ReasoningBudgetState`: Tracks the lifecycle of the first reasoning block. - `TokenMatcher`: Handles incremental matching for multi-token sequences. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 5416ce2416..5b5c533c52 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -1265,6 +1265,60 @@ class CommonSamplerType(enum.IntEnum): CUSTOM = 99 + +# common/reasssoning-budget.h +# +# enum common_reasoning_budget_state { +# REASONING_BUDGET_IDLE, // waiting for start sequence +# REASONING_BUDGET_COUNTING, // counting down tokens +# REASONING_BUDGET_FORCING, // forcing budget message + end sequence +# REASONING_BUDGET_WAITING_UTF8, // budget exhausted, waiting for UTF-8 completion +# REASONING_BUDGET_DONE, // passthrough forever +# }; +class ReasoningBudgetState(enum.IntEnum): + """ + State machine for the generic first-reasoning-block budget controller. + + This sampler only controls the first reasoning block. Once the first block + naturally ends or is forcibly closed, the sampler enters DONE and becomes a + permanent passthrough. + """ + + IDLE = 0 # Waiting for the first reasoning_start sequence. + COUNTING = 1 # Counting generated tokens inside the first reasoning block. + FORCING = 2 # Forcing reasoning_budget_message + reasoning_end. + WAITING_UTF8 = 3 # Budget exhausted; waiting for a complete UTF-8 boundary. + DONE = 4 # Permanent passthrough; later reasoning tags are ignored. + + +class TokenMatcher: + """ + Incremental matcher for a multi-token sequence. + Accepts None as tokens to represent no matcher. + """ + def __init__(self, tokens: Optional[Sequence[int]]): + # If None, matcher never matches anything + self.tokens = list(tokens) if tokens is not None else [] + self.pos = 0 + + def advance(self, token: int) -> bool: + if not self.tokens: + return False + if token == self.tokens[self.pos]: + self.pos += 1 + if self.pos >= len(self.tokens): + self.pos = 0 + return True + else: + self.pos = 0 + if token == self.tokens[0]: + self.pos = 1 + return False + + def reset(self) -> None: + self.pos = 0 + + @dataclass class LlamaSamplingParams: seed: int = llama_cpp.LLAMA_DEFAULT_SEED # the seed used to initialize llama_sampler From ab42b8664313a30c390fcf26caaec9602199c0f4 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 31 May 2026 22:48:42 +0800 Subject: [PATCH 151/304] docs(readme): update supported embeddings models table - Add jina-embeddings-v2-base-zh - Add jina-embeddings-v3 - Minor table formatting clean up Signed-off-by: JamePeng --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c39df4abd7..c5aa1a1b26 100644 --- a/README.md +++ b/README.md @@ -1463,7 +1463,9 @@ run_inference( | Model | Type | Link | Status | |--------------------|-----------|--------------------------------------------------------|--------------| -| `bge-m3` | Embedding |[bge-m3-GGUF](https://huggingface.co/gpustack/bge-m3-GGUF) | Useful āœ… | +|`bge-m3`| Embedding |[bge-m3-GGUF](https://huggingface.co/gpustack/bge-m3-GGUF) | Useful āœ… | +|`jina-embeddings-v2-base-zh`| Embedding |[jina-embeddings-v2-base-zh-GGUF](https://huggingface.co/gpustack/jina-embeddings-v2-base-zh-GGUF) | Useful āœ… | +|`jina-embeddings-v3`| Embedding |[jina-embeddings-v3-GGUF](https://huggingface.co/second-state/jina-embeddings-v3-GGUF) | Useful āœ… | |`bge-reranker-v2-m3`| Rerank |[bge-reranker-v2-m3-GGUF](https://huggingface.co/gpustack/bge-reranker-v2-m3-GGUF) | Useful āœ… | |`qwen3-reranker`| Rerank |[Qwen3-Reranker-GGUF](https://huggingface.co/JamePeng2023/Qwen3-Reranker-GGUF) | Useful āœ… | From 90d610ffd7b491603ca23c3b0027629553731658 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 31 May 2026 23:11:38 +0800 Subject: [PATCH 152/304] docs(llama_embedding): update supported embeddings models table - Add jina-embeddings-v2-base-zh - Add jina-embeddings-v3 - Minor table formatting clean up Signed-off-by: JamePeng --- docs/wiki/modules/LlamaEmbedding.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/wiki/modules/LlamaEmbedding.md b/docs/wiki/modules/LlamaEmbedding.md index 1279db5cab..3aa2427227 100644 --- a/docs/wiki/modules/LlamaEmbedding.md +++ b/docs/wiki/modules/LlamaEmbedding.md @@ -3,7 +3,7 @@ title: Llama Embedding module_name: llama_cpp.llama_embedding source_file: llama_cpp/llama_embedding.py class_name: LlamaEmbedding -last_updated: 2026-05-01 +last_updated: 2026-05-31 version_target: "latest" --- @@ -18,7 +18,9 @@ version_target: "latest" | Model | Type | Link | Status | |--------------------|-----------|--------------------------------------------------------|--------------| -| `bge-m3` | Embedding |[bge-m3-GGUF](https://huggingface.co/gpustack/bge-m3-GGUF) | Useful āœ… | +|`bge-m3`| Embedding |[bge-m3-GGUF](https://huggingface.co/gpustack/bge-m3-GGUF) | Useful āœ… | +|`jina-embeddings-v2-base-zh`| Embedding |[jina-embeddings-v2-base-zh-GGUF](https://huggingface.co/gpustack/jina-embeddings-v2-base-zh-GGUF) | Useful āœ… | +|`jina-embeddings-v3`| Embedding |[jina-embeddings-v3-GGUF](https://huggingface.co/second-state/jina-embeddings-v3-GGUF) | Useful āœ… | |`bge-reranker-v2-m3`| Rerank |[bge-reranker-v2-m3-GGUF](https://huggingface.co/gpustack/bge-reranker-v2-m3-GGUF) | Useful āœ… | |`qwen3-reranker`| Rerank |[Qwen3-Reranker-GGUF](https://huggingface.co/JamePeng2023/Qwen3-Reranker-GGUF) | Useful āœ… | From e174c1073c3c9408b6325ea1fac63688efacbb2e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 1 Jun 2026 09:07:29 +0800 Subject: [PATCH 153/304] feat(sampling): add reasoning budget configurations Introduce reasoning budget and block control parameters to `LlamaSamplingParams` to mirror llama.cpp CLI semantics. This includes: - `reasoning_budget` - `reasoning_start` / `reasoning_end` - `reasoning_budget_message` - `reasoning_start_in_prompt` - `reasoning_start_max_tokens` - Fix typo from typ_p to typical_p in logs Also updated `print_params()` to include these new metrics. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 64 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 5b5c533c52..9a22096a26 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -1363,6 +1363,59 @@ class LlamaSamplingParams: default_factory=lambda: ["\n", ":", "\"", "*"] # default sequence breakers for DRY ) + # Reasoning Budget Params + # + # Generic first-reasoning-block budget control. + # + # This is intentionally model-agnostic: + # - It does not infer model families. + # - It does not guess reasoning tags from chat templates. + # - Downstream code should pass reasoning_start / reasoning_end explicitly + # for models that do not use the default ... tags. + # + # The sampler only controls the first visible reasoning block. After that + # block naturally ends or is forcibly closed, later reasoning tags are ignored. + # Matches llama.cpp CLI semantics: + # --reasoning-budget N + reasoning_budget: int = -1 # -1 = unrestricted / disabled, 0 = immediate end, N > 0 = token budget + + # Token/text sequence that marks the beginning of the first reasoning block. + # This sequence is tokenized with add_bos=False, special=True before building + # the ReasoningBudgetSampler. + reasoning_start: str = "" + + # Token/text sequence that marks the natural end of the reasoning block. + # When the budget is exhausted, the sampler forces: + # reasoning_budget_message + reasoning_end + reasoning_end: str = "" + + # Optional message injected before reasoning_end when the budget is exhausted. + # Mirrors llama.cpp CLI semantics: + # --reasoning-budget-message MESSAGE + # + # Example forced text: + # "[reasoning budget exhausted]\n" + reasoning_budget_message: Optional[str] = None + + # True when the prompt/chat template has already inserted reasoning_start. + # + # In that case, the sampler will not see the start tag during generation, so + # it must start directly in COUNTING state from the first generated token. + reasoning_start_in_prompt: bool = False + + # Safety window for non-reasoning models. + # + # If reasoning_start is not generated within this many output tokens, the + # sampler permanently switches to DONE and becomes a no-op. This prevents + # later literal mentions of "" in normal answer text from accidentally + # activating the budget controller. + # + # Ignored when reasoning_start_in_prompt=True because counting starts from + # the first generated token. + # + # Set to None to keep waiting for reasoning_start indefinitely. + reasoning_start_max_tokens: Optional[int] = 32 + custom_samplers: List['CustomSampler'] = field(default_factory=list) samplers: List[CommonSamplerType] = field( @@ -1402,11 +1455,18 @@ def print_params(self) -> str: f"\ttop_k = {self.top_k}, top_p = {self.top_p:.3f}, min_p = {self.min_p:.3f}, " f"xtc_probability = {self.xtc_probability:.3f}, xtc_threshold = {self.xtc_threshold:.3f}, " - f"typical_p = {self.typ_p:.3f}, top_n_sigma = {self.top_n_sigma:.3f}, temp = {self.temp:.3f}\n" + f"typical_p = {self.typical_p:.3f}, top_n_sigma = {self.top_n_sigma:.3f}, temp = {self.temp:.3f}\n" f"\tmirostat = {self.mirostat}, mirostat_lr = {self.mirostat_eta:.3f}, " f"mirostat_ent = {self.mirostat_tau:.3f}, adaptive_target = {self.adaptive_target:.3f}, " - f"adaptive_decay = {self.adaptive_decay:.3f}" + f"adaptive_decay = {self.adaptive_decay:.3f}\n" + + f"\treasoning_budget = {self.reasoning_budget}, " + f"reasoning_start = {self.reasoning_start!r}, reasoning_end = {self.reasoning_end!r}\n" + + f"\treasoning_budget_message = {self.reasoning_budget_message!r}, " + f"reasoning_start_in_prompt = {self.reasoning_start_in_prompt}, " + f"reasoning_start_max_tokens = {self.reasoning_start_max_tokens}" ) return result From 9bb06dacc676cda4678e20ba3171f90e4e9e9362 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 01:01:31 +0800 Subject: [PATCH 154/304] Update Submodule vendor/llama.cpp d4c8e2c..27d9ed8 Signed-off-by: JamePeng --- llama_cpp/llama.py | 11 +++++++++-- llama_cpp/llama_cpp.py | 6 ++++++ vendor/llama.cpp | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index c2d2757e13..b9a1265b49 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -120,6 +120,7 @@ def __init__( n_ubatch: int = 512, n_seq_max: int = 1, n_rs_seq: int = 0, + n_outputs_max: int = 0, n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, ctx_type: Optional[ @@ -478,7 +479,8 @@ def __init__( self.n_batch = min(n_ctx, n_batch) # ??? self.n_keep = n_keep if n_keep > 0 else 256 self.n_seq_max = n_seq_max - self.n_rs_seq = n_rs_seq + self.n_rs_seq = n_rs_seq + self.n_outputs_max = n_outputs_max self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count() @@ -490,8 +492,13 @@ def __init__( self.context_params.n_ctx = n_ctx self.context_params.n_batch = self.n_batch self.context_params.n_ubatch = min(self.n_batch, n_ubatch) - self.context_params.n_seq_max = self.n_seq_max + + self.context_params.n_seq_max = max(1, self.n_seq_max) + if self.context_params.n_seq_max > llama_cpp_lib.LLAMA_MAX_SEQ: + raise RuntimeError(f"n_seq_max must be <= {llama_cpp_lib.LLAMA_MAX_SEQ}") + self.context_params.n_rs_seq = self.n_rs_seq + self.context_params.n_outputs_max = self.n_batch if self.n_outputs_max == 0 else self.n_outputs_max self.context_params.n_threads = self.n_threads self.context_params.n_threads_batch = self.n_threads_batch diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 62c4c81ef9..01aa8cce9b 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -55,6 +55,8 @@ LLAMA_MAX_DEVICES = _lib.llama_max_devices() +LLAMA_MAX_SEQ = 256 + # define LLAMA_DEFAULT_SEED 0xFFFFFFFF LLAMA_DEFAULT_SEED = 0xFFFFFFFF @@ -847,6 +849,7 @@ class llama_sampler_seq_config(ctypes.Structure): # uint32_t n_ubatch; // physical maximum batch size # uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) # uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] +# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) # int32_t n_threads; // number of threads to use for generation # int32_t n_threads_batch; // number of threads to use for batch processing @@ -905,6 +908,7 @@ class llama_context_params(ctypes.Structure): n_ubatch (int): physical maximum batch size n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_rs_seq (int): number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] + n_outputs_max (int): max outputs in a ubatch (0 = n_batch) n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing @@ -949,6 +953,7 @@ class llama_context_params(ctypes.Structure): n_ubatch: int n_seq_max: int n_rs_seq: int + n_outputs_max: int n_threads: int n_threads_batch: int ctx_type: int @@ -985,6 +990,7 @@ class llama_context_params(ctypes.Structure): ("n_ubatch", ctypes.c_uint32), ("n_seq_max", ctypes.c_uint32), ("n_rs_seq", ctypes.c_uint32), + ("n_outputs_max", ctypes.c_uint32), ("n_threads", ctypes.c_int32), ("n_threads_batch", ctypes.c_int32), ("ctx_type", ctypes.c_int), diff --git a/vendor/llama.cpp b/vendor/llama.cpp index d4c8e2c29c..27d9ed8397 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit d4c8e2c29ce2fb9a251a0a4a16d6c857b4f70f8c +Subproject commit 27d9ed839713e31c7a0ba45e342109a04549834f From a7db23afd86269bb9c08c00b00f2d23288880e50 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 03:09:17 +0800 Subject: [PATCH 155/304] feat(chat-format): improve Jinja2ChatFormatter HF compatibility Enhance Jinja2ChatFormatter to better support HuggingFace-style chat templates while keeping the formatter lightweight and aligned with llama-cpp-python's prompt-rendering needs. This change adds a custom Jinja extension for `{% generation %}` blocks. HuggingFace Transformers uses this tag to track assistant-token spans for assistant masks, but llama-cpp-python only needs the final rendered prompt. The new IgnoreGenerationTags extension therefore treats the tag as a transparent wrapper: it removes the generation/endgeneration tag pair while rendering the inner template body normally. This allows templates that contain `{% generation %}` blocks to render successfully without introducing span tracking overhead. The Jinja environment is also expanded to more closely match Transformers' chat-template runtime behavior. It now enables `jinja2.ext.loopcontrols` for templates that use `{% break %}` or `{% continue %}`, registers a plain JSON `tojson` filter that avoids Jinja's HTML escaping behavior, and exposes `raise_exception` and `strftime_now` as globals instead of passing them on every render call. The formatter now accepts an optional `special_tokens_map`, making additional tokenizer special tokens available to templates. This improves compatibility with templates that reference variables such as `pad_token`, `unk_token`, `sep_token`, or model-specific special tokens beyond `bos_token` and `eos_token`. This also adds optional `documents` support to `__call__`, allowing RAG-style or document-aware chat templates to receive a `documents` variable in the render context. Finally, static stop fields are precomputed during initialization. Text stop sequences and token-id stopping criteria are now built once instead of being recreated for every chat formatting call. The token-id stopping callback also guards against empty token arrays before reading the last token. Key changes: - Add IgnoreGenerationTags Jinja extension for HF `{% generation %}` blocks. - Enable Jinja loop controls for chat templates using break/continue. - Register Transformers-compatible `tojson` behavior. - Register `raise_exception` and `strftime_now` as Jinja globals. - Add `special_tokens_map` support for additional template variables. - Add optional `documents` argument for document-aware templates. - Precompute text stop sequences and token-id stopping criteria. - Improve type normalization for `stop_token_ids`. - Expand docstrings for formatter initialization and render-time variables. Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 264 +++++++++++++++++++++++++++++---- 1 file changed, 232 insertions(+), 32 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 71228d0627..f91844bbb7 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -26,6 +26,7 @@ ) import jinja2 +from jinja2.ext import Extension from jinja2.sandbox import ImmutableSandboxedEnvironment import numpy as np @@ -220,6 +221,46 @@ def __call__( class Jinja2ChatFormatter(ChatFormatter): + class IgnoreGenerationTags(Extension): + """Render HuggingFace `{% generation %}` blocks without tracking. + + HuggingFace chat templates may wrap assistant text with: + + {% generation %} + ... + {% endgeneration %} + + Transformers uses this tag to compute assistant-token masks. In + llama-cpp-python chat formatting we only need the final rendered prompt, + so this extension simply removes the tag pair and renders the inner + content as normal Jinja template content. + + This keeps compatibility with HF templates while avoiding the overhead + of span tracking. + + More information see: + https://github.com/huggingface/transformers/blob/39603d0e5cdb6f00e8d473d7fcbb01032d709181/src/transformers/utils/chat_template_utils.py#L425 + """ + + tags = {"generation"} + + def parse(self, parser: jinja2.parser.Parser): + # Consume the opening `{% generation %}` token. + lineno = next(parser.stream).lineno + + # Parse and return the block body until `{% endgeneration %}`. + # Returning the body directly makes the tag a transparent wrapper. + body = parser.parse_statements( + ("name:endgeneration",), + drop_needle=True, + ) + + # Preserve line numbers for better template error messages. + for node in body: + node.set_lineno(lineno) + + return body + def __init__( self, template: str, @@ -227,21 +268,118 @@ def __init__( bos_token: str, add_generation_prompt: bool = True, stop_token_ids: Optional[List[int]] = None, + special_tokens_map: Optional[Dict[str, str]] = None, ): - """A chat formatter that uses jinja2 templates to format the prompt.""" + """Format chat messages with a HuggingFace-style Jinja2 chat template. + + Args: + template: + Raw HuggingFace chat template string. + eos_token: + Text form of the model EOS token. + bos_token: + Text form of the model BOS token. + add_generation_prompt: + Whether to ask the template to append the assistant generation + prefix. This mirrors Transformers' `add_generation_prompt`. + stop_token_ids: + Optional token ids that should stop generation when they appear + as the last generated token. This is llama-cpp-python specific. + special_tokens_map: + Optional tokenizer special-token map. Some HF templates may + reference extra variables such as `pad_token`, `unk_token`, + `sep_token`, or model-specific special tokens. + """ self.template = template self.eos_token = eos_token self.bos_token = bos_token self.add_generation_prompt = add_generation_prompt + self.special_tokens_map = special_tokens_map or {} + self.stop_token_ids = ( - set(stop_token_ids) if stop_token_ids is not None else None + {int(token_id) for token_id in stop_token_ids} + if stop_token_ids is not None + else None ) - self._environment = ImmutableSandboxedEnvironment( + environment = ImmutableSandboxedEnvironment( loader=jinja2.BaseLoader(), trim_blocks=True, lstrip_blocks=True, - ).from_string(self.template) + # Keep this aligned with Transformers' chat-template Jinja setup: + # - IgnoreGenerationTags supports `{% generation %}` blocks. + # - loopcontrols supports `{% break %}` and `{% continue %}`. + extensions=[ + Jinja2ChatFormatter.IgnoreGenerationTags, + jinja2.ext.loopcontrols, + ], + ) + + # Match Transformers' chat-template JSON behavior. + # Jinja's default `tojson` escapes HTML characters, which is not what + # plain-text chat templates usually expect. + environment.filters["tojson"] = self.tojson + + # Register these as globals once instead of passing them on every render. + environment.globals["raise_exception"] = self.raise_exception + environment.globals["strftime_now"] = self.strftime_now + + self._environment = environment + self._template = environment.from_string(self.template) + + # Precompute static stop fields once. This avoids rebuilding closures and + # StoppingCriteriaList objects for every chat completion request. + self._stop = [self.eos_token] if self.eos_token else [] + self._stopping_criteria = self._build_stopping_criteria() + + @staticmethod + def raise_exception(message: str): + """Raise a Jinja template error from inside a chat template.""" + raise jinja2.exceptions.TemplateError(message) + + @staticmethod + def strftime_now(format_string: str = "%Y-%m-%d %H:%M:%S") -> str: + """Return the current local time formatted with `datetime.strftime`.""" + return datetime.datetime.now().strftime(format_string) + + @staticmethod + def tojson( + x: Any, + ensure_ascii: bool = False, + indent: Optional[int] = None, + separators: Optional[Tuple[str, str]] = None, + sort_keys: bool = False, + ) -> str: + """Serialize an object to JSON for chat-template rendering. + + This intentionally bypasses Jinja's built-in `tojson` filter because + the built-in filter escapes HTML-sensitive characters. HuggingFace chat + templates expect plain JSON text instead. + """ + return json.dumps( + x, + ensure_ascii=ensure_ascii, + indent=indent, + separators=separators, + sort_keys=sort_keys, + ) + + def _build_stopping_criteria(self): + """Create stopping criteria once during initialization.""" + if self.stop_token_ids is None: + return None + + stop_token_ids = self.stop_token_ids + + def stop_on_last_token( + tokens: npt.NDArray[np.intc], + logits: npt.NDArray[np.single], + ) -> bool: + # Defensive guard: generation normally calls this with at least one + # token, but the callback should never crash on empty input. + return len(tokens) > 0 and int(tokens[-1]) in stop_token_ids + + return llama_core.StoppingCriteriaList([stop_on_last_token]) def __call__( self, @@ -251,44 +389,106 @@ def __call__( function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, tools: Optional[List[llama_types.ChatCompletionTool]] = None, tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + documents: Optional[List[Dict[str, Any]]] = None, **kwargs: Any, ) -> ChatFormatterResponse: - def raise_exception(message: str): - raise ValueError(message) + """Render OpenAI-style chat messages into a model prompt. - def strftime_now(format_string="%Y-%m-%d %H:%M:%S") -> str: - """ - Returns the current time formatted as a string. - """ - return datetime.datetime.now().strftime(format_string) + The method builds the variable context expected by HuggingFace-style + Jinja chat templates and renders the final prompt string used by + llama-cpp-python. - prompt = self._environment.render( - messages=messages, - eos_token=self.eos_token, - bos_token=self.bos_token, - raise_exception=raise_exception, - strftime_now=strftime_now, - add_generation_prompt=self.add_generation_prompt, - functions=functions, - function_call=function_call, - tools=tools, - tool_choice=tool_choice, - ) + Template variables provided by default: + messages: + The chat history to render. Each item is expected to be an + OpenAI-style message dictionary, usually containing at least + `role` and `content`. - stopping_criteria = None - if self.stop_token_ids is not None: + eos_token: + The model's end-of-sequence token string. + + bos_token: + The model's beginning-of-sequence token string. + + add_generation_prompt: + Whether the template should append the assistant generation + prefix. This mirrors Transformers' `add_generation_prompt`. + + functions: + Legacy OpenAI-compatible function definitions, if provided. - def stop_on_last_token( - tokens: npt.NDArray[np.intc], logits: npt.NDArray[np.single] - ) -> bool: - return tokens[-1] in self.stop_token_ids + function_call: + Legacy OpenAI-compatible function-call selection, if provided. - stopping_criteria = llama_core.StoppingCriteriaList([stop_on_last_token]) + tools: + OpenAI/HuggingFace-compatible tool definitions, if provided. + This formatter expects tools to already be normalized into + JSON-schema-like dictionaries. It does not auto-convert Python + callables into JSON schemas like Transformers can. + + tool_choice: + Optional tool-choice instruction, such as `"auto"`, `"none"`, + or a specific tool/function selection object. + + documents: + Optional RAG/document context. Some HF chat templates reference + this variable when rendering retrieval-augmented prompts. + + **kwargs: + Extra model-specific or template-specific variables. These are + merged into the template context last, so they can intentionally + override the defaults above when needed. + + Additional variables: + Values from `special_tokens_map` are also exposed to the template, + such as `pad_token`, `unk_token`, `sep_token`, or custom + model-specific special tokens. Core variables like `messages`, + `eos_token`, and `bos_token` override `special_tokens_map` entries + by default. + + Returns: + ChatFormatterResponse: + Contains the rendered prompt, text stop sequences, optional + token-id stopping criteria, and `added_special=True` because the + chat template is responsible for adding model special tokens. + + Raises: + jinja2.exceptions.TemplateError: + If the template calls `raise_exception(...)` or Jinja rendering + fails. + """ + template_kwargs: Dict[str, Any] = {} + + # Make extra tokenizer special tokens available to templates, e.g. + # `pad_token`, `unk_token`, `sep_token`, or model-specific tokens. + template_kwargs.update(self.special_tokens_map) + + # Explicit core variables should override values from special_tokens_map. + template_kwargs.update( + { + "messages": messages, + "eos_token": self.eos_token, + "bos_token": self.bos_token, + "add_generation_prompt": self.add_generation_prompt, + "functions": functions, + "function_call": function_call, + "tools": tools, + "tool_choice": tool_choice, + "documents": documents, + } + ) + + # Let caller-provided kwargs extend the template context. + # If a caller intentionally passes a same-name key, it will override the + # defaults above. This is useful for model-specific template variables. + template_kwargs.update(kwargs) + + prompt = self._template.render(**template_kwargs) return ChatFormatterResponse( prompt=prompt, - stop=[self.eos_token], - stopping_criteria=stopping_criteria, + stop=self._stop, + stopping_criteria=self._stopping_criteria, added_special=True, ) From bbede198b8012b702bc1e6d241f0887b6e3336a2 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 03:40:31 +0800 Subject: [PATCH 156/304] feat(llama): enhance chat template initialization with full special tokens Update Llama.__init__ to register additional tokenizer special tokens and improve stop token handling for chat templates. - Expose extra special tokens (EOT, SEP, NL, PAD, MASK) via `special_tokens_map` to Jinja2ChatFormatter. - Keep BOS and EOS tokens as explicit parameters, no longer redundantly put them in `special_tokens_map`. - Build `stop_token_ids` once, including EOS and EOT tokens, skipping invalid (-1) ids. - Update try-block comment: now `{% generation %}` blocks are supported, guard only against malformed or model-specific templates. - This ensures better compatibility with HuggingFace-style chat templates while maintaining llama-cpp-python prompt-rendering behavior. Signed-off-by: JamePeng --- llama_cpp/llama.py | 48 +++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b9a1265b49..43e3d6f1fd 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -692,9 +692,6 @@ def __init__( self._n_vocab = self.n_vocab() self._n_ctx = self.n_ctx() - self._token_nl = self.token_nl() - self._token_eos = self.token_eos() - self._candidates = internals.LlamaTokenDataArray(n_vocab=self._n_vocab) self.n_tokens = 0 @@ -720,13 +717,38 @@ def __init__( eos_token_id = self.token_eos() bos_token_id = self.token_bos() + eot_token_id = self.token_eot() + sep_token_id = self.token_sep() + nl_token_id = self.token_nl() + pad_token_id = self.token_pad() + mask_token_id = self.token_mask() + + def _token_text(token_id: int) -> str: + return self._model.token_get_text(token_id) if token_id != -1 else "" + + bos_token = _token_text(bos_token_id) + eos_token = _token_text(eos_token_id) + + special_tokens_map = { + name: text + for name, token_id in { + "eot_token": eot_token_id, + "sep_token": sep_token_id, + "nl_token": nl_token_id, + "pad_token": pad_token_id, + "mask_token": mask_token_id, + }.items() + if token_id != -1 and (text := _token_text(token_id)) + } - eos_token = ( - self._model.token_get_text(eos_token_id) if eos_token_id != -1 else "" - ) - bos_token = ( - self._model.token_get_text(bos_token_id) if bos_token_id != -1 else "" - ) + stop_token_ids = [ + token_id + for token_id in (eos_token_id, eot_token_id) + if token_id != -1 + ] + + if not stop_token_ids: + stop_token_ids = None # Unfortunately the llama.cpp API does not return metadata arrays, so we can't get template names from tokenizer.chat_templates template_choices = dict( @@ -750,14 +772,14 @@ def __init__( for name, template in template_choices.items(): try: # Attempt to parse and register the template as a valid chat handler. - # We wrap this in a try-block because some models (like LLaVA) contain - # non-standard Jinja2 tags (e.g., {% generation %}) that cause the - # standard parser to crash. + # Keep this guarded because model metadata may contain malformed or + # model-specific Jinja templates that still cannot be rendered by this runtime. self._chat_handlers[name] = llama_chat_format.Jinja2ChatFormatter( template=template, eos_token=eos_token, bos_token=bos_token, - stop_token_ids=[eos_token_id], + stop_token_ids=stop_token_ids, + special_tokens_map=special_tokens_map, ).to_chat_handler() except Exception as e: # If parsing fails (e.g., TemplateSyntaxError), log a warning but do not crash. From e6b58356323d116df141b163f40be3ec988cf290 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 04:54:18 +0800 Subject: [PATCH 157/304] docs: update SCHEMA.md to v0.4 with full wiki path layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added comprehensive docs/wiki/ directory structure overview. - Reorganized modules description; removed hardcoded module page list. - Clarified top-level file purposes and update guidance. - Updated page type examples and templates (Class/Module, Feature, Example, Development). - Strengthened cross-linking rules and update/placeholder guidance. - Bumped schema version from 0.3 ā†’ 0.4 and last_modified date. Signed-off-by: JamePeng --- docs/wiki/SCHEMA.md | 141 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 120 insertions(+), 21 deletions(-) diff --git a/docs/wiki/SCHEMA.md b/docs/wiki/SCHEMA.md index 23954a156e..1ffcb1e227 100644 --- a/docs/wiki/SCHEMA.md +++ b/docs/wiki/SCHEMA.md @@ -4,14 +4,15 @@ - **Author**: JamePeng - **Maintainer**: LLM-assisted documentation workflow - **Project**: [llama-cpp-python](https://github.com/JamePeng/llama-cpp-python) wiki -- **Last Modified**: 2026-05-16 +- **Last Modified**: 2026-06-02 - **Version Target**: latest source code -- **Schema Version**: 0.3 +- **Schema Version**: 0.4 **Purpose**: - Maintain a living, always-up-to-date, structured documentation wiki for the `llama-cpp-python` library, with LLMs acting as the primary documentation maintainer. - The wiki must help users understand the latest public API, core classes, modules, configuration options, examples, and migration paths based on the current source code. - The wiki should explain not only *how to call an API*, but also *what role the class/module plays in the library*, *how its state is configured*, and *how users should choose between related APIs*. +- The schema also defines the expected wiki directory layout, page ownership, and update rules so new pages can be generated consistently. **Core Principles**: - The source of truth is the latest code in `llama_cpp/`, especially: @@ -29,7 +30,7 @@ - Prefer documenting public and user-facing APIs first. Internal implementation details may be documented only when they help users understand behavior, extension points, debugging, or advanced usage. - All examples must be complete, runnable with the latest API, and include necessary imports. - Clearly mark deprecated, legacy, or changed usage with a warning and show the modern replacement. -- Use internal wiki links (e.g. [[Llama]], [[Qwen35ChatHandler]]) for cross-referencing. +- Use internal wiki links, such as `[[Llama]]`, `[[LlamaCache]]`, `[[LlamaSpeculative]]`, or `[[Qwen35ChatHandler]]`, for cross-referencing. - Keep pages concise, professional, and user-friendly. **Documentation Language**: @@ -38,9 +39,53 @@ - Code comments inside examples should also be in English by default. - If the source code contains Chinese comments or non-English notes, translate them into clear English while preserving the original meaning. +**Wiki Directory Layout**: + +The wiki should be organized by documentation purpose rather than by source-file location alone. + +```text +docs/wiki/ +ā”œā”€ core/ # Core classes and modules (e.g., Llama, main API objects) +ā”œā”€ development/ # Developer-focused pages, tools, agents, CI/CD workflows +ā”œā”€ examples/ # Complete runnable examples for users +ā”œā”€ features/ # High-level features spanning multiple classes/modules +ā”œā”€ modules/ # Specialized modules (cache, embeddings, logging, speculative decoding, bindings) +ā”œā”€ types/ # Type definitions and data structures used across the library +ā”œā”€ .gitkeep # Placeholder for Git to track empty directories +ā”œā”€ contributing-to-wiki.md # Guidelines for contributing to the wiki +ā”œā”€ index.md # Entry point and table of contents +ā”œā”€ install.md # Installation instructions +ā”œā”€ SCHEMA.md # Documentation schema and style guide (this file) +ā”œā”€ troubleshooting.md # Known issues, debugging tips, FAQ +``` + +### Top-Level Files + +| Path | Purpose | Update Guidance | +|---|---|---| +| `docs/wiki/SCHEMA.md` | Defines the documentation contract, directory structure, page templates, and LLM update rules. | Update when adding a new page type, directory, documentation standard, or structural convention. | +| `docs/wiki/index.md` | Main wiki landing page and navigation entry. | Update when important pages are added, renamed, reorganized, or promoted. | +| `docs/wiki/contributing-to-wiki.md` | Human and LLM contribution guide for maintaining the wiki. | Keep aligned with this schema, especially source-reading and accuracy rules. | +| `docs/wiki/install.md` | Installation guide placeholder or final installation documentation. | Convert from placeholder to complete page when installation docs are ready. | +| `docs/wiki/troubleshooting.md` | Troubleshooting guide placeholder or final diagnostics documentation. | Expand with common runtime, build, backend, model loading, and environment issues. | +| `docs/wiki/.gitkeep` | Keeps the wiki directory tracked when needed. | No documentation content is required. | + +### Directory Ownership + +| Directory | Purpose | Typical Content | Primary Audience | +|---|---|---|---| +| `core/` | High-level public entry points and central user APIs. | `Llama`, model lifecycle, generation APIs, chat/completion interfaces. | General users and advanced users. | +| `modules/` | Focused subsystem pages, user-facing modules, low-level bindings, helpers, and advanced API areas. | Cache, embeddings, grammar, speculative decoding, logging, llama.cpp bindings, MTMD bindings. | Advanced users, extension authors, maintainers. | +| `features/` | Workflow-oriented guides that span multiple APIs or modules. | Chat formatting, structured output, multimodal usage, backend loading, caching workflows, speculative decoding workflows. | Users solving a specific task. | +| `examples/` | Complete runnable examples. | Minimal inference, chat completion, embeddings, grammar-constrained generation, speculative decoding, multimodal usage. | Users who want copy-paste starting points. | +| `types/` | Type and schema documentation. | Request/response structures, typed dictionaries, protocol-style types, OpenAI-compatible payloads. | Users integrating with typed code or API-compatible workflows. | +| `development/` | Maintainer-facing documentation and contribution workflows. | Build notes, CI notes, release notes, commit generation workflow, documentation maintenance rules. | Maintainers and contributors. | + **Page Types and Templates**: -1. **Class / Module Page** (e.g. core/Llama.md, modules/LlamaEmbedding.md) +1. **Class / Module Page** + Examples: `core/Llama.md`, `modules/LlamaEmbedding.md`, `modules/LlamaCache.md` + - Frontmatter (YAML): ```yaml --- @@ -51,14 +96,15 @@ version_target: "latest" --- ``` - - Sections (in order): + + - Sections, in order: - Overview - Role in the Library - Constructor (`__init__`) ā€“ full parameter table with types, defaults, and explanations - Important Attributes / State - - Core Methods (with signatures and usage examples) + - Core Methods, with signatures and usage examples - Best Practices & Common Patterns - - Deprecated / Changed APIs (with migration notes) + - Deprecated / Changed APIs, with migration notes - Related Links - The **Overview** should briefly explain: @@ -81,24 +127,77 @@ - Only document attributes that affect user understanding, configuration, lifecycle, inference behavior, caching, chat formatting, embeddings, or debugging. Do not document every trivial private variable. -2. **Feature Page** (features/xxx.md) - - Overview, When to use, Related APIs, Code examples, Configuration Notes, Limitations, Related features - - Feature pages should explain workflows across multiple classes or modules. - -3. **Example Page** (examples/xxx.md) - - Goal, Prerequisites, Complete runnable code block, Expected output, Tips - - Rules: - * Use the latest API. - * Include all imports as need. - * Avoid pseudo-code. - * Keep examples focused. - * Mention required model assumptions when needed, such as GGUF file path or chat format. +2. **Feature Page** + Example: `features/speculative-decoding.md`, `features/embeddings-rerank.md` + + Feature pages should explain workflows across multiple classes or modules. + + Required sections: + - Overview + - When to Use + - Related APIs + - Code Examples + - Configuration Notes + - Limitations + - Related Features + +3. **Example Page** + Example: `examples/chat-completion.md` + + Required sections: + - Goal + - Prerequisites + - Complete Runnable Code + - Expected Output + - Tips + + Rules: + - Use the latest API. + - Include all required imports. + - Avoid pseudo-code. + - Keep examples focused. + - Mention required model assumptions when needed, such as GGUF file path, embedding mode, grammar file, chat format, or multimodal assets. + +4. **Development Page** + Example: `development/GitCommitGenerationAgent.md` + + Development pages are maintainer-facing and may document repository workflows, CI, release notes, build matrix decisions, or documentation maintenance conventions. + + Required sections: + - Overview + - Scope + - Workflow + - Inputs / Outputs + - Rules and Constraints + - Examples + - Related Links + +**Cross-Linking Rules**: + +- Use wiki-style internal links for pages that exist or should exist, such as `[[Llama]]`, `[[LlamaCache]]`, `[[LlamaSpeculative]]`, and `[[Logger]]`. +- Link from high-level pages to lower-level module pages when the module explains advanced details. +- Link from feature pages back to the relevant class/module pages. +- Avoid circular explanations. A page may link to another page for details instead of repeating the same explanation. **Update Rules**: + - Before updating any page, the LLM must read the relevant source files. - Update the `last_updated` date. -- If a new feature appears, such as a new chat handler, sampler, cache type, embedding API, multimodal API, or backend option, create or expand the corresponding page. +- If a new feature appears, such as a new chat handler, sampler, cache type, embedding API, multimodal API, backend option, or binding wrapper, create or expand the corresponding page. - If behavior is inferred from implementation rather than explicitly documented in code, mark the explanation as implementation-based. +- Empty files should be converted into explicit placeholder pages instead of being left blank. - Maintain a high standard of readability and accuracy. -This schema is the contract. All generated content must follow it. \ No newline at end of file +**Quality Checklist**: + +Before finalizing a wiki page, verify: + +- The page reflects the latest source code. +- All parameters, defaults, and return values are accurate. +- Examples are runnable and include necessary imports. +- Internal links point to the correct wiki page names. +- Advanced or low-level APIs are clearly labeled. +- Deprecated behavior is clearly separated from current usage. +- The page avoids undocumented claims, speculative behavior, or outdated assumptions. + +This schema is the contract. All generated content must follow it. From 2fbe63ddf829ab596ce359339f49dd7f110bbe89 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 05:33:37 +0800 Subject: [PATCH 158/304] build(deps): align Jinja2 minimum with Transformers Require Jinja2 >= 3.1.0 for HuggingFace-style chat template support. The updated Jinja2ChatFormatter relies on behavior aligned with Transformers' chat-template runtime, which also requires Jinja2 3.1 or newer. Updating the minimum dependency avoids parser/runtime differences with older Jinja versions. Signed-off-by: JamePeng --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eb4b879dd6..dea9b48ff3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "typing-extensions>=4.8.0", "numpy>=1.21.6,<=2.3.2", "diskcache>=5.6.2", - "jinja2>=2.11.3", + "jinja2>=3.1.0", "Pillow>=9.5.0", ] requires-python = ">=3.9" From acf896381f7b18a92bc0477a0c3939e3a79d910b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 20:21:08 +0800 Subject: [PATCH 159/304] Update Submodule vendor/llama.cpp 27d9ed8..60130d1 Signed-off-by: JamePeng --- llama_cpp/_internals.py | 7 ------- llama_cpp/llama_cpp.py | 10 +++++++--- vendor/llama.cpp | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 9a22096a26..92ff51447f 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -774,13 +774,6 @@ def set_causal_attn(self, causal_attn: bool): """ llama_cpp.llama_set_causal_attn(self.ctx, causal_attn) - def set_warmup(self, warmup: bool): - """ - Set whether the model is in warmup mode or not - If true, all model tensors are activated during llama_decode() to load and cache their weights. - """ - llama_cpp.llama_set_warmup(self.ctx, warmup) - def synchronize(self): """ Wait until all computations are finished diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 01aa8cce9b..9c911bcb14 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -3085,11 +3085,15 @@ def llama_set_causal_attn(ctx: llama_context_p, causal_attn: bool, /): # // Set whether the model is in warmup mode or not # // If true, all model tensors are activated during llama_decode() to load and cache their weights. -# LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup); +# // +# // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models, +# // so it is generally not recommended to use in practice. will be removed in the future +# DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup), +# "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]"); @ctypes_function("llama_set_warmup", [llama_context_p_ctypes, ctypes.c_bool], None) def llama_set_warmup(ctx: llama_context_p, warmup: bool, /): - """ Set whether the model is in warmup mode or not - If true, all model tensors are activated during llama_decode() to load and cache their weights""" + """DEPRECATED: using this can cause extra graph reallocations because it changes the graph topology with MoE models, + so it is generally not recommended to use in practice. will be removed in the future""" ... # // Set abort callback diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 27d9ed8397..60130d18f9 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 27d9ed839713e31c7a0ba45e342109a04549834f +Subproject commit 60130d18f9ac7f42cb4d7f6060b088a45d8f242e From a29c75495d69dd0bcd9596fecd99789d07a09ffa Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 23:22:47 +0800 Subject: [PATCH 160/304] docs(install): add source-aligned build and backend guide Document installation workflows for llama-cpp-python with a focus on the underlying llama.cpp CMake build configuration. - Add virtual environment, source install, editable install, rebuild, and verification guidance. - Document common CMake options such as GGML_NATIVE, GGML_BACKEND_DL, GGML_CPU_ALL_VARIANTS, and compiler selection. - Summarize backend-specific build flags for CUDA, BLAS, Metal, Vulkan, OpenVINO, HIP, SYCL, OpenCL, CANN, ZenDNN, and zDNN. - Include backend runtime notes and common installation pitfalls while keeping server-related installation content out of the page. Signed-off-by: JamePeng --- docs/wiki/install.md | 775 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 775 insertions(+) diff --git a/docs/wiki/install.md b/docs/wiki/install.md index e69de29bb2..576ca14c6f 100644 --- a/docs/wiki/install.md +++ b/docs/wiki/install.md @@ -0,0 +1,775 @@ +--- +title: Installation +page_type: guide +source_files: + - README.md + - vendor/llama.cpp/docs/build.md + - vendor/llama.cpp/docs/backend/ +last_updated: 2026-06-02 +author: JamePeng +version_target: "latest" +--- + +# Installation + +## Overview + +This page explains how to install `llama-cpp-python` from source, with or +without hardware acceleration. + +`llama-cpp-python` builds the native `llama.cpp` libraries during installation +and installs them inside the Python package. The exact build depends on your +Python version, compiler, CMake version, operating system, and selected +`llama.cpp` backend. + +For most users, the safest installation path is: + +1. Create a clean Python virtual environment. +2. Upgrade `pip`. +3. Install from the GitHub repository. +4. Pass `CMAKE_ARGS` only when you need a specific backend. + +--- + +## Requirements + +| Requirement | Notes | +|---|---| +| Python | Python 3.9 or newer. The package metadata currently lists Python 3.9 through 3.14. | +| CMake | CMake 3.21 or newer. | +| C/C++ compiler | Required because the package builds `llama.cpp` native libraries. | +| Git | Required when installing from the GitHub repository or cloning recursively. | +| Backend SDKs | Required only for GPU or accelerator builds, such as CUDA, Vulkan, OpenVINO, ROCm/HIP, or SYCL. | + +Platform compiler guidance: + +| Platform | Typical compiler setup | +|---|---| +| Linux | `gcc` or `clang` plus Python development headers if required by your distribution. | +| Windows | Visual Studio 2022 Build Tools or MinGW. For most native builds, Visual Studio Build Tools is recommended. | +| macOS | Xcode Command Line Tools. Metal is enabled by default on supported macOS builds. | + +--- + +## Use a Virtual Environment + +Using a virtual environment avoids mixing build artifacts and dependencies from +different Python installations. + +### Linux and macOS + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip setuptools wheel +``` + +### Windows PowerShell + +```powershell +py -3 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip setuptools wheel +``` + +If PowerShell blocks activation scripts, run: + +```powershell +Set-ExecutionPolicy -Scope CurrentUser RemoteSigned +``` + +Then activate the environment again. + +--- + +## Basic Installation + +Install directly from the project repository: + +```bash +python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +On Windows PowerShell: + +```powershell +python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +This builds `llama.cpp` from source and installs the generated native runtime +libraries alongside the Python package. + +Use verbose output when diagnosing build failures: + +```bash +python -m pip install --verbose "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +--- + +## Install From a Local Clone + +Clone recursively so the `vendor/llama.cpp` submodule is available: + +```bash +git clone https://github.com/JamePeng/llama-cpp-python --recursive +cd llama-cpp-python +python -m pip install --upgrade pip +python -m pip install . +``` + +If you already cloned without `--recursive`, initialize the submodule manually: + +```bash +git submodule update --init --recursive +``` + +For editable development installs: + +```bash +python -m pip install -e . +``` + +--- + +## Passing CMake Options + +`llama.cpp` backend options are passed through CMake. There are two common +ways to pass those options during `pip install`. + +### Environment Variable + +Linux and macOS: + +```bash +CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Windows PowerShell: + +```powershell +$env:CMAKE_ARGS = "-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" +python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Clear the variable after the build if you do not want it reused: + +```powershell +Remove-Item Env:CMAKE_ARGS +``` + +### `pip --config-settings` + +You can also pass CMake arguments through `pip`: + +```bash +python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" \ + -C cmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS" +``` + +Use semicolons inside `cmake.args` when passing multiple CMake definitions. + +--- + +## Common CMake Options + +The Python package forwards CMake options to the bundled `vendor/llama.cpp` +build. These options are useful across many backends. + +| Option | Typical values | Use | +|---|---|---| +| `CMAKE_BUILD_TYPE` | `Release`, `Debug` | Selects build type for single-config generators such as Ninja or Unix Makefiles. Release is the normal install choice. | +| `GGML_NATIVE` | `ON`, `OFF` | Controls whether ggml builds for the current host CPU/GPU. Use `OFF` for more portable wheels; use `ON` for local machine-specific optimization. | +| `BUILD_SHARED_LIBS` | `ON`, `OFF` | Controls shared versus static native libraries. The Python package normally installs shared runtime libraries. | +| `GGML_BACKEND_DL` | `ON`, `OFF` | Builds backend libraries so they can be loaded dynamically at runtime when supported by the build. | +| `GGML_CPU_ALL_VARIANTS` | `ON`, `OFF` | Builds multiple CPU backend variants for x86 feature sets when supported. Useful for portable x64 wheels. | +| `GGML_OPENMP` | `ON`, `OFF` | Enables OpenMP CPU parallelism. On Windows, OpenMP runtime DLLs may need to be packaged beside backend DLLs. | +| `CMAKE_PREFIX_PATH` | path list | Helps CMake find SDKs or libraries installed outside default locations. | +| `CMAKE_C_COMPILER` / `CMAKE_CXX_COMPILER` | compiler paths or names | Selects compilers, often needed for SYCL, HIP, or custom toolchains. | + +Example portable CUDA build: + +```bash +CMAKE_ARGS="-DGGML_CUDA=ON -DGGML_NATIVE=OFF" \ + python -m pip install --force-reinstall --no-cache-dir \ + "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Example dynamic CPU backend build: + +```bash +CMAKE_ARGS="-DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_NATIVE=OFF" \ + python -m pip install --force-reinstall --no-cache-dir \ + "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +--- + +## Backend Quick Reference + +Choose one backend path that matches your hardware and installed SDKs. + +| Backend | Typical CMake option | Notes | +|---|---|---| +| CPU only | none | Default portable path. Performance depends on CPU features and build options. | +| OpenBLAS | `-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS` | CPU BLAS acceleration for prompt processing and larger batches. | +| BLIS | `-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=FLAME` | CPU BLAS route using BLIS. | +| Intel oneMKL | `-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Intel10_64lp` | Intel CPU BLAS route. This is not the Intel GPU path. | +| CUDA | `-DGGML_CUDA=on` | Requires NVIDIA CUDA Toolkit matching your driver and GPU. | +| Metal | `-DGGML_METAL=on` | Enabled by default on supported macOS builds. Use `-DGGML_METAL=OFF` to disable. | +| Vulkan | `-DGGML_VULKAN=on` | Requires Vulkan SDK and platform-specific setup. | +| OpenVINO | `-DGGML_OPENVINO=ON` | Useful for Intel CPU, GPU, and NPU workflows after OpenVINO environment setup. | +| HIP / ROCm | `-DGGML_HIP=ON` | For supported AMD GPUs. May require `GPU_TARGETS`. | +| SYCL | `-DGGML_SYCL=on` | Usually used with Intel oneAPI compilers. | +| OpenCL | `-DGGML_OPENCL=ON` | Primarily documented for Qualcomm Adreno and Snapdragon workflows; can also apply to some other OpenCL devices. | +| CANN | `-DGGML_CANN=ON` | Ascend NPU backend. Requires Ascend drivers and CANN toolkit. | +| ZenDNN | `-DGGML_ZENDNN=ON` | AMD Zen CPU acceleration, mainly matrix multiplication paths. | +| zDNN | `-DGGML_ZDNN=ON -DZDNN_ROOT=/path/to/zdnn` | IBM Z / LinuxONE acceleration path. | + +For the full list of backend options, check the upstream llama.cpp build +documentation and the current `vendor/llama.cpp` source. + +--- + +## CUDA + +CUDA builds require the NVIDIA CUDA Toolkit. Choose a toolkit version that is +compatible with your driver and GPU. + +Linux: + +```bash +CMAKE_ARGS="-DGGML_CUDA=on" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Windows PowerShell: + +```powershell +$env:CMAKE_ARGS = "-DGGML_CUDA=on" +python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +For newer NVIDIA GPUs with compute capability 90 or higher, the README notes +that Programmatic Dependent Launch can be enabled with: + +```bash +-DGGML_CUDA_PDL=ON +``` + +Example: + +```bash +CMAKE_ARGS="-DGGML_CUDA=on -DGGML_CUDA_PDL=ON" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +If `nvcc` produces large volumes of non-blocking template warnings, the README +documents optional CUDA warning suppression: + +```bash +-DCMAKE_CUDA_FLAGS="--diag-suppress=177 --diag-suppress=221 --diag-suppress=550" +``` + +### CUDA Portability and Architecture Selection + +By default, llama.cpp may build for the GPU detected on the build machine. For +a wheel intended to run across multiple CUDA GPUs, disable native detection: + +```bash +CMAKE_ARGS="-DGGML_CUDA=ON -DGGML_NATIVE=OFF" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +If `nvcc` cannot detect your GPU, or if you want to control the generated +binary size, specify CUDA architectures explicitly: + +```bash +CMAKE_ARGS="-DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86;89" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Use NVIDIA's compute capability table to choose architecture numbers. For +example, RTX 30-series GPUs commonly use `86`, and RTX 4090 uses `89`. + +If multiple CUDA toolkits are installed, point CMake at the intended compiler: + +```bash +CMAKE_ARGS="-DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/opt/cuda-12.8/bin/nvcc" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Runtime variables that may matter after installation: + +| Variable | Use | +|---|---| +| `CUDA_VISIBLE_DEVICES` | Selects or hides CUDA devices for the current process. | +| `GGML_CUDA_ENABLE_UNIFIED_MEMORY` | Enables unified-memory fallback on Linux when VRAM is exhausted. On Windows, similar behavior may be controlled by NVIDIA driver settings. | +| `GGML_CUDA_P2P` | Enables peer-to-peer access between GPUs when driver and hardware support it. | +| `GGML_CUDA_FORCE_CUBLAS_COMPUTE_32F` | Forces FP32 compute in selected cuBLAS paths, trading speed for numerical headroom. | +| `GGML_CUDA_FORCE_CUBLAS_COMPUTE_16F` | Forces FP16 compute in selected cuBLAS paths when supported. | + +--- + +## BLAS and CPU Acceleration + +BLAS acceleration mainly improves prompt processing and larger batch prefill. +It generally does not improve single-token generation speed as much as GPU +offload. + +### OpenBLAS + +Use OpenBLAS when the OpenBLAS development package is available on your system. + +```bash +CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +On Linux, install the OpenBLAS development package with your system package +manager before building. Package names vary by distribution. + +### BLIS + +BLIS is selected through the `FLAME` BLAS vendor after BLIS is installed: + +```bash +CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=FLAME" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +The upstream BLIS guide also notes that runtime variables such as +`BLIS_NUM_THREADS` and OpenMP affinity settings can affect CPU performance. + +### Intel oneMKL for CPU + +Intel oneMKL is a CPU BLAS path. It is different from Intel GPU acceleration, +which is usually handled through SYCL or OpenVINO. + +```bash +source /opt/intel/oneapi/setvars.sh +CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Intel10_64lp -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_NATIVE=ON" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +--- + +## Metal on macOS + +On macOS, Metal is enabled by default by this project when building on Apple +platforms. A normal install is usually enough: + +```bash +python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +To disable Metal at build time: + +```bash +CMAKE_ARGS="-DGGML_METAL=OFF" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +At runtime, use `n_gpu_layers=0` when you want CPU inference even though the +package was built with Metal support. + +--- + +## Vulkan + +Vulkan builds require the Vulkan SDK and any platform-specific environment +setup required by the SDK. + +```bash +CMAKE_ARGS="-DGGML_VULKAN=on" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +On Linux and macOS, make sure the Vulkan SDK setup script has been sourced in +the same shell session before running `pip install`. + +On Windows, install the Vulkan SDK and make sure its environment variables are +available in the shell that runs the build. + +On Linux, system packages can also provide the Vulkan loader and shader tools. +The upstream guide notes that SPIR-V headers may be required separately from +the Vulkan loader development package on some distributions. + +For macOS Vulkan builds, Vulkan usually runs through a Metal translation layer. +The upstream guide builds Vulkan with Metal disabled: + +```bash +CMAKE_ARGS="-DGGML_VULKAN=ON -DGGML_METAL=OFF" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +--- + +## OpenVINO + +OpenVINO builds require the OpenVINO runtime and environment setup first. + +Linux: + +```bash +source /opt/intel/openvino/setupvars.sh +CMAKE_ARGS="-DGGML_OPENVINO=ON" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Windows: + +```powershell +# Run this from a shell where OpenVINO setupvars.bat has been initialized, +# such as an OpenVINO command prompt, or initialize it through cmd first. +$env:CMAKE_ARGS = "-DGGML_OPENVINO=ON" +python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +The OpenVINO backend is intended for Intel CPU, GPU, and NPU workflows when the +OpenVINO runtime supports the target device. + +Runtime variables: + +| Variable | Use | +|---|---| +| `GGML_OPENVINO_DEVICE` | Selects `CPU`, `GPU`, `NPU`, or a specific GPU such as `GPU.0`. Defaults to CPU if unset or unavailable. | +| `GGML_OPENVINO_CACHE_DIR` | Enables OpenVINO model caching when set. Not supported on NPU devices according to upstream docs. | +| `GGML_OPENVINO_STATEFUL_EXECUTION` | Enables stateful KV-cache execution. Upstream docs recommend it for CPU/GPU performance and note it is not effective on NPU. | +| `GGML_OPENVINO_PREFILL_CHUNK_SIZE` | Controls NPU prefill chunk size. | +| `GGML_OPENVINO_PROFILING` | Enables OpenVINO profiling. | + +Important limitations from the upstream OpenVINO backend docs: + +- GPU stateless execution has known issues; use `GGML_OPENVINO_STATEFUL_EXECUTION=1` for GPU workflows. +- NPU runs may fail when context size is too large. Keep context size small for NPU workflows. +- Encoder models such as embedding and reranking models are not supported by the current OpenVINO backend implementation. +- Some benchmark workflows require Flash Attention enabled in the llama.cpp tool layer; in Python, verify behavior against your target model and backend. + +--- + +## HIP / ROCm + +HIP builds are for supported AMD GPUs. + +Linux example: + +```bash +CMAKE_ARGS="-DGGML_HIP=ON -DGPU_TARGETS=gfx1030" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +`GPU_TARGETS` is optional in some setups, but specifying your GPU architecture +can reduce build time and avoid unsupported target issues. + +Windows ROCm builds are more environment-sensitive. The README currently +documents a TheRock ROCm workflow that sets `HIP_PATH`, `ROCM_PATH`, +`HIP_DEVICE_LIB_PATH`, compiler paths, `CMAKE_GENERATOR`, and `CMAKE_ARGS` +before running `pip install`. + +For RDNA3 or CDNA hardware, upstream docs mention optional Flash Attention +acceleration through rocWMMA: + +```bash +CMAKE_ARGS="-DGGML_HIP=ON -DGPU_TARGETS=gfx1100 -DGGML_HIP_ROCWMMA_FATTN=ON" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Runtime variables that may matter: + +| Variable | Use | +|---|---| +| `HIP_VISIBLE_DEVICES` | Selects visible HIP devices. | +| `HSA_OVERRIDE_GFX_VERSION` | Can help unsupported Linux GPUs use a nearby architecture value. Upstream docs note this is not supported on Windows. | +| `HIP_DEVICE_LIB_PATH` | Points to ROCm device bitcode libraries when clang cannot find them. | + +--- + +## SYCL + +SYCL builds are usually used with Intel oneAPI compilers. + +```bash +source /opt/intel/oneapi/setvars.sh +CMAKE_ARGS="-DGGML_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +To request FP16 support: + +```bash +CMAKE_ARGS="-DGGML_SYCL=on -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Useful SYCL build options from the upstream backend docs: + +| Option | Use | +|---|---| +| `GGML_SYCL_F16` | Enables FP16 build path. Test both FP32 and FP16 for your model and device. | +| `GGML_SYCL_TARGET` | Selects SYCL target type. Intel is the default target in upstream docs. | +| `GGML_SYCL_DEVICE_ARCH` | Selects device architecture when known. | +| `GGML_SYCL_GRAPH` | Enables the experimental SYCL graph extension. | +| `GGML_SYCL_DNN` | Enables oneDNN integration. | +| `GGML_SYCL_HOST_MEM_FALLBACK` | Allows host-memory fallback when device memory is full, at reduced speed. | +| `GGML_SYCL_SUPPORT_LEVEL_ZERO` | Enables Level Zero support for Intel GPU memory allocation. | + +Useful SYCL runtime variables: + +| Variable | Use | +|---|---| +| `ONEAPI_DEVICE_SELECTOR` | Selects a SYCL device, such as a specific Level Zero GPU. | +| `GGML_SYCL_ENABLE_FLASH_ATTN` | Enables or disables Flash Attention in the SYCL backend. | +| `GGML_SYCL_ENABLE_LEVEL_ZERO` | Uses Level Zero allocation when support was built in. | +| `GGML_SYCL_DISABLE_DNN` | Disables oneDNN path and uses oneMKL path. | +| `ZES_ENABLE_SYSMAN` | Helps query free GPU memory in some Intel GPU setups. | + +--- + +## OpenCL + +OpenCL support is documented upstream mainly for Qualcomm Adreno GPUs and +Snapdragon devices. It may also work on certain other OpenCL-capable GPUs, but +SYCL is usually preferred for modern Intel GPU workflows. + +```bash +CMAKE_ARGS="-DGGML_OPENCL=ON" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +Useful OpenCL CMake options: + +| Option | Default | Use | +|---|---|---| +| `GGML_OPENCL_EMBED_KERNELS` | `ON` | Embeds OpenCL kernels into the built binary or library. | +| `GGML_OPENCL_USE_ADRENO_KERNELS` | `ON` | Enables kernels optimized for Adreno. | + +For Linux builds where OpenCL headers and ICD loader are installed in a custom +prefix, pass that location through `CMAKE_PREFIX_PATH`. + +--- + +## CANN + +CANN is the Ascend NPU backend. It requires Ascend drivers and the CANN toolkit +before building. + +```bash +CMAKE_ARGS="-DGGML_CANN=ON -DCMAKE_BUILD_TYPE=Release" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +The upstream CANN documentation focuses on Linux and Ascend devices such as +Atlas 300I A2 and Atlas 300I Duo. Supported model families and data types vary +by device generation. + +--- + +## ZenDNN and zDNN + +ZenDNN and zDNN are different backends. + +| Backend | Hardware | CMake option | +|---|---|---| +| ZenDNN | AMD Zen CPUs, especially AMD EPYC | `-DGGML_ZENDNN=ON` | +| zDNN | IBM Z / LinuxONE with NNPA acceleration | `-DGGML_ZDNN=ON -DZDNN_ROOT=/path/to/zdnn` | + +ZenDNN can be downloaded and built automatically by CMake: + +```bash +CMAKE_ARGS="-DGGML_ZENDNN=ON -DCMAKE_BUILD_TYPE=Release" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +If you already have a ZenDNN installation: + +```bash +CMAKE_ARGS="-DGGML_ZENDNN=ON -DZENDNN_ROOT=/path/to/ZenDNN/build/install" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +zDNN requires a zDNN library installation first: + +```bash +CMAKE_ARGS="-DGGML_ZDNN=ON -DZDNN_ROOT=/opt/zdnn-libs" \ + python -m pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +ZenDNN currently accelerates matrix multiplication paths and may fall back to +the standard CPU backend for other operations. + +--- + +## Dynamic Backend Wheels + +The README notes that newer preview wheels may be built with: + +```text +GGML_BACKEND_DL=ON +GGML_CPU_ALL_VARIANTS=ON +``` + +In that build mode, CPU backend variants are installed as separate runtime +libraries under: + +```text +site-packages/llama_cpp/lib +``` + +Examples include: + +```text +ggml-cpu-x64 +ggml-cpu-sse42 +ggml-cpu-haswell +ggml-cpu-skylakex +ggml-cpu-alderlake +ggml-cpu-zen4 +``` + +On Windows, dynamic CPU backend DLLs may also need the LLVM OpenMP runtime +next to them: + +```text +libomp140.x86_64.dll +``` + +Based on the current top-level `CMakeLists.txt`, this project installs many +`llama`, `ggml`, CPU-variant, accelerator backend, and `mtmd` targets into the +Python package runtime directory when those targets are available. + +--- + +## Upgrading and Rebuilding + +Use `--upgrade`, `--force-reinstall`, and `--no-cache-dir` when you need to +force a rebuild with new CMake options: + +```bash +CMAKE_ARGS="-DGGML_CUDA=on" \ + python -m pip install --upgrade --force-reinstall --no-cache-dir \ + "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python.git" +``` + +This is important because `pip` may otherwise reuse cached wheels or build +artifacts from a previous backend configuration. + +For local editable builds, clean old native artifacts before rebuilding when +switching backends: + +```bash +make clean +python -m pip install --verbose -e . +``` + +On Windows, if `make` is not available, remove `_skbuild` and old native +libraries under `llama_cpp/lib` manually before reinstalling. + +--- + +## Verify Installation + +Check that the package imports: + +```bash +python -c "import llama_cpp; print(llama_cpp.__version__)" +``` + +Check where the package was installed: + +```bash +python -c "import llama_cpp, pathlib; print(pathlib.Path(llama_cpp.__file__).parent)" +``` + +Check the bundled native runtime libraries: + +```bash +python -c "import llama_cpp, pathlib; print(list((pathlib.Path(llama_cpp.__file__).parent / 'lib').glob('*')))" +``` + +Run a minimal model load after downloading a GGUF model: + +```python +from llama_cpp import Llama + +llm = Llama( + model_path="./model.gguf", + n_gpu_layers=0, + verbose=False, +) + +output = llm("Hello,", max_tokens=8) +print(output["choices"][0]["text"]) +``` + +For GPU builds, set `n_gpu_layers=-1` or another positive value to offload +layers: + +```python +from llama_cpp import Llama + +llm = Llama( + model_path="./model.gguf", + n_gpu_layers=-1, +) +``` + +--- + +## Development Workflow + +Common local development commands: + +```bash +git clone https://github.com/JamePeng/llama-cpp-python --recursive +cd llama-cpp-python +python -m pip install --upgrade pip +python -m pip install -e . +python -m pytest +``` + +The repository also includes a `Makefile` with useful targets: + +| Target | Purpose | +|---|---| +| `make build` | Editable build with verbose output. | +| `make build.cuda` | Editable build with `GGML_CUDA=on`. | +| `make build.openblas` | Editable build with OpenBLAS. | +| `make build.openvino` | Editable build with OpenVINO. | +| `make build.vulkan` | Editable build with Vulkan. | +| `make build.sycl` | Editable build with SYCL. | +| `make test` | Run pytest with verbose tracing. | +| `make clean` | Remove local native build artifacts. | + +When testing a different `llama.cpp` commit, update the `vendor/llama.cpp` +submodule, clean the local build, and reinstall. If the upstream C API changes, +the ctypes declarations in `llama_cpp/llama_cpp.py` may also need to be updated. + +--- + +## Common Installation Pitfalls + +| Symptom | Likely cause | What to try | +|---|---|---| +| CMake cannot find a compiler | Build tools are missing or not available in the current shell. | Install platform build tools and reopen the terminal. On Windows, use a Developer PowerShell or initialize Visual Studio build variables. | +| Build ignores new backend flags | `pip` reused a cached wheel or previous build. | Reinstall with `--force-reinstall --no-cache-dir`, and clean `_skbuild` for local builds. | +| CUDA backend does not build | CUDA Toolkit is missing, incompatible, or not on `PATH`. | Verify `nvcc --version`, CUDA driver compatibility, and `CUDA_PATH` on Windows. | +| CUDA build targets the wrong GPU generation | Native architecture detection picked the build machine GPU, or `nvcc` could not detect it. | Use `-DGGML_NATIVE=OFF` for portability or set `-DCMAKE_CUDA_ARCHITECTURES=...` explicitly. | +| Native library fails to load on Windows | Required DLLs are missing from `PATH` or `llama_cpp/lib`. | Check `llama_cpp/lib` for `llama.dll`, `ggml*.dll`, backend DLLs, and runtime DLLs such as OpenMP or CUDA dependencies. | +| GPU is not used at runtime | The package was built without that backend or `n_gpu_layers` is `0`. | Rebuild with the correct CMake backend flag and set `n_gpu_layers` to a positive value or `-1`. | +| OpenVINO GPU or NPU behaves unexpectedly | Runtime device selection or context size is unsuitable. | Set `GGML_OPENVINO_DEVICE`, enable `GGML_OPENVINO_STATEFUL_EXECUTION=1` for GPU, and keep context size smaller for NPU workflows. | +| SYCL device is not selected | oneAPI environment or device selector is missing. | Source oneAPI setup and set `ONEAPI_DEVICE_SELECTOR` for the intended device. | +| Submodule files are missing | Repository was cloned without `--recursive`. | Run `git submodule update --init --recursive`. | + +For detailed diagnostics, see [[Troubleshooting]]. + +--- + +## Related Links + +* [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] +* [[Llama Core](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] +* [README Installation](https://github.com/JamePeng/llama-cpp-python/blob/main/README.md#installation) +* [llama.cpp build documentation](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md) +* [llama.cpp backend documentation](https://github.com/ggml-org/llama.cpp/tree/master/docs/backend) From 3bcd8010fb89909d6780acb06de8ae0e537e95d9 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 23:26:59 +0800 Subject: [PATCH 161/304] docs(wiki): link installation guide from index Promote the completed installation guide into the wiki entry point so new users can find build and backend setup instructions before reading API-specific documentation. - Add a Getting Started section that links to install.md. - Move installation to the top of the recommended reading order. - Mark install.md as an available page. - Remove installation from the planned documentation areas. Signed-off-by: JamePeng --- docs/wiki/index.md | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/docs/wiki/index.md b/docs/wiki/index.md index c721fc4e89..8e5dbed14b 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -10,6 +10,16 @@ The documentation is maintained with the help of LLMs, but the source of truth i ## Quick Navigation +### Getting Started + +Start here if you are installing or rebuilding `llama-cpp-python`. + +| Page | Description | +|---|---| +| [install\|Installation] | Source installation guide covering Python setup, CMake options, llama.cpp backend selection, hardware acceleration, rebuilds, and verification. | + +--- + ### Core API Start here if you are using `llama-cpp-python` directly. @@ -42,7 +52,7 @@ This section contains maintainer-facing development notes, workflows, and LLM-as | Page | Description | |---|---| -| [[development/Git Commit Generation Agent]] | Helper workflow for generating clear, structured, and source-aware Git commit messages. | +| [development/Git Commit Generation Agent] | Helper workflow for generating clear, structured, and source-aware Git commit messages. | --- @@ -61,13 +71,14 @@ These pages define how the wiki should be written, updated, and reviewed. If you are new to this wiki, read the pages in this order: -1. [[core/Llama|Llama](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] -2. [[modules/LlamaCache|Llama Cache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] -3. [[modules/LlamaEmbedding|Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] -4. [[modules/LlamaGrammar|Llama Grammar](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaGrammar.md)] -5. [[modules/LlamaSpeculative|Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] -6. [[modules/Logger\|Logger](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/Logger.md)] -7. [[development/Git Commit Generation Agent](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/development/git-commit-generation-agent.md)] +1. [[install|Installation](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/install.md)] +2. [[core/Llama|Llama](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] +3. [[modules/LlamaCache|Llama Cache](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaCache.md)] +4. [[modules/LlamaEmbedding|Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] +5. [[modules/LlamaGrammar|Llama Grammar](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaGrammar.md)] +6. [[modules/LlamaSpeculative|Llama Speculative Decoding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaSpeculative.md)] +7. [[modules/Logger\|Logger](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/Logger.md)] +8. [[development/Git Commit Generation Agent](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/development/git-commit-generation-agent.md)] If you are contributing documentation, start with: 1. [[SCHEMA|Wiki Schema](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/SCHEMA.md)] @@ -81,6 +92,7 @@ The wiki is still being expanded. Currently available pages: +- `install.md` - `core/Llama.md` - `modules/LlamaCache.md` - `modules/LlamaEmbedding.md` @@ -99,7 +111,6 @@ Some planned pages may already exist as empty placeholder files. Empty pages are Future documentation may cover: -- Installation and build options - Chat formats and chat handlers - Low-level ctypes bindings - Multimodal APIs @@ -126,5 +137,6 @@ This wiki follows a few core rules: ## Project Links - GitHub: [llama-cpp-python](https://github.com/JamePeng/llama-cpp-python) +- Installation guide: [install](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/install.md) - Wiki schema: [SCHEMA](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/SCHEMA.md) -- Contribution guide: [contributing-to-wiki](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/contributing-to-wiki.md) \ No newline at end of file +- Contribution guide: [contributing-to-wiki](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/contributing-to-wiki.md) From 7cd0f6081251fba1852b4ecd61378d36a229de6e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 2 Jun 2026 23:33:13 +0800 Subject: [PATCH 162/304] docs(readme): link detailed installation wiki guide Signed-off-by: JamePeng --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c5aa1a1b26..ba1969793c 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,8 @@ Thank you for your continuous support! ## Installation +For a structured source-install and backend build guide, see [docs/wiki/install.md](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/install.md). + Requirements: - Python 3.9+ From 14b3b4624065a4b054f4d07a8ac25f999bc7bd87 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 3 Jun 2026 22:26:28 +0800 Subject: [PATCH 163/304] Update Submodule vendor/llama.cpp 60130d1..9e58d4d Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 60130d18f9..9e58d4d692 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 60130d18f9ac7f42cb4d7f6060b088a45d8f242e +Subproject commit 9e58d4d692ed3d350591cc86d06c73c61c122509 From fed47f2d398fcb971595f53b423f59fd7fe0d3c1 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 4 Jun 2026 21:39:53 +0800 Subject: [PATCH 164/304] Update Submodule vendor/llama.cpp 9e58d4d..7c158fb Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 9e58d4d692..7c158fbb4a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 9e58d4d692ed3d350591cc86d06c73c61c122509 +Subproject commit 7c158fbb4aec1bdc9c81d6ca0e785139f4826fae From fff6812e071d3d24fe57e1f635ed2ced51b8cd4e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 6 Jun 2026 00:54:06 +0800 Subject: [PATCH 165/304] Update Submodule vendor/llama.cpp 7c158fb..c4a278d Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7c158fbb4a..c4a278d68e 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7c158fbb4aec1bdc9c81d6ca0e785139f4826fae +Subproject commit c4a278d68efa17811006f2123a84081dac03fac7 From be123f1c55c4ae503d4ae5edc845c310a313d2b2 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 6 Jun 2026 03:41:22 +0800 Subject: [PATCH 166/304] feat(internals): Add `ReasoningBudgetSampler` support - Add Python-backed ReasoningBudgetSampler for first reasoning-block control - Install the sampler before probability filters to preserve forced end tokens - Support reasoning_budget -1/0/N semantics in sampling params - Force reasoning_budget_message + reasoning_end when the budget is exhausted - Add manual force_reasoning_budget() at the sampling-context level - Match llama.cpp force behavior by allowing only COUNTING -> FORCING - Keep DONE as permanent passthrough and ignore later reasoning tags - Support prefilled reasoning starts with reasoning_start_in_prompt - Preserve UTF-8 boundary safety before forcing the end sequence - Keep Python-backed custom sampler callbacks alive across C sampler usage - Avoid shallow-copying custom_samplers when cloning sampler chains Signed-off-by: JamePeng --- llama_cpp/_internals.py | 525 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 504 insertions(+), 21 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 92ff51447f..c308fae056 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -1527,7 +1527,7 @@ def __init__( _existing_sampler: Optional[LlamaSampler] = None, # Internal use for cloning ): if model is None: - raise RuntimeError("model must not be None") + raise RuntimeError("LlamaSamplingContext: model must not be None") self.model = model self.params = params @@ -1537,8 +1537,8 @@ def __init__( lparams = llama_cpp.llama_sampler_chain_default_params() lparams.no_perf = params.no_perf - # history (bounded) - # last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size) + # History (bounded) + # Last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size) if self.params.penalty_last_n == -1: # full context self.params.penalty_last_n = self.model.n_ctx_train() @@ -1551,10 +1551,10 @@ def __init__( ) self.prev = deque(maxlen=max(self.params.n_prev, 32)) - # reusable token data array + # Reusable token data array self._cur_p = LlamaTokenDataArray(n_vocab=self.n_vocab) - # reusable numpy logits view + # Reusable numpy logits view self._logits_view = None self._logits_ptr_addr = None @@ -1566,14 +1566,14 @@ def __init__( sorted=False, ) - # sampler chain + # Sampler chain if _existing_sampler: self.sampler_chain = _existing_sampler else: self.sampler_chain = LlamaSampler() self._build_sampler_chain() - # grammar sampler + # Grammar sampler self.grammar_sampler = None if params.grammar: self.grammar_sampler = GrammarSampler( @@ -1583,6 +1583,9 @@ def __init__( params.grammar_triggers, ) + # Active Python reasoning-budget sampler for this sampling context. + self.reasoning_budget_sampler: Optional[ReasoningBudgetSampler] = None + def _build_sampler_chain(self): """ Build sampler chain aligned with llama.cpp common_sampler_init @@ -1594,7 +1597,7 @@ def _build_sampler_chain(self): m = self.model if m is None: - raise RuntimeError("Model required to build sampler chain firstly") + raise RuntimeError("LlamaSamplingContext: Model required to build sampler chain firstly") use_adaptive_p = False @@ -1628,7 +1631,66 @@ def _build_sampler_chain(self): p.dry_sequence_breakers ) - # --- 5. Core Sampling Strategies (The "Filter" Loop) --- + # --- 5. Reasoning Budget --- + # + # Install before top-k/top-p/min-p filters so the forced end token cannot + # be removed from the candidate set before forcing happens. + # This sampler only controls the first reasoning block. Later blocks are ignored. + if p.reasoning_budget < -1: + raise ValueError( + "LlamaSamplingContext: reasoning_budget must be -1, 0, or a positive integer" + ) + + if p.reasoning_budget >= 0: + start_tokens = None + if not p.reasoning_start_in_prompt: + start_tokens = m.tokenize( + p.reasoning_start.encode("utf-8"), + add_bos=False, + special=True, + ) + if not start_tokens: + raise ValueError("LlamaSamplingContext: reasoning_start produced no tokens") + + end_tokens = m.tokenize( + p.reasoning_end.encode("utf-8"), + add_bos=False, + special=True, + ) + if not end_tokens: + raise ValueError("LlamaSamplingContext: reasoning_end produced no tokens") + + forced_text = (p.reasoning_budget_message or "") + p.reasoning_end + forced_tokens = m.tokenize( + forced_text.encode("utf-8"), + add_bos=False, + special=True, + ) + if not forced_tokens: + raise ValueError("LlamaSamplingContext: reasoning forced text produced no tokens") + + rb_sampler = ReasoningBudgetSampler( + model=m, + reasoning_budget=p.reasoning_budget, + start_tokens=start_tokens, + end_tokens=end_tokens, + forced_tokens=forced_tokens, + initial_state=( + ReasoningBudgetState.COUNTING + if p.reasoning_start_in_prompt + else ReasoningBudgetState.IDLE + ), + start_max_tokens=p.reasoning_start_max_tokens, + wait_utf8=True, + ) + + # Keep a direct Python reference so force_reasoning_budget() can + # manually transition COUNTING -> FORCING at runtime. + self.reasoning_budget_sampler = rb_sampler + + s.add_custom(rb_sampler) + + # --- 6. Core Sampling Strategies (The "Filter" Loop) --- # We iterate through the list to preserve user-defined order for these specific samplers for stype in p.samplers: if stype == CommonSamplerType.CUSTOM: @@ -1660,7 +1722,7 @@ def _build_sampler_chain(self): elif stype == CommonSamplerType.ADAPTIVE_P: use_adaptive_p = True - # --- 6. Final Distribution / Selection --- + # --- 7. Final Distribution / Selection --- # Mirostat overrides standard greedy/dist sampling if p.mirostat == 1 and m: s.add_mirostat(m.n_vocab(), p.seed, p.mirostat_tau, p.mirostat_eta, 100) @@ -1839,6 +1901,10 @@ def close(self): self.sampler_chain.close() self.sampler_chain = None + # Clear the convenience reference used for manual reasoning-budget force. + # The actual sampler lifetime is owned by sampler_chain.close(). + self.reasoning_budget_sampler = None + # Release large token data buffer used during sampling. # Important for high-vocab models to avoid memory retention. if hasattr(self, "_cur_p"): @@ -1885,24 +1951,53 @@ def prev_str(self, ctx_main: LlamaContext, n: int) -> str: # Use the model linked to the context to detokenize return ctx_main.model.detokenize(last_n_tokens).decode("utf-8", errors="replace") + def force_reasoning_budget(self) -> bool: + """ + Manually force the active reasoning-budget sampler to end thinking. + + This mirrors llama.cpp's common_sampler_reasoning_budget_force() + behavior at the Python sampling-context level. + + Returns: + True if the sampler was actively COUNTING inside the first reasoning + block and was transitioned to FORCING. + + False if: + - no reasoning-budget sampler is installed + - the sampler is IDLE + - the sampler is WAITING_UTF8 + - the sampler is already FORCING + - the sampler is DONE + + Important: + Calling this while already FORCING must not rewind force_pos. The + underlying ReasoningBudgetSampler.force() handles this by allowing + only COUNTING -> FORCING. + """ + if self.reasoning_budget_sampler is None: + return False + + return self.reasoning_budget_sampler.force() + class CustomSampler: """ - Python wrapper for llama.cpp custom sampler. + Base class for Python-backed custom samplers in the Llama sampler chain. - apply_func: - Callable receiving llama_token_data_array - and modifying logits in-place. + Responsibilities: + - Provides apply, accept, reset, free and clone callbacks for the C sampler chain. + - Keeps Python references alive to prevent GC while C sampler still holds function pointers. + - Implements safe close to clear all callback references. """ def __init__( self, apply_func: Callable[[llama_cpp.llama_token_data_array], None], - name: str = "custom", accept_func: Optional[Callable] = None, reset_func: Optional[Callable] = None, free_func: Optional[Callable] = None, clone_func: Optional[Callable] = None, + name: str = "custom", ): if not callable(apply_func): raise TypeError("apply_func must be callable") @@ -2002,6 +2097,389 @@ def __del__(self): self.close() +class ReasoningBudgetSampler(CustomSampler): + """ + Generic first-reasoning-block budget sampler. + + This sampler is intentionally model-agnostic. It does not infer model + families, inspect chat templates, or guess reasoning tags. The caller is + responsible for passing the correct reasoning_start and reasoning_end token + sequences. + + Behavior: + 1. Wait for the first reasoning_start token sequence, unless the prompt + already inserted it and initial_state is COUNTING. + 2. Count accepted tokens inside the first reasoning block. + 3. If reasoning_end appears naturally, switch to DONE. + 4. If the budget is exhausted first, force: + reasoning_budget_message + reasoning_end + token by token. + 5. Once DONE, remain passthrough forever. Later reasoning tags are ignored. + + This mirrors the core idea of llama.cpp's reasoning-budget sampler while + keeping the Python API small and explicit. + """ + + def __init__( + self, + *, + model: LlamaModel, + reasoning_budget: int, + start_tokens: Optional[Sequence[int]], + end_tokens: Sequence[int], + forced_tokens: Sequence[int], + initial_state: ReasoningBudgetState = ReasoningBudgetState.IDLE, + start_max_tokens: Optional[int] = 32, + wait_utf8: bool = True, + ): + """ + Initialize the reasoning budget sampler. + + Args: + model: + The active LlamaModel wrapper. Used for token_to_piece() when + checking UTF-8 boundaries. + + reasoning_budget: + Token budget inside the first reasoning block. + Must be >= 0 here. The disabled value -1 is handled before this + sampler is created. + + 0: + Force the end sequence immediately after reasoning starts. + + N > 0: + Allow at most N accepted tokens inside the reasoning block. + + start_tokens: + Token sequence that starts reasoning budget counting. + Must be provided when initial_state is IDLE. + Can be None when initial_state is COUNTING, which is used when + the prompt/chat template has already inserted reasoning_start. + + end_tokens: + Token sequence that naturally ends the reasoning block. + + forced_tokens: + Token sequence forced when the budget is exhausted. This should + normally be tokenized from: + reasoning_budget_message + reasoning_end + + initial_state: + Initial state of the sampler. + IDLE: + Wait for start_tokens during generation. + COUNTING: + Start counting from the first generated token. Use this when + reasoning_start is already present in the prompt. + + start_max_tokens: + Safety window for non-reasoning models. If start_tokens are not + observed within this many generated tokens, the sampler switches + to DONE and becomes a no-op. Set to None to wait indefinitely. + + wait_utf8: + If True, when the budget is exhausted on an incomplete UTF-8 + token piece, wait until a complete UTF-8 boundary before forcing + the end sequence. + """ + if model is None: + raise ValueError("model must not be None") + + if reasoning_budget < 0: + raise ValueError("reasoning_budget must be >= 0") + + self.model = model + + # Maximum number of tokens allowed inside the first reasoning block. + # The disabled value (-1) should be handled before constructing this sampler. + self.reasoning_budget = int(reasoning_budget) + + # Remaining tokens in the active reasoning block. + self.remaining = int(reasoning_budget) + + # Incremental matcher for the first reasoning_start sequence. + # Empty matcher is allowed only when initial_state=COUNTING. + self.start_matcher = TokenMatcher(start_tokens) + + # Incremental matcher for the natural reasoning_end sequence. + self.end_matcher = TokenMatcher(end_tokens) + + # Token sequence forced after budget exhaustion: + # reasoning_budget_message + reasoning_end + self.forced_tokens = list(forced_tokens) + + if initial_state == ReasoningBudgetState.IDLE and not self.start_matcher.tokens: + raise ValueError( + "start_tokens must not be empty when initial_state=IDLE" + ) + + if not self.end_matcher.tokens: + raise ValueError("end_tokens must not be empty") + + if not self.forced_tokens: + raise ValueError("forced_tokens must not be empty") + + # State used by reset(). This is important for templates that already + # insert reasoning_start into the prompt: reset must return to COUNTING, + # not always IDLE. + self.initial_state = ReasoningBudgetState(initial_state) + + # Current runtime state. + self.state = ReasoningBudgetState(initial_state) + + # Index of the next token in forced_tokens to force. + self.force_pos = 0 + + # Count of generated tokens observed by this sampler. + # Used only in IDLE to enforce start_max_tokens. + self.generated_tokens = 0 + + # Maximum number of generated tokens to wait for reasoning_start. + # None means wait indefinitely. + self.start_max_tokens = start_max_tokens + + # Whether to delay forcing until a complete UTF-8 boundary. + self.wait_utf8 = wait_utf8 + + # Keep cloned Python sampler objects alive when llama.cpp clones the + # sampler chain. Without this, cloned Python callbacks could be garbage + # collected while C still holds function pointers to them. + self._clone_keep_alive: List["ReasoningBudgetSampler"] = [] + + if self.state == ReasoningBudgetState.COUNTING and self.remaining <= 0: + self.state = ReasoningBudgetState.FORCING + + super().__init__( + apply_func=self._apply, + accept_func=self._accept, + reset_func=self._reset, + clone_func=self._clone, + name="reasoning-budget", + ) + + def force(self) -> bool: + """ + Manually transition the active reasoning block into forced ending. + + This method is useful for external interruption scenarios, such as: + - user clicks "stop thinking" + - server-side thinking timeout + - UI wants to skip the rest of the reasoning block while still allowing + the model to continue with the final answer + + The transition is allowed only from COUNTING. This matches llama.cpp's + common_reasoning_budget_force() behavior and avoids unsafe rewinding when + the sampler is already FORCING. + """ + if self.state != ReasoningBudgetState.COUNTING: + return False + + self.state = ReasoningBudgetState.FORCING + self.force_pos = 0 + self.end_matcher.reset() + return True + + def _token_utf8_complete(self, token: int) -> bool: + """ + Return whether the token piece is a complete UTF-8 byte sequence. + + This is a safety feature. If the budget is exhausted in the middle of a + multi-byte UTF-8 sequence, the sampler waits until a complete boundary + before forcing reasoning_budget_message + reasoning_end. + """ + if not self.wait_utf8: + return True + + try: + piece = self.model.token_to_piece(token, special=False) + if not piece: + return True + piece.decode("utf-8") + return True + except UnicodeDecodeError: + return False + except Exception: + # Avoid getting stuck forever if token_to_piece behaves unexpectedly. + return True + + def _start_counting(self) -> None: + """ + Enter COUNTING state and initialize the budget window. + + If reasoning_budget is 0, immediately enter FORCING state. + """ + self.state = ReasoningBudgetState.COUNTING + self.remaining = self.reasoning_budget + self.end_matcher.reset() + self.force_pos = 0 + + if self.remaining <= 0: + self.state = ReasoningBudgetState.FORCING + + def _accept(self, token: int) -> None: + """ + Update sampler state after one token has been accepted. + + This method does not modify logits. It only tracks: + - whether reasoning_start has appeared + - whether reasoning_end has appeared + - how much budget remains + - where we are in the forced token sequence + """ + self.generated_tokens += 1 + + if self.state == ReasoningBudgetState.IDLE: + if self.start_matcher.advance(token): + self._start_counting() + return + + # Safety for non-reasoning models: + # + # If no reasoning_start appears near the beginning, assume this + # completion has no visible reasoning block. Switch to DONE forever + # so later literal mentions of reasoning_start do not accidentally + # activate the budget controller. + if ( + self.start_max_tokens is not None + and self.generated_tokens >= self.start_max_tokens + ): + self.state = ReasoningBudgetState.DONE + return + + if self.state in ( + ReasoningBudgetState.COUNTING, + ReasoningBudgetState.WAITING_UTF8, + ): + if self.end_matcher.advance(token): + self.state = ReasoningBudgetState.DONE + return + + utf8_complete = self._token_utf8_complete(token) + + if self.state == ReasoningBudgetState.WAITING_UTF8: + if utf8_complete: + self.state = ReasoningBudgetState.FORCING + self.force_pos = 0 + self.end_matcher.reset() + return + + self.remaining -= 1 + if self.remaining <= 0: + if utf8_complete: + self.state = ReasoningBudgetState.FORCING + self.force_pos = 0 + self.end_matcher.reset() + else: + self.state = ReasoningBudgetState.WAITING_UTF8 + self.end_matcher.reset() + return + + if self.state == ReasoningBudgetState.FORCING: + self.force_pos += 1 + if self.force_pos >= len(self.forced_tokens): + self.state = ReasoningBudgetState.DONE + return + + if self.state == ReasoningBudgetState.DONE: + # Only the first reasoning block is budget-controlled. + # Later reasoning tags are normal generated text. + return + + def _apply(self, cur_p: llama_cpp.llama_token_data_array) -> None: + """ + Apply logits forcing before sampling. + + In FORCING state, only forced_tokens[force_pos] is allowed. All other + candidate logits are set to -inf. The forced token is set to +inf to make + the intent explicit and robust against previous logit modifications. + """ + if self.state != ReasoningBudgetState.FORCING: + return + + if self.force_pos >= len(self.forced_tokens): + return + + forced = self.forced_tokens[self.force_pos] + data = cur_p.data + found = False + + for i in range(cur_p.size): + if data[i].id == forced: + data[i].logit = float("inf") + found = True + else: + data[i].logit = float("-inf") + + cur_p.sorted = False + cur_p.selected = -1 + + if not found: + raise RuntimeError( + f"ReasoningBudgetSampler: forced token {forced} is not present " + "in the candidate array. Move ReasoningBudgetSampler earlier in " + "the sampler chain." + ) + + def _reset(self) -> None: + """ + Reset the sampler to its configured initial state. + + Uses self.initial_state to determine whether to start in: + - IDLE: wait for reasoning_start token sequence + - COUNTING: prompt already contains start token, begin counting immediately + + Also resets internal counters and matchers: + - remaining budget + - generated_tokens + - start_matcher / end_matcher positions + - force_pos + """ + self.state = self.initial_state + self.remaining = self.reasoning_budget + self.generated_tokens = 0 + self.force_pos = 0 + + if self.start_matcher: + self.start_matcher.reset() + self.end_matcher.reset() + + # If initial_state = COUNTING and budget is zero, immediately enter FORCING + if self.state == ReasoningBudgetState.COUNTING and self.remaining <= 0: + self.state = ReasoningBudgetState.FORCING + + def _clone(self): + """ + Clone the full runtime state. + + This mirrors the newer llama.cpp reasoning-budget sampler behavior where + clone copies the full sampler context, not only the static configuration. + """ + cloned = ReasoningBudgetSampler( + model=self.model, + reasoning_budget=self.reasoning_budget, + start_tokens=self.start_matcher.tokens, + end_tokens=self.end_matcher.tokens, + forced_tokens=self.forced_tokens, + initial_state=self.initial_state, + start_max_tokens=self.start_max_tokens, + wait_utf8=self.wait_utf8, + ) + + cloned.remaining = self.remaining + cloned.state = self.state + cloned.force_pos = self.force_pos + cloned.generated_tokens = self.generated_tokens + cloned.start_matcher.pos = self.start_matcher.pos + cloned.end_matcher.pos = self.end_matcher.pos + + # Keep the cloned Python object alive on the source sampler. The cloned + # LlamaSampler wrapper does not own this object directly because the C + # sampler clone is created through the callback. + self._clone_keep_alive.append(cloned) + + return cloned.get_sampler() + class LlamaSampler: def __init__(self, existing_sampler_p: Optional[llama_cpp.llama_sampler_p] = None): if existing_sampler_p: @@ -2055,12 +2533,13 @@ def clone(self) -> 'LlamaSampler': new_sampler = LlamaSampler(existing_sampler_p=new_sampler_p) - # copy _keep_alive and custom_samplers list to new sampler - if self._keep_alive: - new_sampler._keep_alive = self._keep_alive.copy() - - if self.custom_samplers: - new_sampler.custom_samplers = self.custom_samplers.copy() + # llama_sampler_clone() clones C samplers internally. For Python-backed + # custom samplers, the clone_func returns a new C sampler whose Python + # callback object is kept alive by the original custom sampler. Shallow + # copying custom_samplers would make the cloned chain close the original + # Python custom sampler, causing premature close/double-free issues. + new_sampler._keep_alive = self._keep_alive.copy() if self._keep_alive else [] + new_sampler.custom_samplers = [] return new_sampler @@ -2250,6 +2729,10 @@ def add_custom(self, custom_sampler: CustomSampler): [llama_cpp.llama_sampler_chain_n(self.sampler) - 1, custom_sampler] ) + # Keep the Python callback object alive while the C sampler chain holds + # function pointers to it. + self._keep_alive.append(custom_sampler) + def get_seed(self) -> int: assert self.sampler is not None return llama_cpp.llama_sampler_get_seed(self.sampler) From 82a026687eda143c2877fcee96d5f9dbf64d45e6 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 6 Jun 2026 12:10:10 +0800 Subject: [PATCH 167/304] feat(internals): add verbose logging to ReasoningBudgetSampler - Add `verbose` parameter to ReasoningBudgetSampler to print high-level state transitions to stderr. - Log key events: initialization, reasoning_start matched, budget exhausted, forced end sequence, UTF-8 boundary waiting, manual force, natural end, reset. - Pass `verbose=getattr(model, "verbose", False)` from LlamaSamplingContext when building the sampler chain. - Preserve verbose flag when cloning the sampler. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 46 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index c308fae056..434921e6bd 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -3,6 +3,7 @@ import ctypes import enum import os +import sys from typing import ( Callable, @@ -1566,6 +1567,9 @@ def __init__( sorted=False, ) + # Active Python reasoning-budget sampler for this sampling context. + self.reasoning_budget_sampler: Optional[ReasoningBudgetSampler] = None + # Sampler chain if _existing_sampler: self.sampler_chain = _existing_sampler @@ -1583,9 +1587,6 @@ def __init__( params.grammar_triggers, ) - # Active Python reasoning-budget sampler for this sampling context. - self.reasoning_budget_sampler: Optional[ReasoningBudgetSampler] = None - def _build_sampler_chain(self): """ Build sampler chain aligned with llama.cpp common_sampler_init @@ -1682,6 +1683,7 @@ def _build_sampler_chain(self): ), start_max_tokens=p.reasoning_start_max_tokens, wait_utf8=True, + verbose=getattr(m, "verbose", False), ) # Keep a direct Python reference so force_reasoning_budget() can @@ -2131,6 +2133,7 @@ def __init__( initial_state: ReasoningBudgetState = ReasoningBudgetState.IDLE, start_max_tokens: Optional[int] = 32, wait_utf8: bool = True, + verbose: bool = False, ): """ Initialize the reasoning budget sampler. @@ -2182,6 +2185,11 @@ def __init__( If True, when the budget is exhausted on an incomplete UTF-8 token piece, wait until a complete UTF-8 boundary before forcing the end sequence. + + verbose: + If True, print high-level reasoning-budget state transitions to + stderr. Logging is intentionally limited to transitions instead + of per-token events to avoid noisy generation output. """ if model is None: raise ValueError("model must not be None") @@ -2242,6 +2250,10 @@ def __init__( # Whether to delay forcing until a complete UTF-8 boundary. self.wait_utf8 = wait_utf8 + # Whether to print high-level state transition logs. + # This follows the model/runtime verbose flag and avoids per-token spam. + self.verbose = verbose + # Keep cloned Python sampler objects alive when llama.cpp clones the # sampler chain. Without this, cloned Python callbacks could be garbage # collected while C still holds function pointers to them. @@ -2258,6 +2270,19 @@ def __init__( name="reasoning-budget", ) + if self.verbose: + print( + f"ReasoningBudgetSampler: initialized " + f"(state={self.state.name}, budget={self.reasoning_budget}, " + f"start_max_tokens={self.start_max_tokens}, wait_utf8={self.wait_utf8}).", + file=sys.stderr, + ) + + def _log(self, message: str) -> None: + """Print a verbose reasoning-budget state transition message.""" + if self.verbose: + print(f"ReasoningBudgetSampler: {message}", file=sys.stderr) + def force(self) -> bool: """ Manually transition the active reasoning block into forced ending. @@ -2278,6 +2303,7 @@ def force(self) -> bool: self.state = ReasoningBudgetState.FORCING self.force_pos = 0 self.end_matcher.reset() + self._log("manual force requested; entering FORCING state.") return True def _token_utf8_complete(self, token: int) -> bool: @@ -2313,9 +2339,11 @@ def _start_counting(self) -> None: self.remaining = self.reasoning_budget self.end_matcher.reset() self.force_pos = 0 + self._log(f"reasoning_start matched; entering COUNTING state (budget={self.reasoning_budget}).") if self.remaining <= 0: self.state = ReasoningBudgetState.FORCING + self._log("budget is 0; entering FORCING state immediately.") def _accept(self, token: int) -> None: """ @@ -2345,6 +2373,10 @@ def _accept(self, token: int) -> None: and self.generated_tokens >= self.start_max_tokens ): self.state = ReasoningBudgetState.DONE + self._log( + f"reasoning_start not found within {self.start_max_tokens} generated tokens; " + "switching to DONE passthrough." + ) return if self.state in ( @@ -2353,6 +2385,7 @@ def _accept(self, token: int) -> None: ): if self.end_matcher.advance(token): self.state = ReasoningBudgetState.DONE + self._log("reasoning_end matched naturally; switching to DONE passthrough.") return utf8_complete = self._token_utf8_complete(token) @@ -2362,6 +2395,7 @@ def _accept(self, token: int) -> None: self.state = ReasoningBudgetState.FORCING self.force_pos = 0 self.end_matcher.reset() + self._log("UTF-8 boundary reached; entering FORCING state.") return self.remaining -= 1 @@ -2370,15 +2404,18 @@ def _accept(self, token: int) -> None: self.state = ReasoningBudgetState.FORCING self.force_pos = 0 self.end_matcher.reset() + self._log("reasoning budget exhausted; entering FORCING state.") else: self.state = ReasoningBudgetState.WAITING_UTF8 self.end_matcher.reset() + self._log("reasoning budget exhausted; waiting for UTF-8 boundary before forcing.") return if self.state == ReasoningBudgetState.FORCING: self.force_pos += 1 if self.force_pos >= len(self.forced_tokens): self.state = ReasoningBudgetState.DONE + self._log("forced end sequence completed; switching to DONE passthrough.") return if self.state == ReasoningBudgetState.DONE: @@ -2448,6 +2485,8 @@ def _reset(self) -> None: if self.state == ReasoningBudgetState.COUNTING and self.remaining <= 0: self.state = ReasoningBudgetState.FORCING + self._log(f"reset to {self.state.name} state.") + def _clone(self): """ Clone the full runtime state. @@ -2464,6 +2503,7 @@ def _clone(self): initial_state=self.initial_state, start_max_tokens=self.start_max_tokens, wait_utf8=self.wait_utf8, + verbose=self.verbose, ) cloned.remaining = self.remaining From 1b472b354b6d0dbb841b8b29e260a8544453ecf3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 6 Jun 2026 12:13:18 +0800 Subject: [PATCH 168/304] feat: pass reasoning budget params through Llama APIs - Add reasoning budget params to public completion and chat entry points - Forward the params from chat handlers into create_completion - Propagate reasoning budget controls down to generate and sampling params - Document -1/0/N reasoning_budget behavior in completion docstrings - Support custom reasoning_start and reasoning_end tags without model-specific inference - Support reasoning_budget_message and reasoning_start_in_prompt - Wire MTMD chat handler to the same reasoning budget controls Signed-off-by: JamePeng --- llama_cpp/llama.py | 122 +++++++++++++++++++++++++++++++++ llama_cpp/llama_chat_format.py | 40 +++++++++++ 2 files changed, 162 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 43e3d6f1fd..2bab3709e6 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1355,6 +1355,13 @@ def sample( grammar_lazy: bool = False, idx: Optional[int] = None, seed: Optional[int] = None, + # Reasoning Budget Params + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, ): """Sample a token from the model. Returns: @@ -1413,6 +1420,16 @@ def sample( logit_bias=self._convert_logit_bias(logit_bias), grammar=grammar.grammar if grammar else "", grammar_lazy=grammar_lazy, + + # Reasoning Budget + # This generic controller only counts the first visible reasoning + # block. Use reasoning_budget=-1 to leave it disabled. + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, ) # LogitsProcessor Adapter @@ -1487,6 +1504,13 @@ def generate( seed: Optional[int] = None, active_loras: Optional[List[Dict[str, Union[str, float]]]] = None, control_vector: Optional[Dict[str, Any]] = None, + # Reasoning Budget Params + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, ) -> Generator[int, Optional[Sequence[int]], None]: """Create a generator of tokens from a prompt. @@ -1532,6 +1556,18 @@ def generate( grammar: Optional BNF-like grammar (GBNF) to constrain sampling syntax. grammar_lazy: If True, activates grammar constraints only on specific trigger tokens. seed: RNG seed for sampling. Overrides the instance seed. + reasoning_budget: Token budget for the first visible reasoning block. + -1 disables the reasoning budget sampler, 0 forces the block to end + immediately after it starts, and N > 0 allows at most N generated tokens. + reasoning_start: Token/text sequence that marks the beginning of the first reasoning block. + Defaults to "". Pass a model-specific value for non-default tags. + reasoning_end: Token/text sequence that marks the natural and forced end of the reasoning block. + Defaults to "". + reasoning_budget_message: Optional message inserted before reasoning_end when the budget is exhausted. + reasoning_start_in_prompt: Set True when the prompt/template has already inserted reasoning_start, + so counting starts from the first generated token. + reasoning_start_max_tokens: Safety window for non-reasoning models. If reasoning_start is not + generated within this many output tokens, the sampler becomes a no-op. Set None to wait indefinitely. active_loras: A list of dictionaries specifying the LoRA adapters to dynamically apply during generation. Each dictionary must contain a "name" key (matching a LoRA previously loaded into VRAM via `load_lora()`) and an optional "scale" key (float, defaults to 1.0). @@ -1682,6 +1718,16 @@ def generate( grammar=grammar._grammar if grammar else "", grammar_lazy=grammar_lazy, seed=seed if seed is not None else self._seed, + + # Reasoning Budget + # Keeps the core sampler model-agnostic: callers provide the visible + # reasoning start/end tags, and -1 keeps the controller disabled. + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, ) # Register custom python-level logits processors if provided @@ -2065,6 +2111,13 @@ def _create_completion( seed: Optional[int] = None, active_loras: Optional[List[Dict[str, Union[str, float]]]] = None, control_vector: Optional[Dict[str, Any]] = None, + # Reasoning Budget Params + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, ) -> Union[ Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse] ]: @@ -2253,6 +2306,12 @@ def _create_completion( seed=seed if seed is not None else self._seed, active_loras=active_loras, control_vector=control_vector, + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, ): if llama_cpp_lib.llama_token_is_eog(self._model.vocab, token): text = self.detokenize(completion_tokens, prev_tokens=prompt_tokens) @@ -2717,6 +2776,13 @@ def create_completion( grammar_lazy: bool = False, active_loras: Optional[List[Dict[str, Union[str, float]]]] = None, control_vector: Optional[Dict[str, Any]] = None, + # Reasoning Budget Params + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, ) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]: """Generate text from a prompt. @@ -2761,6 +2827,14 @@ def create_completion( logits_processor: A list of logits processors to use. grammar: A grammar to use for constrained sampling. grammar_lazy: If True, enables lazy evaluation. + reasoning_budget: Token budget for the first visible reasoning block. + -1 disables the sampler, 0 forces an immediate end after reasoning starts, + and N > 0 allows at most N generated tokens inside the block. + reasoning_start: Token/text sequence that marks the beginning of the first reasoning block. + reasoning_end: Token/text sequence that naturally and forcibly ends the reasoning block. + reasoning_budget_message: Optional message inserted before reasoning_end when the budget is exhausted. + reasoning_start_in_prompt: Set True when the prompt/template already inserted reasoning_start. + reasoning_start_max_tokens: Safety window before disabling the sampler for non-reasoning outputs. active_loras: A list of dictionaries specifying the LoRA adapters to dynamically apply during generation. Each dictionary must contain a "name" key (matching a LoRA previously loaded into VRAM via `load_lora()`) and an optional "scale" key (float, defaults to 1.0). @@ -2820,6 +2894,12 @@ def create_completion( grammar_lazy=grammar_lazy, active_loras=active_loras, control_vector=control_vector, + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, ) if stream: chunks: Iterator[CreateCompletionStreamResponse] = completion_or_chunks @@ -2871,6 +2951,13 @@ def __call__( grammar_lazy: bool = False, active_loras: Optional[List[Dict[str, Union[str, float]]]] = None, control_vector: Optional[Dict[str, Any]] = None, + # Reasoning Budget Params + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, ) -> Union[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]: """Generate text from a prompt. @@ -2915,6 +3002,14 @@ def __call__( logits_processor: A list of logits processors to use. grammar: A grammar to use for constrained sampling. grammar_lazy: If True, enables lazy evaluation. + reasoning_budget: Token budget for the first visible reasoning block. + -1 disables the sampler, 0 forces an immediate end after reasoning starts, + and N > 0 allows at most N generated tokens inside the block. + reasoning_start: Token/text sequence that marks the beginning of the first reasoning block. + reasoning_end: Token/text sequence that naturally and forcibly ends the reasoning block. + reasoning_budget_message: Optional message inserted before reasoning_end when the budget is exhausted. + reasoning_start_in_prompt: Set True when the prompt/template already inserted reasoning_start. + reasoning_start_max_tokens: Safety window before disabling the sampler for non-reasoning outputs. active_loras: A list of dictionaries specifying the LoRA adapters to dynamically apply during generation. Each dictionary must contain a "name" key (matching a LoRA previously loaded into VRAM via `load_lora()`) and an optional "scale" key (float, defaults to 1.0). @@ -2974,6 +3069,12 @@ def __call__( grammar_lazy=grammar_lazy, active_loras=active_loras, control_vector=control_vector, + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, ) def create_chat_completion( @@ -3025,6 +3126,13 @@ def create_chat_completion( top_logprobs: Optional[int] = None, assistant_prefill: bool = False, add_generation_prompt: bool = True, + # Reasoning Budget Params + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, ) -> Union[ CreateChatCompletionResponse, Iterator[CreateChatCompletionStreamResponse] ]: @@ -3072,6 +3180,14 @@ def create_chat_completion( logits_processor: A list of logits processors to use. grammar: A grammar to use. grammar_lazy: If True, enables lazy evaluation. + reasoning_budget: Token budget for the first visible reasoning block. + -1 disables the sampler, 0 forces an immediate end after reasoning starts, + and N > 0 allows at most N generated tokens inside the block. + reasoning_start: Token/text sequence that marks the beginning of the first reasoning block. + reasoning_end: Token/text sequence that naturally and forcibly ends the reasoning block. + reasoning_budget_message: Optional message inserted before reasoning_end when the budget is exhausted. + reasoning_start_in_prompt: Set True when the prompt/template already inserted reasoning_start. + reasoning_start_max_tokens: Safety window before disabling the sampler for non-reasoning outputs. active_loras: A list of dictionaries specifying the LoRA adapters to dynamically apply during generation. Each dictionary must contain a "name" key (matching a LoRA previously loaded into VRAM via `load_lora()`) and an optional "scale" key (float, defaults to 1.0). @@ -3138,6 +3254,12 @@ def create_chat_completion( control_vector=control_vector, assistant_prefill=assistant_prefill, add_generation_prompt=add_generation_prompt, + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, ) def create_chat_completion_openai_v1( diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index f91844bbb7..f502d68dc9 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -131,6 +131,17 @@ def __call__( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, assistant_prefill: bool = False, + # Reasoning Budget Params + # + # Generic first-reasoning-block budget control. These parameters are + # passed through to llama.create_completion() without model-specific + # inference or template guessing. + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, @@ -829,6 +840,17 @@ def chat_completion_handler( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, assistant_prefill: bool = False, + # Reasoning Budget Params + # + # Generic first-reasoning-block budget control. These parameters are + # passed through to llama.create_completion() without model-specific + # inference or template guessing. + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, @@ -964,6 +986,12 @@ def chat_completion_handler( stopping_criteria=stopping_criteria, grammar=grammar, logit_bias=logit_bias, + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, ) if tool is not None: tool_name = tool["function"]["name"] @@ -3512,6 +3540,12 @@ def __call__( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, add_generation_prompt: bool = True, + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, **kwargs, # type: ignore ) -> Union[ llama_types.CreateChatCompletionResponse, @@ -3772,6 +3806,12 @@ def __call__( logits_processor=logits_processor, grammar=grammar, logit_bias=logit_bias, + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, ) if tool is not None: From fb65ed793957f6a197b910af65e2fe7c7cf215e6 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 6 Jun 2026 20:01:50 +0800 Subject: [PATCH 169/304] Update Submodule vendor/llama.cpp c4a278d..6b80c74 Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 3 ++- llama_cpp/mtmd_cpp.py | 34 ++++++++++++++++++++++++++++------ vendor/llama.cpp | 2 +- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index f502d68dc9..f929bcd150 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3292,7 +3292,8 @@ def _create_bitmap_from_bytes(self, media_bytes: bytes): bitmap = self._mtmd_cpp.mtmd_helper_bitmap_init_from_buf( self.mtmd_ctx, (ctypes.c_uint8 * len(media_bytes)).from_buffer(bytearray(media_bytes)), - len(media_bytes) + len(media_bytes), + False, ) if bitmap is None: diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 839c718ccd..4542555c65 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -326,7 +326,10 @@ def mtmd_get_audio_sample_rate(ctx: mtmd_context_p) -> c_int: # // if bitmap is audio: # // length of data must be n_samples * sizeof(float) # // the data is in float format (PCM F32) - +# // if data == nullptr: +# // the bitmap is considered "empty", and will be treated as a placeholder for counting tokens +# // you can pass the bitmap via mtmd_tokenize(), then call mtmd_*_get_n_tokens() to count the tokens +# // note: passing a placeholder bitmap to mtmd_encode() will return an error # MTMD_API mtmd_bitmap * mtmd_bitmap_init (uint32_t nx, uint32_t ny, const unsigned char * data); @ctypes_function_mtmd( "mtmd_bitmap_init", [ @@ -787,11 +790,22 @@ def mtmd_helper_log_set(log_callback: ggml_log_callback, user_data: c_void_p): # # // it calls mtmd_helper_bitmap_init_from_buf() internally # // returns nullptr on failure # // this function is thread-safe -# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname); +# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); @ctypes_function_mtmd( - "mtmd_helper_bitmap_init_from_file", [mtmd_context_p_ctypes, c_char_p], mtmd_bitmap_p_ctypes) -def mtmd_helper_bitmap_init_from_file(ctx: mtmd_context_p, fname: c_char_p) -> mtmd_bitmap_p: + "mtmd_helper_bitmap_init_from_file", [ + mtmd_context_p_ctypes, + c_char_p, + c_bool, + ], + mtmd_bitmap_p_ctypes +) +def mtmd_helper_bitmap_init_from_file( + ctx: mtmd_context_p, + fname: c_char_p, + placeholder: c_bool, + /, +) -> mtmd_bitmap_p: """ helper function to construct a mtmd_bitmap from a file it calls mtmd_helper_bitmap_init_from_buf() internally @@ -807,13 +821,21 @@ def mtmd_helper_bitmap_init_from_file(ctx: mtmd_context_p, fname: c_char_p) -> m # // note: audio files will be auto-detected based on magic bytes # // returns nullptr on failure # // this function is thread-safe -# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len); +# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); @ctypes_function_mtmd( - "mtmd_helper_bitmap_init_from_buf", [mtmd_context_p_ctypes, POINTER(c_uint8), c_size_t], mtmd_bitmap_p_ctypes) + "mtmd_helper_bitmap_init_from_buf", [ + mtmd_context_p_ctypes, + POINTER(c_uint8), + c_size_t, + c_bool, + ], + mtmd_bitmap_p_ctypes +) def mtmd_helper_bitmap_init_from_buf( ctx: mtmd_context_p, buf: CtypesArray[c_uint8], len: c_size_t, + placeholder: c_bool, /, ) -> mtmd_bitmap_p: """ diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c4a278d68e..6b80c74f28 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c4a278d68efa17811006f2123a84081dac03fac7 +Subproject commit 6b80c74f285390368b3c99c5e750f19e9b096e98 From e001886f18d574b818a1963035fbc35ecfe1287c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 6 Jun 2026 20:15:51 +0800 Subject: [PATCH 170/304] fix(mtmd): memory_can_shift() logic bug Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index f929bcd150..4e7c045127 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3641,7 +3641,7 @@ def __call__( # Stage 5: Multimodal Physical OOM Defense if n_past + chunk_n_tokens > llama.n_ctx(): - if llama._ctx.memory_can_shift(): + if not llama._ctx.memory_can_shift(): raise RuntimeError( f"{self.log_prefix}(__call__): Context Shift is explicitly disabled by the C++ backend " f"(n_pos_per_embd > 1 or incompatible M-RoPE). " From 07afd3bc02cad3af20f10f83973b9a87c770dccb Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 6 Jun 2026 21:07:34 +0800 Subject: [PATCH 171/304] docs(README): document reasoning budget sampler usage - Add README section for first reasoning-block budget control - Document reasoning_budget -1/0/N semantics and related sampler parameters - Explain reasoning_budget_message injection before reasoning_end - Add examples for default tags, Mistral [THINK] tags, and Gemma4 channel tags - Clarify when to use reasoning_start_in_prompt for prefilled thinking tags - Note that reasoning_start_in_prompt is not a generic thinking-enabled switch - Mention verbose transition logs for reasoning-budget state changes Signed-off-by: JamePeng --- README.md | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/README.md b/README.md index ba1969793c..03f66a8cf9 100644 --- a/README.md +++ b/README.md @@ -899,6 +899,83 @@ Mirostat actively maintains a target entropy (`tau`) during generation to preven * **`logits_processor`** (`LogitsProcessorList`, optional): Custom Python callbacks to modify the logits tensor in-place before sampling. * **`stopping_criteria`** (`StoppingCriteriaList`, optional): Custom Python callbacks to halt generation based on the current sequence or scores. + +### Reasoning Budget (First Reasoning Block) + +`llama-cpp-python` provides a generic reasoning-budget sampler for models that expose their thinking content with visible start/end tags. It controls only the **first visible reasoning block** in the generated output. After that block naturally ends or is forcibly closed, the sampler switches to passthrough mode and later reasoning tags are ignored. + +This feature is intentionally model-agnostic. It does not infer model families, inspect chat templates, or guess thinking tags. If a model uses tags other than `...`, pass the correct `reasoning_start` and `reasoning_end` explicitly. + +| Parameter | Default | Description | +| --- | --- | --- | +| `reasoning_budget` | `-1` | Token budget for the first visible reasoning block. `-1` disables the sampler, `0` forces an immediate end after the block starts, and `N > 0` allows at most `N` generated tokens inside the block. | +| `reasoning_start` | `""` | Token/text sequence that marks the beginning of the first reasoning block. | +| `reasoning_end` | `""` | Token/text sequence that naturally ends the reasoning block. When the budget is exhausted, the sampler forces this sequence. | +| `reasoning_budget_message` | `None` | Optional message inserted before `reasoning_end` when the budget is exhausted. | +| `reasoning_start_in_prompt` | `False` | Set to `True` only when the prompt/chat template has already inserted `reasoning_start`, so the sampler should start counting from the first generated token. | +| `reasoning_start_max_tokens` | `32` | Safety window for non-reasoning outputs. If `reasoning_start` is not generated within this many output tokens, the sampler becomes a no-op. Set to `None` to wait indefinitely. | + +Basic usage with the default `...` tags: + +```python +response = llm.create_chat_completion( + messages=[{"role": "user", "content": "Solve this carefully."}], + max_tokens=1024, + reasoning_budget=256, + reasoning_budget_message="\n[reasoning budget exhausted]\n", + # You can also inject a natural-language transition before reasoning_end: + # reasoning_budget_message="\n...Wait, I have been thinking long enough. Let me start answering the user's question.\n", +) +``` +When the budget is exhausted, the sampler forces: `reasoning_budget_message` + `reasoning_end` + +For Mistral-style thinking tags, pass the tags explicitly: + +```python +response = llm.create_chat_completion( + messages=[{"role": "user", "content": "Solve this carefully."}], + max_tokens=1024, + reasoning_budget=256, + reasoning_start="[THINK]", + reasoning_end="[/THINK]", +) +``` + +For Gemma4 channel-style thinking, adjust the start and end markers to match the visible channel tags: + +```python +response = llm.create_chat_completion( + messages=[{"role": "user", "content": "Solve this carefully."}], + max_tokens=1024, + reasoning_budget=256, + reasoning_start="<|channel>", + reasoning_end="", +) +``` + +Use `reasoning_start_in_prompt=True` when the prompt or chat template has already inserted the reasoning start tag. In that case, the sampler will not see the start tag during generation, so it must start directly in `COUNTING` state from the first generated token. This is suitable for thinking models or handlers that prefill the assistant prefix with a thinking tag, for example: + +```text +<|im_start|>assistant\n\n +``` + +Example: + +```python +response = llm.create_chat_completion( + messages=[{"role": "user", "content": "Solve this carefully."}], + max_tokens=1024, + reasoning_budget=256, + reasoning_start="", + reasoning_end="", + reasoning_start_in_prompt=True, +) +``` + +`reasoning_start_in_prompt` is **not** a generic "thinking enabled" switch. It should only be set when the final prompt already contains `reasoning_start` before generation begins. For templates that merely enable thinking but still expect the model to generate the start tag itself, keep `reasoning_start_in_prompt=False`. + +When `verbose=True`, high-level reasoning-budget transitions are printed to stderr, such as initialization, start-tag detection, budget exhaustion, forced ending, and final passthrough. + ### šŸ› ļø Usage Example You can pass these parameters directly when calling the model to generate text. From 504f7477847fe9149e185fe00843681e88ec6736 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 6 Jun 2026 21:14:59 +0800 Subject: [PATCH 172/304] docs(README): Update `ReasoningBudgetSampler` quick link Signed-off-by: JamePeng --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 03f66a8cf9..c605c94542 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ This package provides: - [Dynamic LoRA Example](https://github.com/JamePeng/llama-cpp-python#dynamic-lora-example) - [Control Vector Injection (Representation Engineering)](https://github.com/JamePeng/llama-cpp-python#control-vector-injection-representation-engineering) - [Sampling Configuration & Usage (LlamaSamplingParams)](https://github.com/JamePeng/llama-cpp-python#sampling-configuration--usage-llamasamplingparams) + - [How to use the ReasoningBudgetSampler](https://github.com/JamePeng/llama-cpp-python#reasoning-budget-first-reasoning-block) - [Multi-modal Models Support](https://github.com/JamePeng/llama-cpp-python#multi-modal-models) - Support Models Lists - [Loading a Local Image With Qwen3VL(Thinking/Instruct)](https://github.com/JamePeng/llama-cpp-python#loading-a-local-image-with-qwen3vlthinkinginstruct) From 5929d86c76d248d58daf38045cb38b974e5107d6 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 7 Jun 2026 04:49:25 +0800 Subject: [PATCH 173/304] feat(chat-format): Update google/gemma-4 chat template jinja Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 75 ++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 4e7c045127..fb42a59f23 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -4820,12 +4820,12 @@ class Gemma4ChatHandler(MTMDChatHandler): GEMMA4_ETR_TOKEN = "" CHAT_FORMAT = ( - "{%- macro format_parameters(properties, required) -%}\n" + "{%- macro format_parameters(properties, required, filter_keys=false) -%}\n" " {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n" " {%- set ns = namespace(found_first=false) -%}\n" " {%- for key, value in properties | dictsort -%}\n" " {%- set add_comma = false -%}\n" - " {%- if key not in standard_keys -%}\n" + " {%- if not filter_keys or key not in standard_keys -%}\n" " {%- if ns.found_first %},{% endif -%}\n" " {%- set ns.found_first = true -%}\n" " {{ key }}:{\n" @@ -4887,7 +4887,7 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- elif value is mapping -%}\n" " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" " properties:{\n" - " {{- format_parameters(value, value['required'] | default([])) -}}\n" + " {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}\n" " }\n" " {%- endif -%}\n" " {%- if value['required'] -%}\n" @@ -4910,10 +4910,10 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set params = tool_data['function']['parameters'] -%}\n" " {%- if params -%}\n" " ,parameters:{\n" - " {%- if params['properties'] -%}\n" + " {%- if params.get('properties') -%}\n" " properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n" " {%- endif -%}\n" - " {%- if params['required'] -%}\n" + " {%- if params.get('required') -%}\n" " required:[\n" " {%- for item in params['required'] -%}\n" " <|\"|>{{- item -}}<|\"|>\n" @@ -4921,7 +4921,7 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor -%}\n" " ],\n" " {%- endif -%}\n" - " {%- if params['type'] -%}\n" + " {%- if params.get('type') -%}\n" " type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n" " {%- endif -%}\n" " {%- endif -%}\n" @@ -4978,6 +4978,7 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor -%}\n" " {{- ns.result | trim -}}\n" "{%- endmacro -%}\n" + "\n" "{%- macro format_tool_response_block(tool_name, response) -%}\n" " {{- '<|tool_response>' -}}\n" " {%- if response is mapping -%}\n" @@ -4992,6 +4993,7 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endif -%}\n" " {{- '' -}}\n" "{%- endmacro -%}\n" + "\n" "{%- set ns = namespace(prev_message_type=None) -%}\n" "{%- set loop_messages = messages -%}\n" "{{- bos_token -}}\n" @@ -5004,7 +5006,13 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set ns.prev_message_type = 'think' -%}\n" " {%- endif -%}\n" " {%- if messages[0]['role'] in ['system', 'developer'] -%}\n" - " {{- messages[0]['content'] | trim -}}\n" + " {%- if messages[0]['content'] is string -%}\n" + " {{- messages[0]['content'] | trim -}}\n" + " {%- elif messages[0]['content'] is sequence -%}\n" + " {%- for item in messages[0]['content'] -%}\n" + " {{- item['text'] | trim + ' '-}}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" " {%- set loop_messages = messages[1:] -%}\n" " {%- endif -%}\n" " {%- if tools -%}\n" @@ -5017,6 +5025,7 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endif -%}\n" " {{- '\\n' -}}\n" "{%- endif %}\n" + "\n" "{#- Pre-scan: find last user message index for reasoning guard -#}\n" "{%- set ns_turn = namespace(last_user_idx=-1) -%}\n" "{%- for i in range(loop_messages | length) -%}\n" @@ -5024,6 +5033,7 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set ns_turn.last_user_idx = i -%}\n" " {%- endif -%}\n" "{%- endfor -%}\n" + "\n" "{#- Loop through messages -#}\n" "{%- for message in loop_messages -%}\n" " {%- if message['role'] != 'tool' -%}\n" @@ -5045,12 +5055,14 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- if not continue_same_model_turn -%}\n" " {{- '<|turn>' + role + '\\n' }}\n" " {%- endif -%}\n" + "\n" " {#- Render reasoning/reasoning_content as thinking channel -#}\n" " {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n" " {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n" " {{- '<|channel>thought\\n' + thinking_text + '\\n' -}}\n" " {%- endif -%}\n" - " {%- if message['tool_calls'] -%}\n" + "\n" + " {%- if message.get('tool_calls') -%}\n" " {%- for tool_call in message['tool_calls'] -%}\n" " {%- set function = tool_call['function'] -%}\n" " {{- '<|tool_call>call:' + function['name'] + '{' -}}\n" @@ -5068,6 +5080,7 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor -%}\n" " {%- set ns.prev_message_type = 'tool_call' -%}\n" " {%- endif -%}\n" + "\n" " {%- set ns_tr_out = namespace(flag=false) -%}\n" " {%- if message.get('tool_responses') -%}\n" " {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n" @@ -5104,6 +5117,23 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endif -%}\n" " {%- endfor -%}\n" " {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n" + " {%- for part in tool_body -%}\n" + " {%- if part.get('type') == 'image_url' -%}\n" + " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- elif part.get('type') in ['audio_url', 'input_audio'] -%}\n" + " {%- if part.get('type') == 'audio_url' -%}\n" + " {%- set audio_val = part['audio_url'] if part['audio_url'] is string else part['audio_url']['url'] -%}\n" + " {{- '<|audio|>' + audio_val -}}\n" + " {%- elif part.get('type') == 'input_audio' -%}\n" + " {%- set audio_val = part['input_audio'] if part['input_audio'] is string else ('data:audio/' + part['input_audio']['format'] + ';base64,' + part['input_audio']['data']) -%}\n" + " {{- '<|audio|>' + audio_val -}}\n" + " {%- endif -%}\n" + # " {%- elif part.get('type') == 'video_url' -%}\n" + # " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" + # " {{- '<|video|>' + video_val -}}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" " {%- else -%}\n" " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n" " {%- endif -%}\n" @@ -5112,6 +5142,8 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endif -%}\n" " {%- endfor -%}\n" " {%- endif -%}\n" + "\n" + " {%- set captured_content -%}\n" " {%- if message['content'] is string -%}\n" " {%- if role == 'model' -%}\n" " {{- strip_thinking(message['content']) -}}\n" @@ -5130,28 +5162,35 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" " {{- '<|image|>' + url_val -}}\n" " {%- set ns.prev_message_type = 'image' -%}\n" - " {%- elif item['type'] == 'audio_url' -%}\n" - " {%- set audio_val = item['audio_url'] if item['audio_url'] is string else item['audio_url']['url'] -%}\n" - " {{- '<|audio|>' + audio_val -}}\n" - " {%- set ns.prev_message_type = 'audio' -%}\n" - " {%- elif item['type'] == 'input_audio' -%}\n" - " {%- set audio_val = item['input_audio'] if item['input_audio'] is string else ('data:audio/' + item['input_audio']['format'] + ';base64,' + item['input_audio']['data']) -%}\n" - " {{- '<|audio|>' + audio_val -}}\n" + " {%- elif item['type'] in ['audio_url', 'input_audio'] -%}\n" + " {%- if item['type'] == 'audio_url' -%}\n" + " {%- set audio_val = item['audio_url'] if item['audio_url'] is string else item['audio_url']['url'] -%}\n" + " {{- '<|audio|>' + audio_val -}}\n" + " {%- elif item['type'] == 'input_audio' -%}\n" + " {%- set audio_val = item['input_audio'] if item['input_audio'] is string else ('data:audio/' + item['input_audio']['format'] + ';base64,' + item['input_audio']['data']) -%}\n" + " {{- '<|audio|>' + audio_val -}}\n" + " {%- endif -%}\n" " {%- set ns.prev_message_type = 'audio' -%}\n" + " {%- endif -%}\n" # " {%- elif item['type'] == 'video_url' -%}\n" # " {%- set video_val = item['video_url'] if item['video_url'] is string else item['video_url']['url'] -%}\n" # " {{- '<|video|>' + video_val -}}\n" # " {%- set ns.prev_message_type = 'video' -%}\n" - " {%- endif -%}\n" " {%- endfor -%}\n" " {%- endif -%}\n" + " {%- endset -%}\n" + "\n" + " {{- captured_content -}}\n" + " {%- set has_content = captured_content | trim | length > 0 -%}\n" + "\n" " {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n" " {{- '<|tool_response>' -}}\n" - " {%- elif not (ns_tr_out.flag and not message.get('content')) -%}\n" + " {%- elif not (ns_tr_out.flag and not has_content) -%}\n" " {{- '\\n' -}}\n" " {%- endif -%}\n" " {%- endif -%}\n" "{%- endfor -%}\n" + "\n" "{%- if add_generation_prompt -%}\n" " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n" " {{- '<|turn>model\\n' -}}\n" @@ -5180,7 +5219,7 @@ def __call__(self, **kwargs): self.extra_template_arguments["enable_thinking"] = self.enable_thinking # Set the stop token based on Gemma 4's format () - # generation_config.json: "eos_token_id": [ 1, 106, 50] + # generation_config.json: "eos_token_id": [1, 106, 50] kwargs['stop'] = [self.GEMMA4_EOS_TOKEN, self.GEMMA4_EOT_TOKEN, self.GEMMA4_STR_TOKEN] if self.verbose: From d154e63e2e916e734bee29b3926abe80a9923fce Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 7 Jun 2026 15:42:15 +0800 Subject: [PATCH 174/304] docs(README): update MinerU2.5-Pro-2605-1.2B OCR model support and link Signed-off-by: JamePeng --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c605c94542..433e031ae0 100644 --- a/README.md +++ b/README.md @@ -1034,6 +1034,7 @@ Below are the supported multi-modal models and their respective chat handlers (P | [lfm2-vl](https://huggingface.co/LiquidAI/LFM2-VL-3B-GGUF) | `LFM2VLChatHandler` | `lfm2-vl` | | [lfm2.5-vl](https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B-GGUF) | `LFM25VLChatHandler` | `lfm2.5-vl` | | [deepseek-ocr](https://huggingface.co/JamePeng2023/DeepSeek-OCR-2-GGUF) | `MTMDChatHandler` | `None` | +| [mineru2.5-pro](https://huggingface.co/JamePeng2023/MinerU2.5-Pro-2605-1.2B-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | | [paddleocr-vl-1.5](https://huggingface.co/JamePeng2023/PaddleOCR-VL-1.5-GGUF) | `PaddleOCRChatHandler` | `paddleocr` | | [qwen2.5-vl](https://huggingface.co/unsloth/Qwen2.5-VL-3B-Instruct-GGUF) | `Qwen25VLChatHandler` | `qwen2.5-vl` | | [qwen3-asr](https://huggingface.co/JamePeng2023/Qwen3-ASR-1.7B-GGUF) | `Qwen3ASRChatHandler` | `qwen3-asr` | From db8292d336ae1e708623792426481c414754353e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 7 Jun 2026 17:15:32 +0800 Subject: [PATCH 175/304] Update Submodule vendor/llama.cpp 6b80c74..f71af35 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 6b80c74f28..f71af352a5 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 6b80c74f285390368b3c99c5e750f19e9b096e98 +Subproject commit f71af352a52b8efe824c7a698d0632afa4794c01 From 12861b918f67b62f78f28c5cabb7223f766e1097 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 7 Jun 2026 18:08:31 +0800 Subject: [PATCH 176/304] Bump version to 0.3.40-Milestone - Reasoning Budget Control, Gemma 4 12B Support, Enhanced Jinja2ChatFormatter, NGram k/k4v Speculative Decoding, Faster Native Sampling and Multimodal Improvements Signed-off-by: JamePeng --- CHANGELOG.md | 304 +++++++++++++++++++++++++++++++++++++++++- llama_cpp/__init__.py | 2 +- 2 files changed, 304 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8ebb5cd3e..1865195db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,308 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.40-Milestone] Reasoning Budget Control, Gemma 4 12B Support, Enhanced Jinja2ChatFormatter, NGram k/k4v Speculative Decoding, Faster Native Sampling and Multimodal Improvements + +- feat(internals): Add `ReasoningBudgetSampler` support + - Add Python-backed `ReasoningBudgetSampler` for first reasoning-block control + - Install the sampler before probability filters to preserve forced end tokens + - Support `reasoning_budget` **-1/0/N** semantics in sampling params + - Force `reasoning_budget_message` + `reasoning_end` when the budget is exhausted + - Add manual `force_reasoning_budget()` at the sampling-context level + - Match llama.cpp force behavior by allowing only `COUNTING -> FORCING` + - Keep DONE as permanent passthrough and ignore later reasoning tags + - Support prefilled reasoning starts with `reasoning_start_in_prompt` + - Preserve UTF-8 boundary safety before forcing the end sequence + - Keep Python-backed custom sampler callbacks alive across C sampler usage + - Avoid shallow-copying custom_samplers when cloning sampler chains + - Add `verbose` parameter to `ReasoningBudgetSampler` to print high-level + state transitions to stderr. + - Log key events: initialization, `reasoning_start matched`, `budget exhausted`, + `forced end sequence`, `UTF-8 boundary waiting`, `manual force`, `natural end`, `reset`. + - Pass `verbose=getattr(model, "verbose", False)` from `LlamaSamplingContext` + when building the sampler chain. + - Preserve verbose flag when cloning the sampler. + +- feat(Llama): pass `reasoning budget` params through Llama APIs + - Add `reasoning budget` params to public completion and chat entry points + - Forward the params from chat handlers into `create_completion` + - Propagate reasoning budget controls down to `generate` and `sampling params` + - Document -1/0/N reasoning_budget behavior in completion docstrings + - Support custom `reasoning_start` and `reasoning_end` tags without model-specific inference + - Support `reasoning_budget_message` and `reasoning_start_in_prompt` + - Wire `MTMD chat handler` to the same reasoning budget controls + +- feat(sampling): add reasoning budget configurations + * Introduce reasoning budget and block control parameters to `LlamaSamplingParams` + to mirror llama.cpp CLI semantics. This includes: + - `reasoning_budget` + - `reasoning_start` / `reasoning_end` + - `reasoning_budget_message` + - `reasoning_start_in_prompt` + - `reasoning_start_max_tokens` + - Fix typo from typ_p to typical_p in logs + - Also updated `print_params()` to include these new metrics. + +- feat: add `ReasoningBudgetState` enum and `TokenMatcher` helper class to _internals.py + * Introduce `ReasoningBudgetState` enum and `TokenMatcher` helper class + to `_internals.py`. This lays the groundwork for the upcoming + `ReasoningBudgetSampler`, mirroring the state machine defined in + `common/reasoning-budget.h`. + + - `ReasoningBudgetState`: Tracks the lifecycle of the first reasoning block. + - `TokenMatcher`: Handles incremental matching for multi-token sequences. + +- docs(README): document reasoning budget sampler usage + - Add README section for first reasoning-block budget control + - Document reasoning_budget -1/0/N semantics and related sampler parameters + - Explain reasoning_budget_message injection before reasoning_end + - Add examples for default tags, Mistral [THINK] tags, and Gemma4 channel tags + - Clarify when to use reasoning_start_in_prompt for prefilled thinking tags + - Note that reasoning_start_in_prompt is not a generic thinking-enabled switch + - Mention verbose transition logs for reasoning-budget state changes + - docs(README): Update ReasoningBudgetSampler quick link + +- feat(chat-format): Update `google/gemma-4` chat template jinja + +- feat(llama): enhance chat template initialization with full special tokens + * Update Llama.__init__ to register additional tokenizer special tokens + and improve stop token handling for chat templates. + + - Expose extra special tokens (EOT, SEP, NL, PAD, MASK) via + `special_tokens_map` to Jinja2ChatFormatter. + - Keep BOS and EOS tokens as explicit parameters, no longer redundantly + put them in `special_tokens_map`. + - Build `stop_token_ids` once, including EOS and EOT tokens, skipping + invalid (-1) ids. + - Update try-block comment: now `{% generation %}` blocks are supported, + guard only against malformed or model-specific templates. + - This ensures better compatibility with HuggingFace-style chat templates + while maintaining llama-cpp-python prompt-rendering behavior. + +- **feat(chat-format): improve Jinja2ChatFormatter HF compatibility** + * Enhance Jinja2ChatFormatter to better support HuggingFace-style chat + templates while keeping the formatter lightweight and aligned with + llama-cpp-python's prompt-rendering needs. + + - Key changes: + - Add IgnoreGenerationTags Jinja extension for HF `{% generation %}` blocks. + - Enable Jinja loop controls for chat templates using break/continue. + - Register Transformers-compatible `tojson` behavior. + - Register `raise_exception` and `strftime_now` as Jinja globals. + - Add `special_tokens_map` support for additional template variables. + - Add optional `documents` argument for document-aware templates. + - Precompute text stop sequences and token-id stopping criteria. + - Improve type normalization for `stop_token_ids`. + - Expand docstrings for formatter initialization and render-time variables. + +- docs(wiki): update SCHEMA.md to v0.4 with full wiki path layout + - Added comprehensive docs/wiki/ directory structure overview. + - Reorganized modules description; removed hardcoded module page list. + - Clarified top-level file purposes and update guidance. + - Updated page type examples and templates (Class/Module, Feature, Example, Development). + - Strengthened cross-linking rules and update/placeholder guidance. + - Bumped schema version from 0.3 ā†’ 0.4 and last_modified date. + +- docs(install): add source-aligned build and backend guide + * Document installation workflows for llama-cpp-python with a focus on + the underlying llama.cpp CMake build configuration. + - Add virtual environment, source install, editable install, rebuild, and + verification guidance. + - Document common CMake options such as GGML_NATIVE, + GGML_BACKEND_DL, GGML_CPU_ALL_VARIANTS, and compiler selection. + - Summarize backend-specific build flags for CUDA, BLAS, Metal, Vulkan, + OpenVINO, HIP, SYCL, OpenCL, CANN, ZenDNN, and zDNN. + - Include backend runtime notes and common installation pitfalls while + keeping server-related installation content out of the page. + - docs(wiki): link installation guide from index + * Promote the completed installation guide into the wiki entry point so + new users can find build and backend setup instructions before reading + API-specific documentation. + - Add a Getting Started section that links to install.md. + - Move installation to the top of the recommended reading order. + - Mark install.md as an available page. + - Remove installation from the planned documentation areas. + - docs(readme): link detailed installation wiki guide + +- feat(mtmd): improve fallback chat template for multimodal models + - Add BOS/EOS token handling to the default MTMD chat format. + - Use a clearer role-based template with explicit USER and ASSISTANT prefixes. + - Append a newline after each message to keep generated prompts readable. + - Treat EOS as the end marker for the serialized conversation history before + the optional generation prompt. + - Improve fallback behavior for multimodal GGUF models that do not provide a + chat template, such as OCR-oriented models like `DeepSeek-OCR 1/2`. + - Make the default system prompt a single normalized string while preserving + its original meaning. + - Clean up minor formatting around MTMD context parameter initialization. + - docs(Readme): Update `Deepseek-OCR-2-GGUF` Link + - docs(README): update `MinerU2.5-Pro-2605-1.2B` OCR model support and link + + This improves prompt compatibility for multimodal models that either lack a + GGUF chat template or are not yet covered by a complete custom chat handler. + +- refactor(internals): align model metadata wrappers with llama.cpp API + - Use `llama_vocab_n_tokens()` instead of the old vocab size helper. + - Add Python wrappers for model description, size, chat template, and + trained RoPE frequency scaling. + - Clarify model capability helpers with docstrings matching llama.cpp + semantics. + - Rename `desc()` and `size()` to `model_desc()` and `model_size()` to + make their scope explicit. + - Drop the unused `get_tensor()` stub since llama.cpp does not expose it. + - Route rerank template lookup through `LlamaModel.model_chat_template()` for + consistency with the internal model abstraction. + +- feat(chat_handler): update multimodal handlers for Qwen2.5-VL, Qwen3-VL, and PaddleOCR + - Update PaddleOCRChatHandler to support version 1.6 + - Add token configuration and stop sequences for Qwen2.5-VL and Qwen3-VL + - Standardize input_ids initialization in __call__ methods for Qwen2.5-VL, Qwen3-ASR, and Qwen3-VL handlers + +- **perf(eval): skip unnecessary logit array copies during native sampling** + * Introduce the `copy_logits` parameter to `Llama.eval()` to control + whether C-level logits are copied into the Python `self.scores` array. + - Automatically disable `copy_logits` during the generation loop unless + Python-side hooks (`logits_processor`, `stopping_criteria`) or + `logits_all` explicitly require them. + - Skip logit copies entirely for intermediate prompt evaluations (e.g., + before hybrid checkpoints). + - Update logit retrieval to use `get_logits_ith(-1)` to accurately fetch + the final token's logits when copying is required. + + In a PDF-reading summarization workload, this reduced the end-to-end completion + time from 41.32s to 25.93s, a ~37.2% improvement. The main generation hot path + also improved noticeably: + + - `_create_completion`: 41.32s -> 25.93s + - `generate`: 37.82s -> below the top sampled entries + - `eval`: 35.14s -> 21.96s + - logits retrieval/copy path: 29.89s `get_logits()` -> 18.68s `get_logits_ith()` + - `decode`: 3.89s -> 2.25s + - `detokenize`: 2.60s -> 1.33s + - `sample`: 2.35s -> 2.03s + + This significantly reduces CPU overhead and memory bandwidth during generation, + as the native `llama.cpp` sampler reads directly from the C context without + needing to expose the `n_vocab` array to Python on every token. + +- docs(CUDA): Add note about PDL optimization for newer NVIDIA GPUs (CC ā‰„ 90) + +- docs(readme/wiki): update supported embeddings models table + - Add `jina-embeddings-v2-base-zh` + - Add `jina-embeddings-v3` + - Minor table formatting clean up + +- docs(development): add AI agent prompt for git commit generation + * Introduce `git-commit-generation-agent.md` to the development wiki to + standardize the creation of high-quality git commit messages using LLM + assistants. + + - Define the system persona, core principles (Conventional Commits, DCO), + and strict formatting rules for generating commits. + - Provide concrete template examples for build, performance, and + documentation updates. + - Ensure future maintainers and contributors can easily generate + consistent, maintainer-level commits that explicitly explain the "Why" + and "How" of code changes. + +- docs(wiki): add development helper to index + * Introduce the development section in the wiki index so maintainer-facing + workflows and LLM-assisted helper tools are discoverable from the main + navigation. + + - Add a Development section with a link to the Git commit generation agent. + Include the helper in the recommended reading order for new wiki users. + - Add development/git-commit-generation-agent.md to the available pages list. + +- feat(LlamaContext): add safety checks and docstrings to logits retrieval + - Add explicit null pointer validation to `get_logits` and `get_logits_ith`. + These methods now raise a `RuntimeError` instead of silently returning + invalid pointers when logits are unavailable or the index is out of bounds. + - Add comprehensive docstrings to both methods, detailing the underlying + buffer shape and memory layout. + - Include a performance warning in `get_logits_ith` about the internal + synchronization/reordering overhead to discourage its use on the hot path. + +- **feat(speculative): upgrade ngram map decoder with k/k4v modes +Enhance `LlamaNGramMapDecoding` to align with the upstream llama.cpp +ngram-map algorithm, offering better memory management and draft quality.** + - Introduce `mode` selection ("k" and "k4v"): "k" stores only historical + positions for memory efficiency, while "k4v" caches continuation values + directly for faster lookups. + - Add `min_hits` threshold to filter out low-confidence drafts. + - Implement `max_entries_per_key` to cap dictionary growth and prevent + memory bloat during long-context generations. + - Improve state synchronization (`_sync_and_index`) using `sync_check_tokens` + to safely verify incremental history appends. + - Add explicit lifecycle management methods (`clear`, `close`, `accept`) + for better API symmetry and resource cleanup. + - examples: add benchmark script for speculative decoding + - Add `benchmark_speculative.py` to the `examples/benchmark` directory. + - Test `LlamaPromptLookupDecoding` and `LlamaNGramMapDecoding` (k/k4v). + - Include diverse test scenarios (code, JSON logs, tables, essays) to + measure tokens-per-second (TPS) speedup compared to baseline generation. + +- docs(speculative): update wiki for NGramMap k/k4v modes and lifecycle APIs +Reflect the recent architectural upgrades to `LlamaNGramMapDecoding` in +the official documentation. + + - Document the new `__init__` parameters (`mode`, `min_hits`, + `max_entries_per_key`, `sync_check_tokens`) and their validation rules. + - Add a detailed comparison table explaining the memory and behavior + differences between the `"k"` and `"k4v"` lookup modes. + - Document the newly exposed lifecycle methods (`clear`, `close`, `accept`). + - Add comprehensive usage examples demonstrating `k4v` mode with memory caps. + - Update internal state descriptions (replacing `_ngram_map` with `_map_k` + and `_map_k4v`). + - Add a strong production warning against the legacy `LlamaPromptLookupDecoding` + and cross-link the new `benchmark_speculative.py` script. + +- docs(readme): revamp speculative decoding documentation +Expand the Speculative Decoding section to fully document the +new `LlamaNGramMapDecoding` capabilities and configuration options. + + - Clarify that `LlamaNGramMapDecoding` is a model-free prompt lookup + decoder that does not require a secondary GGUF draft model. + - Add a detailed parameter table explaining `mode` (k vs. k4v), + `min_hits`, memory caps, and sync thresholds. + - Provide usage examples and tuning recommendations for different + hardware (e.g., lowering `num_pred_tokens` for CPU setups). + - Demote the older `LlamaPromptLookupDecoding` to a legacy section, + warning about its sliding-window overhead on long contexts. + - Add practical notes on performance and state management (`clear()`). + +- docs(readme): Removed outdated macOS installation guides and added the latest installation notes. + +- docs(readme): Add Windows ROCm build instructions(by **@0xDELUXA**) + - Optimize the formatting of the ROCm section in README.md. + +- fix: wire LFM VL chat handlers into server loader(by **@JayAnderson360**) + +- build(cmake): disable building of upstream unified binary + - Set `LLAMA_BUILD_APP` to `OFF` to prevent the compilation of the new + unified `llama` binary introduced in upstream llama.cpp. + + - Since the Python package only requires the underlying shared libraries + and specific targets, explicitly disabling the standalone application + reduces build times and prevents unnecessary executable artifacts from + being compiled. + +- build(deps): align Jinja2 minimum with Transformers + - Require Jinja2 >= 3.1.0 for HuggingFace-style chat template support. + + - The updated Jinja2ChatFormatter relies on behavior aligned with Transformers' + chat-template runtime, which also requires Jinja2 3.1 or newer. Updating the + minimum dependency avoids parser/runtime differences with older Jinja versions. + +- ci : update metal build/test job to macos-26/macos-15-intel + - Build on the Tahoe runners in order to enable the tensor API for M5 and A19. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/f71af352a52b8efe824c7a698d0632afa4794c01](https://github.com/ggml-org/llama.cpp/commit/f71af352a52b8efe824c7a698d0632afa4794c01) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260606 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/a778c57d73ec7d4f43e2518a513e7d4cf68a0df8...db8292d336ae1e708623792426481c414754353e + ## [0.3.39] Dynamic GGML Backends, Qwen3-ASR/MiniCPM-V-4.6, On-Device Hybrid Checkpoint, and Granular Logging - **ci(cu131/128/126/124): build wheels with GGML dynamic backends for windows/Linux** @@ -513,7 +815,7 @@ This commit significantly overhauls the media parsing and loading pipeline in `M - feat: Update llama.cpp to [ggml-org/llama.cpp/commit/f5ddcd1696eca5069dc7915f4d4c03c9a709afea](https://github.com/ggml-org/llama.cpp/commit/f5ddcd1696eca5069dc7915f4d4c03c9a709afea) -## [0.3.30] Milestone Release +## [0.3.30-Milestone] Milestone Release I will update the release notes for version 0.3.30 in the [discussion](https://github.com/JamePeng/llama-cpp-python/discussions). diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index ec28faae66..1650e6af69 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.39" +__version__ = "0.3.40" From b1ad4452e24baac561e75b254192ebf55f1fbd3c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 8 Jun 2026 01:03:44 +0800 Subject: [PATCH 177/304] Update Submodule vendor/llama.cpp f71af35..f0156d1 Signed-off-by: JamePeng --- llama_cpp/llama.py | 1 + llama_cpp/llama_cpp.py | 7 +++++++ vendor/llama.cpp | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2bab3709e6..d89f4c361d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -503,6 +503,7 @@ def __init__( self.context_params.n_threads_batch = self.n_threads_batch self.context_params.ctx_type = ctx_type + self.context_params.ctx_other = None self.context_params.rope_scaling_type = ( rope_scaling_type if rope_scaling_type is not None diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 9c911bcb14..1e81d80f65 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -898,6 +898,9 @@ class llama_sampler_seq_config(ctypes.Structure): # // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init) # struct llama_sampler_seq_config * samplers; # size_t n_samplers; +# // a source/target/parent context +# // can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts +# struct llama_context * ctx_other; # }; class llama_context_params(ctypes.Structure): """Parameters for llama_context @@ -945,6 +948,8 @@ class llama_context_params(ctypes.Structure): samplers(llama_sampler_seq_config *): the samplers must be sampler chains (i.e. use llama_sampler_chain_init) n_samplers(size_t): numbers of sampler chains + + ctx_other(llama_context *): a source/target/parent context can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts """ if TYPE_CHECKING: @@ -983,6 +988,7 @@ class llama_context_params(ctypes.Structure): kv_unified:bool samplers: ctypes.c_void_p n_samplers: int + ctx_other: ctypes.c_void_p _fields_ = [ ("n_ctx", ctypes.c_uint32), @@ -1020,6 +1026,7 @@ class llama_context_params(ctypes.Structure): ("kv_unified", ctypes.c_bool), ("samplers", llama_sampler_seq_config_p), ("n_samplers", ctypes.c_int), + ("ctx_other", ctypes.c_void_p), ] llama_context_params_p = ctypes.POINTER(llama_context_params) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f71af352a5..f0156d1401 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f71af352a52b8efe824c7a698d0632afa4794c01 +Subproject commit f0156d1401500512ad85042ccf38970568b12253 From 7a8272e6b928974efc8c131d518b1363e2e47263 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 8 Jun 2026 01:24:53 +0800 Subject: [PATCH 178/304] feat(_ctypes_extensions): improve error diagnostics for shared library loading When `load_shared_library` fails, the resulting `RuntimeError` now includes a listing of the contents of the searched directories. This provides immediate context to help developers diagnose missing, misplaced, or incorrectly named library files. - Added `_format_library_dir_contents` to safely format directory listings. - Appended the directory listing to the failure message. - Confined this diagnostic work strictly to the failure path to avoid any performance overhead during successful imports. Signed-off-by: JamePeng --- llama_cpp/_ctypes_extensions.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index a8936fa2bf..1a9f8eb8c5 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -18,6 +18,37 @@ ) from typing_extensions import TypeAlias +def _format_library_dir_contents(base_paths: list[pathlib.Path]) -> str: + """Format directory contents for diagnostics after library loading fails.""" + sections = [] + + for base_path in base_paths: + p = pathlib.Path(base_path) + + if not p.exists(): + sections.append(f"{p}: ") + continue + + if not p.is_dir(): + sections.append(f"{p}: ") + continue + + try: + # Only list files when reporting a final loading failure. + files = sorted(x.name for x in p.iterdir()) + except Exception as e: + sections.append(f"{p}: ") + continue + + if files: + sections.append( + f"{p}:\n" + + "\n".join(f" - {name}" for name in files) + ) + else: + sections.append(f"{p}: ") + + return "\n".join(sections) # Load the library def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list[pathlib.Path]]): @@ -114,9 +145,12 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list except Exception as e: errors.append(f"{lib_path}: {e}") + # Include directory contents only in the failure path to avoid extra work during successful imports. raise RuntimeError( f"Failed to load '{lib_base_name}' from {base_paths}\n" + "\n".join(errors) + + "\nLibrary search path contents:\n" + + _format_library_dir_contents(base_paths) ) From 323da373ad2f30409123bfba8322041113f0eba8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 8 Jun 2026 23:08:36 +0800 Subject: [PATCH 179/304] build(CMakelists): Improve Windows LLVM OpenMP runtime discovery - Also improve diagnostics by reporting the selected runtime source and path, warning when an explicit override points to a missing file, and keeping a clear runtime warning when no OpenMP DLL can be found. Signed-off-by: JamePeng --- CMakeLists.txt | 79 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f09cdb783..1ace43c4aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,7 +60,8 @@ function(llama_cpp_python_install_target target) endfunction() -# Install an extra Windows runtime DLL into the Python package runtime directory. +# Copy an extra Windows runtime DLL into the Python package runtime directory +# during the CMake install step. # # Some dynamically loaded backend libraries depend on runtime DLLs that are not # always discoverable through $. One important example @@ -75,7 +76,10 @@ function(llama_cpp_python_install_windows_runtime_file runtime_file) endif() if(NOT EXISTS "${runtime_file}") - message(WARNING "Windows runtime file does not exist and will not be installed: ${runtime_file}") + message(WARNING + "Windows runtime DLL was selected but does not exist and will not be copied: " + "${runtime_file}" + ) return() endif() @@ -92,6 +96,11 @@ function(llama_cpp_python_install_windows_runtime_file runtime_file) foreach(DIR ${INSTALL_DIRS}) file(TO_CMAKE_PATH "${DIR}" DIR_CMAKE) + message(STATUS + "Will copy Windows runtime DLL during install: " + "${runtime_file_cmake} -> ${DIR_CMAKE}" + ) + install( FILES "${runtime_file_cmake}" DESTINATION "${DIR_CMAKE}" @@ -115,42 +124,62 @@ function(llama_cpp_python_install_windows_openmp_runtime) endif() set(OPENMP_RUNTIME_DLL "") + set(OPENMP_RUNTIME_SOURCE "") + set(FOUND_OPENMP_DLLS "") + + if(DEFINED LLAMA_CPP_OPENMP_RUNTIME_DLL) + if(EXISTS "${LLAMA_CPP_OPENMP_RUNTIME_DLL}") + set(OPENMP_RUNTIME_DLL "${LLAMA_CPP_OPENMP_RUNTIME_DLL}") + set(OPENMP_RUNTIME_SOURCE "LLAMA_CPP_OPENMP_RUNTIME_DLL") + else() + message(WARNING + "LLAMA_CPP_OPENMP_RUNTIME_DLL was set, but the file does not exist: " + "${LLAMA_CPP_OPENMP_RUNTIME_DLL}. Falling back to Visual Studio " + "LLVM OpenMP runtime discovery." + ) + endif() + endif() - if(DEFINED LLAMA_CPP_OPENMP_RUNTIME_DLL AND EXISTS "${LLAMA_CPP_OPENMP_RUNTIME_DLL}") - set(OPENMP_RUNTIME_DLL "${LLAMA_CPP_OPENMP_RUNTIME_DLL}") - else() + if(NOT OPENMP_RUNTIME_DLL) file(TO_CMAKE_PATH "$ENV{ProgramFiles}" PROGRAMFILES_CMAKE) file(TO_CMAKE_PATH "$ENV{ProgramFiles\(x86\)}" PROGRAMFILES_X86_CMAKE) - set(VS_OPENMP_SEARCH_ROOTS - "${PROGRAMFILES_CMAKE}/Microsoft Visual Studio/2022/Enterprise/VC/Redist/MSVC" - "${PROGRAMFILES_CMAKE}/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" - "${PROGRAMFILES_X86_CMAKE}/Microsoft Visual Studio/2022/Enterprise/VC/Redist/MSVC" - "${PROGRAMFILES_X86_CMAKE}/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC" - ) + set(VS_OPENMP_SEARCH_PATTERNS + # Prefer the exact VS 2022 Enterprise / BuildTools LLVM OpenMP redist layout. + "${PROGRAMFILES_CMAKE}/Microsoft Visual Studio/2022/Enterprise/VC/Redist/MSVC/*/debug_nonredist/x64/Microsoft.VC143.OpenMP.LLVM/libomp140.x86_64.dll" + "${PROGRAMFILES_X86_CMAKE}/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC/*/debug_nonredist/x64/Microsoft.VC143.OpenMP.LLVM/libomp140.x86_64.dll" - foreach(ROOT ${VS_OPENMP_SEARCH_ROOTS}) - if(EXISTS "${ROOT}") - file( - GLOB_RECURSE FOUND_OPENMP_DLLS - "${ROOT}/*/debug_nonredist/x64/Microsoft.VC*.OpenMP.LLVM/libomp140.x86_64.dll" - "${ROOT}/**/libomp140.x86_64.dll" - ) + # Keep these as secondary fallbacks for non-standard installs. + "${PROGRAMFILES_CMAKE}/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC/*/debug_nonredist/x64/Microsoft.VC143.OpenMP.LLVM/libomp140.x86_64.dll" + "${PROGRAMFILES_X86_CMAKE}/Microsoft Visual Studio/2022/Enterprise/VC/Redist/MSVC/*/debug_nonredist/x64/Microsoft.VC143.OpenMP.LLVM/libomp140.x86_64.dll" + "C:/Windows/System32/libomp140.x86_64.dll" + ) - if(FOUND_OPENMP_DLLS) - list(GET FOUND_OPENMP_DLLS 0 OPENMP_RUNTIME_DLL) - break() - endif() - endif() + foreach(PATTERN ${VS_OPENMP_SEARCH_PATTERNS}) + file(GLOB PATTERN_OPENMP_DLLS "${PATTERN}") + list(APPEND FOUND_OPENMP_DLLS ${PATTERN_OPENMP_DLLS}) endforeach() + + if(FOUND_OPENMP_DLLS) + list(REMOVE_DUPLICATES FOUND_OPENMP_DLLS) + list(SORT FOUND_OPENMP_DLLS COMPARE NATURAL ORDER DESCENDING) + list(GET FOUND_OPENMP_DLLS 0 OPENMP_RUNTIME_DLL) + set(OPENMP_RUNTIME_SOURCE "Visual Studio 2022 LLVM OpenMP redist fallback") + endif() endif() if(OPENMP_RUNTIME_DLL) - message(STATUS "Installing Windows LLVM OpenMP runtime: ${OPENMP_RUNTIME_DLL}") + message(STATUS + "Selected Windows LLVM OpenMP runtime from ${OPENMP_RUNTIME_SOURCE}: " + "${OPENMP_RUNTIME_DLL}" + ) llama_cpp_python_install_windows_runtime_file("${OPENMP_RUNTIME_DLL}") else() message(WARNING - "Could not find libomp140.x86_64.dll. " + "Could not find libomp140.x86_64.dll for Windows LLVM OpenMP. " + "Searched LLAMA_CPP_OPENMP_RUNTIME_DLL and Visual Studio 2022 " + "Enterprise/BuildTools redist paths under Program Files and Program Files (x86), " + "with a fuzzy MSVC version match such as 14.44.35112 or 14.44.35207. " "If GGML_OPENMP=ON and GGML CPU backend DLLs are built with LLVM OpenMP, " "the packaged ggml-cpu-*.dll files may fail to load at runtime. " "Set LLAMA_CPP_OPENMP_RUNTIME_DLL to the full path of libomp140.x86_64.dll " From 111819832614d488c840b266ad95f894f420bfea Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 8 Jun 2026 23:48:49 +0800 Subject: [PATCH 180/304] ci(test): add cuda 13.0.2 build workflow Signed-off-by: JamePeng --- .github/workflows/build-wheels-cu130-win.yml | 249 +++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 .github/workflows/build-wheels-cu130-win.yml diff --git a/.github/workflows/build-wheels-cu130-win.yml b/.github/workflows/build-wheels-cu130-win.yml new file mode 100644 index 0000000000..790d7c9665 --- /dev/null +++ b/.github/workflows/build-wheels-cu130-win.yml @@ -0,0 +1,249 @@ +name: Build Wheels (CU130) for Windows + +on: + workflow_dispatch: + +permissions: + contents: write + +jobs: + build_wheels: + name: Build Wheel ${{ matrix.os }} py${{ matrix.pyver }} cu130 + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: ["windows-2022"] + pyver: ["3.10", "3.11", "3.12", "3.13", "3.14"] + cuda: ["13.0.2"] + cudaarch: ["75-real;80-real;86-real;87-real;89-real;90-real;100-real;120-real"] + + defaults: + run: + shell: pwsh + + env: + CUDAVER: ${{ matrix.cuda }} + CUDAARCHVER: ${{ matrix.cudaarch }} + MAX_JOBS: 12 + + steps: + - name: Add MSBuild to PATH + uses: microsoft/setup-msbuild@v3 + with: + msbuild-architecture: x64 + + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Inspect Visual Studio OpenMP runtime paths + run: | + Write-Output "ProgramFiles=$env:ProgramFiles" + Write-Output "ProgramFiles(x86)=${env:ProgramFiles(x86)}" + Write-Output "" + + $vsRoots = @( + "$env:ProgramFiles\Microsoft Visual Studio\2022\Enterprise\VC\Redist\MSVC", + "$env:ProgramFiles\Microsoft Visual Studio\2022\BuildTools\VC\Redist\MSVC", + "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\Enterprise\VC\Redist\MSVC", + "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Redist\MSVC" + ) + + foreach ($root in $vsRoots) { + Write-Output "Checking root: $root" + + if (Test-Path $root) { + Write-Output " Exists: yes" + Write-Output " MSVC version directories:" + + Get-ChildItem $root -Directory -ErrorAction SilentlyContinue | + Sort-Object Name | + ForEach-Object { + Write-Output " $($_.FullName)" + } + + Write-Output " OpenMP runtime candidates:" + + Get-ChildItem $root -Recurse -Filter "libomp140.x86_64.dll" -ErrorAction SilentlyContinue | + Sort-Object FullName | + ForEach-Object { + $sizeKB = [Math]::Round($_.Length / 1KB, 2) + $sizeMB = [Math]::Round($_.Length / 1MB, 4) + + Write-Output " Path: $($_.FullName)" + Write-Output " Size: $($_.Length) bytes / $sizeKB KB / $sizeMB MB" + } + } else { + Write-Output " Exists: no" + } + + Write-Output "" + } + + Write-Output "Checking System32 fallback:" + $system32OpenMP = "C:\Windows\System32\libomp140.x86_64.dll" + + if (Test-Path $system32OpenMP) { + $dll = Get-Item $system32OpenMP + $sizeKB = [Math]::Round($dll.Length / 1KB, 2) + $sizeMB = [Math]::Round($dll.Length / 1MB, 4) + + Write-Output " Path: $($dll.FullName)" + Write-Output " Size: $($dll.Length) bytes / $sizeKB KB / $sizeMB MB" + } else { + Write-Output " Not found: $system32OpenMP" + } + + - name: Install CUDA ${{ matrix.cuda }} + uses: Jimver/cuda-toolkit@v0.2.35 + id: cuda-toolkit + with: + cuda: ${{ matrix.cuda }} + use-github-cache: false + + - name: Install uv and Python ${{ matrix.pyver }} + uses: astral-sh/setup-uv@v7 + with: + python-version: ${{ matrix.pyver }} + activate-environment: true + enable-cache: true + + - name: Install dependencies + run: | + git config --system core.longpaths true + uv pip install --upgrade build setuptools wheel packaging + + - name: Setup MSVC environment for nvcc + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + echo PATH=%PATH%>>%GITHUB_ENV% + echo INCLUDE=%INCLUDE%>>%GITHUB_ENV% + echo LIB=%LIB%>>%GITHUB_ENV% + echo LIBPATH=%LIBPATH%>>%GITHUB_ENV% + + - name: Build wheel + run: | + $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.', '') + + $env:CUDA_HOME = $env:CUDA_PATH + $env:CUDA_TOOLKIT_ROOT_DIR = $env:CUDA_PATH + $env:VERBOSE = '1' + + # Force CMake to use Ninja + LLVM/Clang instead of the default + # Visual Studio generator. MSVC skips several GGML CPU all-variant + # backends, such as ivybridge, piledriver, cooperlake, zen4, and + # sapphirerapids. + $env:CMAKE_GENERATOR = 'Ninja Multi-Config' + + $toolchainCandidates = @( + (Join-Path $env:GITHUB_WORKSPACE "vendor\llama.cpp\cmake\x64-windows-llvm.cmake"), + (Join-Path $env:GITHUB_WORKSPACE "cmake\x64-windows-llvm.cmake") + ) + + $toolchainFile = $toolchainCandidates | + Where-Object { Test-Path $_ } | + Select-Object -First 1 + + if (!$toolchainFile) { + Write-Error "Toolchain file not found. Checked: $($toolchainCandidates -join ', ')" + exit 1 + } + + $toolchainFile = $toolchainFile.Replace('\', '/') + Write-Output "Using toolchain file: $toolchainFile" + + # Build one CUDA wheel with dynamic GGML backends: + # - GGML_BACKEND_DL enables runtime-loadable backend DLLs. + # - GGML_CPU_ALL_VARIANTS builds CPU variant DLLs such as ggml-cpu-x64, + # ggml-cpu-haswell, ggml-cpu-alderlake, etc. + # - GGML_NATIVE=OFF avoids binding the wheel to the runner CPU. + + # Suppress CUDA compiler warnings + $cudaDiagSuppress = '--diag-suppress=177,221,550' + + $cmakeArgs = @( + # Windows toolchain / common runtime + '-DCMAKE_TOOLCHAIN_FILE=vendor/llama.cpp/cmake/x64-windows-llvm.cmake' + '-DLLAMA_BUILD_BORINGSSL=ON' + + # Disable non-wheel targets + '-DLLAMA_BUILD_EXAMPLES=OFF' + '-DLLAMA_BUILD_TESTS=OFF' + '-DLLAMA_BUILD_TOOLS=OFF' + '-DLLAMA_BUILD_SERVER=OFF' + '-DLLAMA_BUILD_UI=OFF' + '-DLLAMA_USE_PREBUILT_UI=OFF' + '-DLLAMA_CURL=OFF' + + # GGML dynamic backend layout + '-DGGML_CPU=ON' + '-DGGML_CUDA=ON' + '-DGGML_NATIVE=OFF' + '-DGGML_BACKEND_DL=ON' + '-DGGML_CPU_ALL_VARIANTS=ON' + '-DGGML_OPENMP=ON' + + # CUDA backend + "-DCMAKE_CUDA_ARCHITECTURES=$env:CUDAARCHVER" + '-DGGML_CUDA_FORCE_MMQ=ON' + '-DCUDA_SEPARABLE_COMPILATION=ON' + "-DCMAKE_CUDA_FLAGS=$cudaDiagSuppress" + + # Build behavior + "-DCMAKE_BUILD_PARALLEL_LEVEL=$env:MAX_JOBS" + '-DENABLE_CCACHE=ON' + ) + + $env:CMAKE_ARGS = $cmakeArgs -join ' ' + Write-Output "CMAKE_ARGS=$env:CMAKE_ARGS" + + python -m build --wheel + + # Check if wheel was built + if (!(Test-Path '.\dist\*.whl')) { + Write-Error "No wheel built in dist/ directory" + exit 1 + } + + $wheelFile = Get-Item '.\dist\*.whl' | Select-Object -First 1 + + # Wheel filename format: + # name-version-python_tag-abi_tag-platform_tag.whl + $parts = $wheelFile.Name.Split('-') + $distName = $parts[0] + $version = $parts[1] + $pyTag = $parts[2] + $abiTag = $parts[3] + $platTag = $parts[4] + + # CPU all-variants is now an internal runtime layout detail. + $newVersion = "$version+cu$cudaVersion" + $newName = "$distName-$newVersion-$pyTag-$abiTag-$platTag" + + # Rename wheel file + Rename-Item -Path $wheelFile.FullName -NewName $newName + Write-Output "Renamed wheel to: $newName" + + # Write the build tag to the output + Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV + Write-Output "TAG_VERSION=$version" >> $env:GITHUB_ENV + + - name: Get current date + id: get-date + run: | + $currentDate = Get-Date -UFormat "%Y%m%d" + Write-Output "BUILD_DATE=$currentDate" >> $env:GITHUB_ENV + + - name: Create release + if: always() && env.TAG_VERSION != '' + uses: softprops/action-gh-release@v3 + with: + files: dist/* + # Set tag_name to v-cu-win- + tag_name: v${{ env.TAG_VERSION }}-cu${{ env.CUDA_VERSION }}-win-${{ env.BUILD_DATE }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 7a6ee9fcd57438a950eb2ee6c8e079f2409c2765 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 9 Jun 2026 00:33:10 +0800 Subject: [PATCH 181/304] =?UTF-8?q?build(CMakeLists):=20prefer=20VS=202022?= =?UTF-8?q?=20VC143=20OpenMP=20redist=20and=20keep=20System32=20as=20final?= =?UTF-8?q?=20fallback=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: JamePeng --- CMakeLists.txt | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ace43c4aa..5b2cfeeb8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,7 +135,7 @@ function(llama_cpp_python_install_windows_openmp_runtime) message(WARNING "LLAMA_CPP_OPENMP_RUNTIME_DLL was set, but the file does not exist: " "${LLAMA_CPP_OPENMP_RUNTIME_DLL}. Falling back to Visual Studio " - "LLVM OpenMP runtime discovery." + "VC143 LLVM OpenMP runtime discovery." ) endif() endif() @@ -144,18 +144,19 @@ function(llama_cpp_python_install_windows_openmp_runtime) file(TO_CMAKE_PATH "$ENV{ProgramFiles}" PROGRAMFILES_CMAKE) file(TO_CMAKE_PATH "$ENV{ProgramFiles\(x86\)}" PROGRAMFILES_X86_CMAKE) - set(VS_OPENMP_SEARCH_PATTERNS - # Prefer the exact VS 2022 Enterprise / BuildTools LLVM OpenMP redist layout. + set(VS_OPENMP_VC143_PATTERNS + # Prefer VS 2022 VC143 LLVM OpenMP redist paths. + # The MSVC version directory is intentionally globbed because + # GitHub runners may contain versions such as 14.44.35112 or 14.44.35207. "${PROGRAMFILES_CMAKE}/Microsoft Visual Studio/2022/Enterprise/VC/Redist/MSVC/*/debug_nonredist/x64/Microsoft.VC143.OpenMP.LLVM/libomp140.x86_64.dll" "${PROGRAMFILES_X86_CMAKE}/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC/*/debug_nonredist/x64/Microsoft.VC143.OpenMP.LLVM/libomp140.x86_64.dll" - # Keep these as secondary fallbacks for non-standard installs. + # Secondary VS layout fallbacks for unusual installations. "${PROGRAMFILES_CMAKE}/Microsoft Visual Studio/2022/BuildTools/VC/Redist/MSVC/*/debug_nonredist/x64/Microsoft.VC143.OpenMP.LLVM/libomp140.x86_64.dll" "${PROGRAMFILES_X86_CMAKE}/Microsoft Visual Studio/2022/Enterprise/VC/Redist/MSVC/*/debug_nonredist/x64/Microsoft.VC143.OpenMP.LLVM/libomp140.x86_64.dll" - "C:/Windows/System32/libomp140.x86_64.dll" ) - foreach(PATTERN ${VS_OPENMP_SEARCH_PATTERNS}) + foreach(PATTERN ${VS_OPENMP_VC143_PATTERNS}) file(GLOB PATTERN_OPENMP_DLLS "${PATTERN}") list(APPEND FOUND_OPENMP_DLLS ${PATTERN_OPENMP_DLLS}) endforeach() @@ -164,7 +165,16 @@ function(llama_cpp_python_install_windows_openmp_runtime) list(REMOVE_DUPLICATES FOUND_OPENMP_DLLS) list(SORT FOUND_OPENMP_DLLS COMPARE NATURAL ORDER DESCENDING) list(GET FOUND_OPENMP_DLLS 0 OPENMP_RUNTIME_DLL) - set(OPENMP_RUNTIME_SOURCE "Visual Studio 2022 LLVM OpenMP redist fallback") + set(OPENMP_RUNTIME_SOURCE "Visual Studio 2022 VC143 LLVM OpenMP redist") + endif() + endif() + + if(NOT OPENMP_RUNTIME_DLL) + set(SYSTEM32_OPENMP_RUNTIME_DLL "C:/Windows/System32/libomp140.x86_64.dll") + + if(EXISTS "${SYSTEM32_OPENMP_RUNTIME_DLL}") + set(OPENMP_RUNTIME_DLL "${SYSTEM32_OPENMP_RUNTIME_DLL}") + set(OPENMP_RUNTIME_SOURCE "System32 fallback") endif() endif() @@ -177,9 +187,10 @@ function(llama_cpp_python_install_windows_openmp_runtime) else() message(WARNING "Could not find libomp140.x86_64.dll for Windows LLVM OpenMP. " - "Searched LLAMA_CPP_OPENMP_RUNTIME_DLL and Visual Studio 2022 " - "Enterprise/BuildTools redist paths under Program Files and Program Files (x86), " - "with a fuzzy MSVC version match such as 14.44.35112 or 14.44.35207. " + "Searched LLAMA_CPP_OPENMP_RUNTIME_DLL, Visual Studio 2022 " + "Enterprise/BuildTools VC143 redist paths under Program Files and " + "Program Files (x86), with a fuzzy MSVC version match such as " + "14.44.35112 or 14.44.35207, and C:/Windows/System32 as a final fallback. " "If GGML_OPENMP=ON and GGML CPU backend DLLs are built with LLVM OpenMP, " "the packaged ggml-cpu-*.dll files may fail to load at runtime. " "Set LLAMA_CPP_OPENMP_RUNTIME_DLL to the full path of libomp140.x86_64.dll " From 50bbdd61fdf7e2e1cd7582a2183e476c98a47c17 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 9 Jun 2026 02:54:00 +0800 Subject: [PATCH 182/304] Update Submodule vendor/llama.cpp f0156d1..7d2b45b Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index f0156d1401..7d2b45b4f7 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit f0156d1401500512ad85042ccf38970568b12253 +Subproject commit 7d2b45b4f7b663cda74f23fbc3ce6dc3bd4f6545 From 55e855b75f901b494259a1c81b45ac80f0e3013f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 9 Jun 2026 05:03:15 +0800 Subject: [PATCH 183/304] Update mtmd API 20260609 Signed-off-by: JamePeng --- llama_cpp/mtmd_cpp.py | 293 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 283 insertions(+), 10 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 4542555c65..61fb0e7859 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -10,12 +10,14 @@ c_uint8, c_int32, c_uint32, + c_int64, c_float, c_void_p, c_size_t, POINTER, _Pointer, # type: ignore Structure, + CFUNCTYPE ) import pathlib from typing import ( @@ -318,6 +320,16 @@ def mtmd_get_audio_sample_rate(ctx: mtmd_context_p) -> c_int: """ ... +# // get the current marker string +# MTMD_API const char * mtmd_get_marker(const mtmd_context * ctx); +@ctypes_function_mtmd( + "mtmd_get_marker", [mtmd_context_p_ctypes], c_char_p) +def mtmd_get_marker(ctx: mtmd_context_p) -> c_char_p: + """ + get the current marker string + """ + ... + # // mtmd_bitmap # // # // if bitmap is image: @@ -420,6 +432,58 @@ def mtmd_bitmap_set_id( ... +# // mtmd_bitmap lazy +# // +# // this is a special bitmap that: +# // - does not hold the actual data +# // - can be expanded into one or more chunks (either media to text chunks) +# // user must provide a callback to fill in the data when mtmd_tokenize() is called +# // this is useful for large video inputs: +# // - allow reading video frame by frame, without loading the entire video into memory +# // - allow tracking the whole video with a single ID (for example, the file hash) + +# // set (*out_bitmap) to non-nullptr to emit a bitmap chunk; it will be freed automatically +# // set (*out_text) to non-nullptr to emit a text chunk; it must be heap-allocated, null-terminated and will be freed automatically +# // either out_bitmap or out_text can be set, but not both +# // out_bitmap cannot be another lazy bitmap (no nested lazy allowed) +# // return value: +# // 0 on success +# // -1 on EOF (signal to mtmd_tokenize to move on) +# // -2 on error (signal to mtmd_tokenize to abort) +# typedef int(* mtmd_bitmap_lazy_callback)( +# size_t chunk_idx, +# void * user_data, +# mtmd_bitmap ** out_bitmap, +# char ** out_text); +mtmd_bitmap_lazy_callback = CFUNCTYPE( + c_int, + c_size_t, # chunk_idx + c_void_p, # user_data + POINTER(mtmd_bitmap_p), # mtmd_bitmap ** out_bitmap + POINTER(c_char_p), # char ** out_text +) + +# MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx, +# const char * id, // usually set to file hash +# void * user_data, +# mtmd_bitmap_lazy_callback callback); +@ctypes_function_mtmd( + "mtmd_input_chunks_get", [ + mtmd_context_p_ctypes, + c_char_p, + c_void_p, + mtmd_bitmap_lazy_callback, + ], mtmd_bitmap_p_ctypes) +def mtmd_input_chunks_get( + ctx: mtmd_context_p, + id: c_char_p, + user_data: c_void_p, + callback: mtmd_bitmap_lazy_callback, # type: ignore + /, +) -> mtmd_bitmap_p: + ... + + # // mtmd_input_chunks # // # // this is simply a list of mtmd_input_chunk @@ -772,6 +836,9 @@ def mtmd_test_create_input_chunks() -> mtmd_input_chunk_p: # // BREAKING CHANGES are expected. # // +# struct mtmd_helper_video; +mtmd_helper_video_p = NewType("mtmd_helper_video_p", int) +mtmd_helper_video_p_ctypes = c_void_p # // Set callback for all future logging events. # // If this is not called, or NULL is supplied, everything is output on stderr. @@ -786,11 +853,33 @@ def mtmd_helper_log_set(log_callback: ggml_log_callback, user_data: c_void_p): # ... +# // Returns true if this build includes video support (MTMD_VIDEO was ON at compile time). +# MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx); +@ctypes_function_mtmd( + "mtmd_helper_support_video", [mtmd_context_p], c_bool) +def mtmd_helper_support_video(ctx: mtmd_context_p) -> c_bool: + """ + Returns true if this build includes video support (MTMD_VIDEO was ON at compile time). + """ + ... + + +# struct mtmd_helper_bitmap_wrapper { +# mtmd_bitmap * bitmap; +# mtmd_helper_video * video_ctx; +# }; +class mtmd_helper_bitmap_wrapper(Structure): + _fields_ = [ + ("bitmap", mtmd_bitmap_p), + ("video_ctx", mtmd_helper_video_p), + ] +mtmd_helper_bitmap_wrapper_p_ctypes = POINTER(mtmd_helper_bitmap_wrapper) + # // helper function to construct a mtmd_bitmap from a file # // it calls mtmd_helper_bitmap_init_from_buf() internally # // returns nullptr on failure # // this function is thread-safe -# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); +# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname, bool placeholder); @ctypes_function_mtmd( "mtmd_helper_bitmap_init_from_file", [ @@ -798,14 +887,14 @@ def mtmd_helper_log_set(log_callback: ggml_log_callback, user_data: c_void_p): # c_char_p, c_bool, ], - mtmd_bitmap_p_ctypes + mtmd_helper_bitmap_wrapper ) def mtmd_helper_bitmap_init_from_file( ctx: mtmd_context_p, fname: c_char_p, placeholder: c_bool, /, -) -> mtmd_bitmap_p: +) -> mtmd_helper_bitmap_wrapper: """ helper function to construct a mtmd_bitmap from a file it calls mtmd_helper_bitmap_init_from_buf() internally @@ -818,10 +907,13 @@ def mtmd_helper_bitmap_init_from_file( # // supported formats: # // image: formats supported by stb_image: jpg, png, bmp, gif, etc. # // audio: formats supported by miniaudio: wav, mp3, flac -# // note: audio files will be auto-detected based on magic bytes +# // note: +# // - for now, video input is only supported via C++ helper functions +# // - audio files will be auto-detected based on magic bytes +# // - output bitmap will have FNV hash as the ID # // returns nullptr on failure # // this function is thread-safe -# MTMD_API mtmd_bitmap * mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); +# MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); @ctypes_function_mtmd( "mtmd_helper_bitmap_init_from_buf", [ mtmd_context_p_ctypes, @@ -829,7 +921,7 @@ def mtmd_helper_bitmap_init_from_file( c_size_t, c_bool, ], - mtmd_bitmap_p_ctypes + mtmd_helper_bitmap_wrapper ) def mtmd_helper_bitmap_init_from_buf( ctx: mtmd_context_p, @@ -837,13 +929,16 @@ def mtmd_helper_bitmap_init_from_buf( len: c_size_t, placeholder: c_bool, /, -) -> mtmd_bitmap_p: +) -> mtmd_helper_bitmap_wrapper: """ helper function to construct a mtmd_bitmap from a buffer containing a file supported formats: - image: formats supported by stb_image: jpg, png, bmp, gif, etc. - audio: formats supported by miniaudio: wav, mp3, flac - note: audio files will be auto-detected based on magic bytes + image: formats supported by stb_image: jpg, png, bmp, gif, etc. + audio: formats supported by miniaudio: wav, mp3, flac + note: + - for now, video input is only supported via C++ helper functions + - audio files will be auto-detected based on magic bytes + - output bitmap will have FNV hash as the ID returns nullptr on failure """ ... @@ -1020,3 +1115,181 @@ def mtmd_helper_decode_image_chunk( ret 0 on success, -1 on chunk not being a valid image chunk, 1 on decode failure """ ... + +# // +# // video input helpers (requires ffmpeg/ffprobe installed on the system) +# // the notion of video only exists at the helper level, it is not visible to the core mtmd library +# // +# // NOTE: this implementation is model-agnostic, it can be used with any vision-capable model +# // however, it may not be accurate for some specific models +# // (this is expected for now, to keep the implementation simple) +# // + +# struct mtmd_helper_video_info { +# uint32_t width; +# uint32_t height; +# float fps; // effective fps (fps_target if set, else original video fps) +# int32_t n_frames; // estimated total frames at effective fps (-1 if unknown) +# }; +class mtmd_helper_video_info(Structure): + _fields_ = [ + ("width", c_uint32), + ("height", c_uint32), + ("fps", c_float), + ("n_frames", c_int32), + ] +mtmd_helper_video_info_p_ctypes = POINTER(mtmd_helper_video_info) + + +# struct mtmd_helper_video_init_params { +# float fps_target; // desired output fps; <= 0 means use the video's native fps, defaulted to 4.0f +# const char * ffmpeg_bin_dir; // directory containing ffmpeg/ffprobe binaries; NULL means search PATH +# int64_t timestamp_interval_ms; // interval for adding timestamp as text chunk (example: "[10m50.5s]"); <= 0 means no timestamp, defaulted to 5000ms +# // TODO @ngxson : allow "placeholder" bitmap output for counting tokens +# }; +class mtmd_helper_video_init_params(Structure): + _fields_ = [ + ("fps_target", c_float), + ("ffmpeg_bin_dir", c_char_p), + ("timestamp_interval_ms", c_int64), + ] +mtmd_helper_video_init_params_p_ctypes = POINTER(mtmd_helper_video_init_params) + + +# MTMD_API struct mtmd_helper_video_init_params mtmd_helper_video_init_params_default(void); +@ctypes_function_mtmd( + "mtmd_helper_video_init_params_default", + [], + mtmd_helper_video_init_params, +) +def mtmd_helper_video_init_params_default( + /, +) -> mtmd_helper_video_init_params: + """ + get default init params for mtmd_helper_video + """ + ... + + +# // returns NULL on failure (ffprobe not found, file unreadable, etc.) +# MTMD_API mtmd_helper_video * mtmd_helper_video_init( +# struct mtmd_context * mctx, +# const char * path, +# struct mtmd_helper_video_init_params params); +@ctypes_function_mtmd( + "mtmd_helper_video_init", [ + mtmd_context_p_ctypes, + c_char_p, + mtmd_helper_video_init_params, + ], + mtmd_helper_video_p) +def mtmd_helper_video_init( + mctx: mtmd_context_p, + path: c_char_p, + params: mtmd_helper_video_init_params, + /, +) -> mtmd_helper_video_p: + """ + helper function to init an mtmd_helper_video object + returns NULL on failure (ffprobe not found, file unreadable, etc.) + """ + ... + + +# // Same as mtmd_helper_video_init(), but reads from an in-memory buffer. +# // The buffer is copied internally; the caller does not need to keep it alive. +# // Note: pipe input is not seekable, so seeking will use output-side seeking +# // (ffmpeg decodes and discards frames up to the target position). +# MTMD_API mtmd_helper_video * mtmd_helper_video_init_from_buf( +# struct mtmd_context * mctx, +# const unsigned char * buf, size_t len, +# struct mtmd_helper_video_init_params params); +@ctypes_function_mtmd( + "mtmd_helper_video_init_from_buf", + [ + mtmd_context_p_ctypes, + c_char_p, + c_size_t, + mtmd_helper_video_init_params, + ], + mtmd_helper_video_p_ctypes, +) +def mtmd_helper_video_init_from_buf( + mctx: mtmd_context_p, + buf: c_char_p, + len: int, + params: mtmd_helper_video_init_params, + /, +) -> mtmd_helper_video_p: + """ + helper function to init an mtmd_helper_video object from an in-memory video buffer + + The buffer is copied internally, so the caller does not need to keep it alive + after this function returns. + """ + ... + + +# MTMD_API void mtmd_helper_video_free(mtmd_helper_video * ctx); +@ctypes_function_mtmd("mtmd_helper_video_free", [mtmd_helper_video_p_ctypes], None) +def mtmd_helper_video_free( + ctx: mtmd_helper_video_p, + /, +) -> None: + """ + free an mtmd_helper_video object + """ + ... + + +# MTMD_API struct mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx); +@ctypes_function_mtmd("mtmd_helper_video_get_info", [mtmd_helper_video_p_ctypes], mtmd_helper_video_info) +def mtmd_helper_video_get_info( + ctx: mtmd_helper_video_p, + /, +) -> mtmd_helper_video_info: + """ + get video information from an mtmd_helper_video object + """ + ... + + +# // Read the next item from the video stream; exactly one of out_bitmap or out_text is set per call. +# // *out_bitmap - heap-allocated; caller must free with mtmd_bitmap_free() +# // *out_text - heap-allocated (always via strdup/malloc); caller must free with free() +# // returns 0 on success, -1 on EOF, -2 on error +# MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, +# mtmd_bitmap ** out_bitmap, +# char ** out_text); +@ctypes_function_mtmd( + "mtmd_helper_video_read_next", + [ + mtmd_helper_video_p_ctypes, + POINTER(mtmd_bitmap_p_ctypes), + POINTER(c_char_p), + ], + c_int32, +) +def mtmd_helper_video_read_next( + ctx: mtmd_helper_video_p, + out_bitmap: POINTER(mtmd_bitmap_p_ctypes), # type: ignore + out_text: POINTER(c_char_p), # type: ignore + /, +) -> int: + """ + read the next item from the video stream + + Exactly one of out_bitmap or out_text is set per successful call. + + out_bitmap: + heap-allocated bitmap; caller must free it with mtmd_bitmap_free() + + out_text: + heap-allocated string via strdup/malloc; caller must free it with free() + + returns: + 0 on success + -1 on EOF + -2 on error + """ + ... From 10b4addb9d5f2ff71bddde34f43f8a43fac44b61 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 9 Jun 2026 05:08:49 +0800 Subject: [PATCH 184/304] feat(mtmd): add video input support to MTMDChatHandler - Add video_url handling to the MTMD chat template and media extraction pipeline. Detect whether the loaded libmtmd build supports video helpers and reject video inputs early when MTMD_VIDEO is unavailable. - Update media loading and bitmap creation for the new helper wrapper API. mtmd_helper_bitmap_init_from_buf now returns a bitmap wrapper containing both the decoded bitmap and an optional video helper context, so keep the video context alive until mtmd_tokenize completes and release it afterward. - Also consolidate duplicated audio/video byte loading into a shared _load_bytes helper, reuse it for image loading, and add richer default HTTP headers for remote media requests. Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 173 ++++++++++++++++++++++----------- 1 file changed, 115 insertions(+), 58 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index fb42a59f23..2224466436 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3064,6 +3064,8 @@ class MTMDChatHandler: "{% else %}" "data:audio/{{ content.input_audio.format }};base64,{{ content.input_audio.data }}" "{% endif %}" + "{% elif content.type == 'video_url' %}" + "{{ content.video_url if content.video_url is string else content.video_url.url }}" "{% elif content.type == 'text' %}" "{{ content.text }}" "{% endif %}" @@ -3114,6 +3116,10 @@ def __init__( self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None self.extra_template_arguments: dict[str, Any] = {} + self.is_support_vision = False + self.is_support_audio = False + self.is_support_video = False + if not os.path.exists(clip_model_path): raise ValueError(f"{self.log_prefix}(__init__): Clip model path does not exist: {clip_model_path}") @@ -3182,6 +3188,15 @@ def _init_mtmd_context(self, llama_model: llama_core.Llama): if self.verbose: print(f"{self.log_prefix}(_init_mtmd_context): Audio is NOT supported by this mmproj model backend.", file=sys.stderr) + # Check if video is supported + self.is_support_video = self._mtmd_cpp.mtmd_helper_support_video(self.mtmd_ctx) + if self.is_support_video: + if self.verbose: + print(f"{self.log_prefix}(_init_mtmd_context): Video support detected.", file=sys.stderr) + else: + if self.verbose: + print(f"{self.log_prefix}(_init_mtmd_context): Video support is NOT available in this build.", file=sys.stderr) + def close(self) -> None: """Explicitly free the mtmd context and vision model resources.""" if getattr(self, "mtmd_ctx", None) is not None: @@ -3259,7 +3274,16 @@ def _get_media_items(self, messages: List[llama_types.ChatCompletionRequestMessa if url: media_items.append({"url": url, "type": "audio"}) - # 3. Text & Unknown Types + # 3. Video Processing + elif content_type == "video_url": + if not self.is_support_video: + raise ValueError(f"{self.log_prefix}: This libmtmd build does not support video inputs.") + + video_url = content["video_url"] + url = video_url if isinstance(video_url, str) else video_url["url"] + media_items.append({"url": url, "type": "video"}) + + # 4. Text & Unknown Types elif content_type == "text": continue else: @@ -3274,6 +3298,7 @@ def _create_bitmap_from_bytes(self, media_bytes: bytes): Supported formats: - Images (via stb_image): jpg, png, bmp, etc. - Audio (via miniaudio): wav, mp3, flac. + - Video: depends on whether MTMD_VIDEO was enabled at build time. Note: - Media types (Image vs. Audio) are auto-detected by the C++ backend using magic bytes. @@ -3283,25 +3308,35 @@ def _create_bitmap_from_bytes(self, media_bytes: bytes): media_bytes (bytes): The raw byte content of the media file. Returns: - mtmd_bitmap: A pointer to the allocated bitmap structure containing decoded media features. + bitmap: mtmd_bitmap * + video_ctx: mtmd_helper_video * or NULL """ if self.mtmd_ctx is None: raise ValueError(f"{self.log_prefix}(_create_bitmap_from_bytes): mtmd context not initialized.") - # Create bitmap from buffer using helper function - bitmap = self._mtmd_cpp.mtmd_helper_bitmap_init_from_buf( + if not media_bytes: + raise ValueError(f"{self.log_prefix}(_create_bitmap_from_bytes): empty media bytes.") + + buf = (ctypes.c_uint8 * len(media_bytes)).from_buffer_copy(media_bytes) + + wrapper = self._mtmd_cpp.mtmd_helper_bitmap_init_from_buf( self.mtmd_ctx, - (ctypes.c_uint8 * len(media_bytes)).from_buffer(bytearray(media_bytes)), + buf, len(media_bytes), False, ) - if bitmap is None: - raise ValueError(f"{self.log_prefix}(_create_bitmap_from_bytes): " - "Failed to load image or audio file from media bytes " - "(unsupported media format or corrupted data).") + if not wrapper.bitmap: + if wrapper.video_ctx: + self._mtmd_cpp.mtmd_helper_video_free(wrapper.video_ctx) - return bitmap + raise ValueError( + f"{self.log_prefix}(_create_bitmap_from_bytes): " + "Failed to load media from bytes " + "(unsupported media format, corrupted data, or missing helper support)." + ) + + return wrapper.bitmap, wrapper.video_ctx def _process_mtmd_prompt( @@ -3360,16 +3395,17 @@ def _process_mtmd_prompt( # 3. Pre-allocate bitmap array to guarantee chronological order during concurrent decoding bitmaps = [None] * len(media_items) bitmap_cleanup = [] + video_cleanup = [] chunks = None try: # Concurrent Media Decoding import concurrent.futures if media_items: - def _create_bitmap_func(idx: int, item: str): + def _create_bitmap_func(idx: int, item: dict): media_bytes = self.load_media(item["url"], item["type"]) - bitmap = self._create_bitmap_from_bytes(media_bytes) - return idx, bitmap + bitmap, video_ctx = self._create_bitmap_from_bytes(media_bytes) + return idx, bitmap, video_ctx # This method uses multi-threaded parallel processing to convert images or audio to bitmaps, # which can be used in the future to process large numbers of video frames. max_workers = min(llama.n_threads, len(media_items)) @@ -3377,10 +3413,14 @@ def _create_bitmap_func(idx: int, item: str): futures = [executor.submit(_create_bitmap_func, i, item) for i, item in enumerate(media_items)] for future in concurrent.futures.as_completed(futures): - idx, bitmap = future.result() + idx, bitmap, video_ctx = future.result() + bitmaps[idx] = bitmap bitmap_cleanup.append(bitmap) + if video_ctx: + video_cleanup.append(video_ctx) + # Strict validation: Abort if any thread failed to decode its assigned media if any(b is None for b in bitmaps): raise RuntimeError(f"{self.log_prefix}(_create_bitmap_func): Failed to decode one or more media files.") @@ -3415,6 +3455,12 @@ def _create_bitmap_func(idx: int, item: str): if result != 0: raise ValueError(f"{self.log_prefix}(mtmd_tokenize): Unable to tokenize prompt, res = {result}.") + # Video helper contexts only need to stay alive until mtmd_tokenize() completes. + if video_cleanup: + for video_ctx in video_cleanup: + self._mtmd_cpp.mtmd_helper_video_free(video_ctx) + video_cleanup.clear() + # 6. Virtual Token Ledger Construction full_prompt_ids = [] chunk_token_spans = [] @@ -3424,6 +3470,7 @@ def _create_bitmap_func(idx: int, item: str): # Cursor to track the actual media contents (URLs or base64 data) provided by the user media_items_count = len(media_items) media_items_cur = 0 + last_media_id = None for i in range(n_chunks): chunk = self._mtmd_cpp.mtmd_input_chunks_get(chunks, i) @@ -3463,7 +3510,11 @@ def _create_bitmap_func(idx: int, item: str): # while instantly breaking the match if the image content changes. # media_id = - (zlib.crc32(real_media_url.encode('utf-8')) % (2**24)) - 100 media_id = - (zlib.crc32(real_media_url.encode('utf-8')) & 0xFFFFFF) - 100 + last_media_id = media_id media_items_cur += 1 + elif last_media_id is not None: + # video may expand into multiple image chunks from one media marker + media_id = last_media_id else: # Magic Negative Number as fallback :) media_id = -314159 @@ -3492,6 +3543,12 @@ def _create_bitmap_func(idx: int, item: str): for bitmap in bitmap_cleanup: self._mtmd_cpp.mtmd_bitmap_free(bitmap) bitmap_cleanup = None + # Free videos + if len(video_cleanup) > 0: + for video_ctx in video_cleanup: + self._mtmd_cpp.mtmd_helper_video_free(video_ctx) + video_cleanup = None + bitmaps = None raise e @@ -3825,18 +3882,22 @@ def __call__( def load_media(self, media_url: str, media_type: str) -> bytes: """ Unified dispatcher for loading media payloads. - Routes the URL/URI to the specific image or audio processor based on the media_type. + Routes the URL/URI to the specific image, audio, or video processor based on the media_type. """ if media_type == "image": return self._load_image(media_url) + elif media_type == "audio": - audio_bytes = self._load_audio(media_url) - # Apply ironclad magic bytes validation before returning + audio_bytes = self._load_bytes(media_url, timeout=15, kind="audio") try: self.detect_audio_format(audio_bytes) except ValueError as e: raise ValueError(f"{self.log_prefix}(load_media): {e}") return audio_bytes + + elif media_type == "video": + return self._load_bytes(media_url, timeout=30, kind="video") + else: raise ValueError(f"{self.log_prefix}(load_media): Unknown media type '{media_type}'") @@ -3876,41 +3937,51 @@ def detect_audio_format(audio_bytes: bytes) -> str: "The underlying C++ miniaudio backend ONLY supports WAV, MP3, and FLAC." ) + DEFAULT_HTTP_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/148.0.0.0 Safari/537.36" + ), + } + @staticmethod - def _load_audio(audio_url: str) -> bytes: + def _load_bytes(media_url: str, timeout: int = 15, kind: str = "media") -> bytes: """ - Load audio from either a URL, local path, or a data URI and return raw bytes. + Load raw bytes from a data URI, local file path, or remote HTTP/HTTPS URL. """ + media_bytes = b"" - audio_bytes = b"" - - # 1. Handle data URI (base64) - if audio_url.strip().startswith("data:"): - comma_pos = audio_url.find(",") + # 1. Handle data URI + if media_url.strip().startswith("data:"): + comma_pos = media_url.find(",") if comma_pos == -1: raise ValueError("Invalid data URI: missing comma separator") - base64_data = audio_url[comma_pos + 1 :] - audio_bytes = base64.b64decode(base64_data) + + base64_data = media_url[comma_pos + 1:] + media_bytes = base64.b64decode(base64_data) # 2. Handle local file path - elif os.path.exists(audio_url): - with open(audio_url, "rb") as f: - audio_bytes = f.read() + elif os.path.exists(media_url): + with open(media_url, "rb") as f: + media_bytes = f.read() # 3. Handle remote URL via HTTP/HTTPS else: - headers = {"User-Agent": "Mozilla/5.0"} - req = urllib.request.Request(audio_url, headers=headers) + req = urllib.request.Request( + media_url, + headers=MTMDChatHandler.DEFAULT_HTTP_HEADERS, + ) try: - with urllib.request.urlopen(req, timeout=15) as f: - audio_bytes = f.read() + with urllib.request.urlopen(req, timeout=timeout) as f: + media_bytes = f.read() except (URLError, HTTPError) as e: - raise ConnectionError(f"Failed to download audio from {audio_url}: {e}") + raise ConnectionError(f"Failed to download {kind} from {media_url}: {e}") - if not audio_bytes: - raise ValueError("Empty audio data received") + if not media_bytes: + raise ValueError(f"Empty {kind} data received") - return audio_bytes + return media_bytes @staticmethod def _load_image(image_url: str) -> bytes: @@ -3926,28 +3997,14 @@ def _load_image(image_url: str) -> bytes: Returns: JPEG-encoded bytes (quality=95) in RGB mode, suitable for most vision models. """ - image_bytes = b"" - - # 1. Handle data URI (base64) - if image_url.strip().startswith("data:"): - # Split only once from the right to correctly handle mime types containing commas - comma_pos = image_url.find(",") - if comma_pos == -1: - raise ValueError("Invalid data URI: missing comma separator") - base64_data = image_url[comma_pos + 1 :] - image_bytes = base64.b64decode(base64_data) - - # 2. Handle local/remote URL - else: - headers = {"User-Agent": "Mozilla/5.0"} - req = urllib.request.Request(image_url, headers=headers) - - try: - with urllib.request.urlopen(req, timeout=15) as f: - image_bytes = f.read() - except (URLError, HTTPError) as e: - raise ConnectionError(f"Failed to download image from {image_url}: {e}") + # 1. Load image bytes from image_url + image_bytes = MTMDChatHandler._load_bytes( + image_url, + timeout=15, + kind="image", + ) + # 2. Check if image_bytes is empty. if not image_bytes: raise ValueError("Empty image data received") From e4dcac1af57b58973ecf7e206a3c25b3c367d881 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 9 Jun 2026 22:22:15 +0800 Subject: [PATCH 185/304] Update Submodule vendor/llama.cpp 7d2b45b..d6d0ce8 Signed-off-by: JamePeng --- llama_cpp/mtmd_cpp.py | 14 ++++++-------- vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 61fb0e7859..30ca8fab90 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -459,8 +459,8 @@ def mtmd_bitmap_set_id( c_int, c_size_t, # chunk_idx c_void_p, # user_data - POINTER(mtmd_bitmap_p), # mtmd_bitmap ** out_bitmap - POINTER(c_char_p), # char ** out_text + POINTER(mtmd_bitmap_p_ctypes), # mtmd_bitmap ** out_bitmap + POINTER(c_char_p), # char ** out_text ) # MTMD_API mtmd_bitmap * mtmd_bitmap_init_lazy(mtmd_context * ctx, @@ -856,7 +856,7 @@ def mtmd_helper_log_set(log_callback: ggml_log_callback, user_data: c_void_p): # # // Returns true if this build includes video support (MTMD_VIDEO was ON at compile time). # MTMD_API bool mtmd_helper_support_video(mtmd_context * ctx); @ctypes_function_mtmd( - "mtmd_helper_support_video", [mtmd_context_p], c_bool) + "mtmd_helper_support_video", [mtmd_context_p_ctypes], c_bool) def mtmd_helper_support_video(ctx: mtmd_context_p) -> c_bool: """ Returns true if this build includes video support (MTMD_VIDEO was ON at compile time). @@ -870,8 +870,8 @@ def mtmd_helper_support_video(ctx: mtmd_context_p) -> c_bool: # }; class mtmd_helper_bitmap_wrapper(Structure): _fields_ = [ - ("bitmap", mtmd_bitmap_p), - ("video_ctx", mtmd_helper_video_p), + ("bitmap", mtmd_bitmap_p_ctypes), + ("video_ctx", mtmd_helper_video_p_ctypes), ] mtmd_helper_bitmap_wrapper_p_ctypes = POINTER(mtmd_helper_bitmap_wrapper) @@ -1162,9 +1162,7 @@ class mtmd_helper_video_init_params(Structure): [], mtmd_helper_video_init_params, ) -def mtmd_helper_video_init_params_default( - /, -) -> mtmd_helper_video_init_params: +def mtmd_helper_video_init_params_default() -> mtmd_helper_video_init_params: """ get default init params for mtmd_helper_video """ diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7d2b45b4f7..d6d0ce8215 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7d2b45b4f7b663cda74f23fbc3ce6dc3bd4f6545 +Subproject commit d6d0ce8215a1c324e8de04b52f9dd65c5edc129f From 54f56bd8f89769f2021f31eba0aa377dc290f203 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 13 Jun 2026 00:46:09 +0800 Subject: [PATCH 186/304] Update Submodule vendor/llama.cpp d6d0ce8..ebc1077 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index d6d0ce8215..ebc10770ac 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit d6d0ce8215a1c324e8de04b52f9dd65c5edc129f +Subproject commit ebc10770ac5a9331824c53ef0c6adad780904dc3 From 6d1bd3b8d751a3a2ac86d377ecd34a3b37278b15 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 14 Jun 2026 00:04:56 +0800 Subject: [PATCH 187/304] Update Submodule vendor/llama.cpp ebc1077..e8067a8 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ebc10770ac..e8067a8b36 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ebc10770ac5a9331824c53ef0c6adad780904dc3 +Subproject commit e8067a8b3624aa40cc88ecb2940060e5d65b7532 From 971ee384227f6268f244c93f620b12f0a6ff47c0 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 14 Jun 2026 01:03:09 +0800 Subject: [PATCH 188/304] Update(mtmd): Append mtmd batching API - Sync upstream: mtmd: add batching API (#24384) Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 3 + llama_cpp/mtmd_cpp.py | 142 ++++++++++++++++++++++++++++++--- 2 files changed, 134 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 2224466436..520d2429d4 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3094,6 +3094,7 @@ def __init__( use_gpu: bool = True, image_min_tokens: int = -1, image_max_tokens: int = -1, + batch_max_tokens: int = 1024, **kwargs ): @@ -3108,6 +3109,7 @@ def __init__( self.clip_model_path = clip_model_path self.image_min_tokens = image_min_tokens self.image_max_tokens = image_max_tokens + self.batch_max_tokens = batch_max_tokens self.use_gpu = use_gpu self.verbose = verbose @@ -3152,6 +3154,7 @@ def _init_mtmd_context(self, llama_model: llama_core.Llama): if (self.image_max_tokens < self.image_min_tokens) and self.image_max_tokens > 0: raise ValueError(f"{self.log_prefix}(_init_mtmd_context): Configuration Error! image_max_tokens ({self.image_max_tokens}) " f"cannot be less than image_min_tokens ({self.image_min_tokens}).") + self.mctx_params.batch_max_tokens = self.batch_max_tokens # Cache the model's eos token and bos token self.mtmd_eos_token=llama_model.detokenize([llama_model.token_eos()]).decode('utf-8', errors='ignore') diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 30ca8fab90..4513761a63 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -153,6 +153,21 @@ class mtmd_pos_type(enum.IntEnum): mtmd_input_chunks_p = NewType("mtmd_input_chunks_p", int) mtmd_input_chunks_p_ctypes = c_void_p +# struct mtmd_batch { +# mtmd_context * ctx; +# std::vector entries; +# std::vector output_embd; // aggregated output embedding for the whole batch +# mtmd_batch(mtmd_context * ctx): ctx(ctx) {} +# int32_t n_tokens() const { +# int32_t n = 0; +# for (const auto * chunk : entries) { +# n += mtmd_input_chunk_get_n_tokens(chunk); +# } +# return n; +# } +# }; +mtmd_batch_p = NewType("mtmd_batch_p", int) +mtmd_batch_p_ctypes = c_void_p # struct mtmd_input_text { # const char * text; @@ -210,6 +225,11 @@ class clip_context_params(Structure): # // callback function passed over to mtmd proper # ggml_backend_sched_eval_callback cb_eval; # void * cb_eval_user_data; +# +# // batching params +# int32_t batch_max_tokens; // maximum number of output tokens in a batch +# // (note: this is not a hard-limit, the first image will always be added even if it exceeds this limit) +# // (default: 1024) # }; class mtmd_context_params(Structure): _fields_ = [ @@ -224,6 +244,7 @@ class mtmd_context_params(Structure): ("image_max_tokens", c_int), ("cb_eval", ggml_backend_sched_eval_callback), ("cb_eval_user_data", c_void_p), + ("batch_max_tokens", c_int32), ] mtmd_context_params_p_ctypes = POINTER(mtmd_context_params) @@ -731,8 +752,8 @@ def mtmd_tokenize( # // returns 0 on success # // TODO: deprecate -# MTMD_API int32_t mtmd_encode(mtmd_context * ctx, -# const mtmd_image_tokens * image_tokens); +# DEPRECATED(MTMD_API int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens), +# "use mtmd_encode_chunk() instead"); @ctypes_function_mtmd( "mtmd_encode", [ mtmd_context_p_ctypes, @@ -745,10 +766,15 @@ def mtmd_encode( image_tokens: mtmd_image_tokens_p, /, ) -> c_int32: + """ + DEPRECATED: use mtmd_encode_chunk() instead + """ ... +# // text chunk will be ignored silently, only media chunk will be encoded # // returns 0 on success +# // returns 1 on generic error # MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx, # const mtmd_input_chunk * chunk); @ctypes_function_mtmd( @@ -763,6 +789,11 @@ def mtmd_encode_chunk( chunk: mtmd_input_chunk_p, /, ) -> c_int32: + """ + text chunk will be ignored silently, only media chunk will be encoded + returns 0 on success + returns 1 on generic error + """ ... # // get output embeddings from the last encode pass @@ -778,6 +809,95 @@ def mtmd_get_output_embd(ctx: mtmd_context_p) -> POINTER(c_float): # type: ignor ... +# // batch encoding API +# // chunks are not owned by the batch, they will not be freed by mtmd_batch_free() +# // batch is valid for a given context, cannot be shared across contexts +# MTMD_API mtmd_batch * mtmd_batch_init(mtmd_context * ctx); +@ctypes_function_mtmd( + "mtmd_batch_init", + [mtmd_context_p_ctypes], + mtmd_batch_p_ctypes, +) +def mtmd_batch_init(ctx: mtmd_context_p, /) -> mtmd_batch_p: + ... + + +# MTMD_API void mtmd_batch_free(mtmd_batch * batch); +@ctypes_function_mtmd( + "mtmd_batch_free", + [mtmd_batch_p_ctypes], + None, +) +def mtmd_batch_free(batch: mtmd_batch_p, /): + """ + chunks are not owned by the batch, they will not be freed by mtmd_batch_free() + batch is valid for a given context, cannot be shared across contexts + """ + ... + + +# // only media chunks are allowed, text chunks will be rejected +# // returns 0 on success +# // returns 1 on generic error +# // returns 2 if the batch is too large (chunk won't be added) +# // returns 3 if it cannot be batched with the existing chunks in the batch +# MTMD_API int32_t mtmd_batch_add_chunk(mtmd_batch * batch, const mtmd_input_chunk * chunk); +@ctypes_function_mtmd( + "mtmd_batch_add_chunk", + [ + mtmd_batch_p_ctypes, + mtmd_input_chunk_p_ctypes, + ], + c_int32, +) +def mtmd_batch_add_chunk( + batch: mtmd_batch_p, + chunk: mtmd_input_chunk_p, + /, +) -> c_int32: + """ + only media chunks are allowed, text chunks will be rejected + returns 0 on success + returns 1 on generic error + returns 2 if the batch is too large (chunk won't be added) + returns 3 if it cannot be batched with the existing chunks in the batch + """ + ... + + +# // returns 0 on success +# // returns 1 on generic error +# MTMD_API int32_t mtmd_batch_encode(mtmd_batch * batch); +@ctypes_function_mtmd( + "mtmd_batch_encode", + [mtmd_batch_p_ctypes], + c_int32, +) +def mtmd_batch_encode(batch: mtmd_batch_p, /) -> c_int32: + """ + returns 0 on success + returns 1 on generic error + """ + ... + + +# MTMD_API float * mtmd_batch_get_output_embd(mtmd_batch * batch, const mtmd_input_chunk * chunk); +@ctypes_function_mtmd( + "mtmd_batch_get_output_embd", + [ + mtmd_batch_p_ctypes, + mtmd_input_chunk_p_ctypes, + ], + POINTER(c_float), +) +def mtmd_batch_get_output_embd( + batch: mtmd_batch_p, + chunk: mtmd_input_chunk_p, + /, +) -> POINTER(c_float): # type: ignore + ... + + # // Set callback for all future logging events. # // If this is not called, or NULL is supplied, everything is output on stderr. # MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); @@ -947,8 +1067,8 @@ def mtmd_helper_bitmap_init_from_buf( # // helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache # MTMD_API size_t mtmd_helper_get_n_tokens(const mtmd_input_chunks * chunks); @ctypes_function_mtmd( - "mtmd_helper_get_n_tokens", [mtmd_input_chunk_p_ctypes], c_size_t) -def mtmd_helper_get_n_tokens(chunks: mtmd_input_chunk_p) -> c_size_t: + "mtmd_helper_get_n_tokens", [mtmd_input_chunks_p_ctypes], c_size_t) +def mtmd_helper_get_n_tokens(chunks: mtmd_input_chunks_p) -> c_size_t: """ helper to count the total number of tokens from a list of chunks, useful to keep track of KV cache """ @@ -959,8 +1079,8 @@ def mtmd_helper_get_n_tokens(chunks: mtmd_input_chunk_p) -> c_size_t: # // normally, n_pos is equal to n_tokens, but for M-RoPE it is different # MTMD_API llama_pos mtmd_helper_get_n_pos(const mtmd_input_chunks * chunks); @ctypes_function_mtmd( - "mtmd_helper_get_n_pos", [mtmd_input_chunk_p_ctypes], c_int32) -def mtmd_helper_get_n_pos(chunks: mtmd_input_chunk_p) -> c_int32: + "mtmd_helper_get_n_pos", [mtmd_input_chunks_p_ctypes], c_int32) +def mtmd_helper_get_n_pos(chunks: mtmd_input_chunks_p) -> c_int32: """ helper to count the total position of tokens from a list of chunks, useful to keep track of n_past normally, n_pos is equal to n_tokens, but for M-RoPE it is different @@ -991,8 +1111,8 @@ def mtmd_helper_image_get_decoder_pos( # // helper function that automatically: # // 1. run llama_decode() on text chunks -# // 2. run mtmd_encode() on image chunks, then mtmd_get_output_embd() and then llama_decode() -# // if any of the mtmd_encode() or llama_decode() calls return non-zero, stop and forward the error +# // 2. run mtmd_encode_chunk() on image chunks, then mtmd_get_output_embd() and then llama_decode() +# // if any of the mtmd_encode_chunk() or llama_decode() calls return non-zero, stop and forward the error # // otherwise, returns 0 on success # // this function is NOT thread-safe # MTMD_API int32_t mtmd_helper_eval_chunks(mtmd_context * ctx, @@ -1007,7 +1127,7 @@ def mtmd_helper_image_get_decoder_pos( "mtmd_helper_eval_chunks", [ mtmd_context_p_ctypes, llama_cpp.llama_context_p_ctypes, - mtmd_input_chunk_p_ctypes, + mtmd_input_chunks_p_ctypes, c_int32, c_int32, c_int32, @@ -1018,7 +1138,7 @@ def mtmd_helper_image_get_decoder_pos( def mtmd_helper_eval_chunks( ctx: mtmd_context_p, lctx: llama_cpp.llama_context_p, - chunks: mtmd_input_chunk_p, + chunks: mtmd_input_chunks_p, n_past: c_int32, seq_id: c_int32, n_batch: c_int32, @@ -1106,7 +1226,7 @@ def mtmd_helper_decode_image_chunk( n_past: c_int32, seq_id: c_int32, n_batch: c_int32, - new_n_past: c_int32, + new_n_past: POINTER(c_int32), # type: ignore /, ) -> c_int32: """ From cb299e67e51e5aff061ebcf9f1521695ad3f1a5d Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 14 Jun 2026 03:05:32 +0800 Subject: [PATCH 189/304] Update(MTMDChatHandler): add chunk type helpers - Add small helper methods `_is_text_chunk`/`_is_image_chunk`/`_is_audio_chunk` for checking MTMD text, image, and audio chunk type enum values. - This keeps MTMD prompt processing easier to read and avoids repeating direct enum comparisons when building token spans for text and media chunks. Signed-off-by: JamePeng --- llama_cpp/llama_chat_format.py | 36 +++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 520d2429d4..aadec4600e 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -3341,6 +3341,26 @@ def _create_bitmap_from_bytes(self, media_bytes: bytes): return wrapper.bitmap, wrapper.video_ctx + def _is_text_chunk(self, chunk_type: int) -> bool: + """Return True if `chunk_type` is the MTMD text chunk type enum value.""" + return ( + chunk_type + == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_TEXT + ) + + def _is_image_chunk(self, chunk_type: int) -> bool: + """Return True if `chunk_type` is the MTMD image chunk type enum value.""" + return ( + chunk_type + == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE + ) + + def _is_audio_chunk(self, chunk_type: int) -> bool: + """Return True if `chunk_type` is the MTMD audio chunk type enum value.""" + return ( + chunk_type + == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO + ) def _process_mtmd_prompt( self, @@ -3480,7 +3500,7 @@ def _create_bitmap_func(idx: int, item: dict): if chunk is None: continue chunk_type = self._mtmd_cpp.mtmd_input_chunk_get_type(chunk) - if chunk_type == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_TEXT: + if self._is_text_chunk(chunk_type): # Extract standard text token IDs n_tokens_out = ctypes.c_size_t() tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text(chunk, ctypes.byref(n_tokens_out)) @@ -3489,10 +3509,7 @@ def _create_bitmap_func(idx: int, item: dict): chunk_token_spans.append((current_idx, current_idx + len(tokens), chunk, chunk_type, None)) full_prompt_ids.extend(tokens) current_idx += len(tokens) - elif chunk_type in [ - self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE, - self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO - ]: + elif self._is_image_chunk(chunk_type) or self._is_audio_chunk(chunk_type): # Extract media properties # Note(JamePeng): # The M-RoPE model is based on `n_pos` instead of `n_tokens` (of course, there's no difference in non-M-RoPE models). @@ -3673,7 +3690,7 @@ def __call__( if end_idx <= n_past: continue - if chunk_type == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_TEXT: + if self._is_text_chunk(chunk_type): unprocessed_start = max(start_idx, n_past) - start_idx n_tokens_out = ctypes.c_size_t() tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text(chunk_ptr, ctypes.byref(n_tokens_out)) @@ -3689,14 +3706,11 @@ def __call__( llama.eval(tokens_to_eval) n_past = llama.n_tokens - elif chunk_type in [ - self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE, - self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO - ]: + elif self._is_image_chunk(chunk_type) or self._is_audio_chunk(chunk_type): chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk_ptr) if self.verbose: - media_str = "IMAGE" if chunk_type == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE else "AUDIO" + media_str = "IMAGE" if self._is_image_chunk(chunk_type) else "AUDIO" print(f"{self.log_prefix}(__call__): Evaluating {media_str} chunk ({chunk_n_tokens} tokens) at pos {llama.n_tokens}...", file=sys.stderr) # Stage 5: Multimodal Physical OOM Defense From d8ee3eed7163c6c1f3802a9b979f9009e5e96c53 Mon Sep 17 00:00:00 2001 From: Alcoft Date: Sun, 14 Jun 2026 08:00:09 +0200 Subject: [PATCH 190/304] Change 'clip_model_path' to 'mmproj_path'. Implemented 'chat_template_override'. Only the chat template is passed from llama to the chat handler; not the entire model's metadata. --- llama_cpp/llama.py | 10 ++++----- llama_cpp/llama_chat_format.py | 39 +++++++++++++++++++--------------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 544e755ea9..1f5ffa20b5 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -96,7 +96,7 @@ class Llama: def __init__( self, model_path: str, - clip_model_path: Optional[str] = None, + mmproj_path: Optional[str] = None, *, # Model Params n_gpu_layers: Union[int, Literal["auto", "all"]] = "auto", @@ -710,13 +710,13 @@ def __init__( if self.verbose: print(f"Model metadata: {self.metadata}", file=sys.stderr) - if clip_model_path is not None: + if mmproj_path is not None: if self.chat_handler is not None and self.verbose: - print("Warning: Both `chat_handler` and `clip_model_path` are not null. Chat handler will be overwritten.", flush = True) + print("Warning: Both `chat_handler` and `mmproj_path` are not null. Chat handler will be overwritten.", flush = True) self.chat_handler = llama_chat_format.GenericMTMDChatHandler( - gguf_metadata = self.metadata, - clip_model_path = clip_model_path, + chat_format = self.metadata.get("tokenizer.chat_template", None), + mmproj_path = mmproj_path, verbose = self.verbose, **chat_handler_kwargs ) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 254195f95a..966c2e28fa 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2856,11 +2856,12 @@ class MTMDChatHandler: def __init__( self, - clip_model_path: str, + mmproj_path: str, verbose: bool = True, use_gpu: bool = True, image_min_tokens: int = -1, image_max_tokens: int = -1, + chat_template_override: Optional[str] = None, **kwargs ): @@ -2872,7 +2873,7 @@ def __init__( f"If you are passing model-specific parameters, ensure they are supported by {self.log_prefix}." ) - self.clip_model_path = clip_model_path + self.mmproj_path = mmproj_path self.image_min_tokens = image_min_tokens self.image_max_tokens = image_max_tokens self.use_gpu = use_gpu @@ -2883,20 +2884,25 @@ def __init__( self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None self.extra_template_arguments: dict[str, Any] = {} - if not os.path.exists(clip_model_path): - raise ValueError(f"{self.log_prefix}(__init__): Clip model path does not exist: {clip_model_path}") + if not os.path.exists(mmproj_path): + raise ValueError(f"{self.log_prefix}(__init__): Clip model path does not exist: {mmproj_path}") # Pre-compile Jinja template - if not hasattr(self, "chat_format") or self.chat_format is None: + if (not hasattr(self, "chat_format") or self.chat_format is None) and chat_template_override is None: self.chat_format = self.CHAT_FORMAT + elif chat_template_override is not None: + self.chat_format = chat_template_override self._chat_format_parser_tags = [] - self.chat_template = ImmutableSandboxedEnvironment( - trim_blocks=True, - lstrip_blocks=True, - ).from_string(self.chat_format) + self.change_chat_template(self.chat_format) self._exit_stack = ExitStack() + + def change_chat_template(self, new_template: str): + self.chat_template = ImmutableSandboxedEnvironment( + trim_blocks=True, + lstrip_blocks=True + ).from_string(new_template) def _init_mtmd_context(self, llama_model: llama_core.Llama): """Initialize mtmd context with the llama model.""" @@ -2929,13 +2935,13 @@ def _init_mtmd_context(self, llama_model: llama_core.Llama): # Initialize mtmd context self.mtmd_ctx = self._mtmd_cpp.mtmd_init_from_file( - self.clip_model_path.encode(), + self.mmproj_path.encode(), llama_model.model, self.mctx_params ) if self.mtmd_ctx is None: - raise ValueError(f"{self.log_prefix}(_init_mtmd_context): Failed to load mtmd context from: {self.clip_model_path}") + raise ValueError(f"{self.log_prefix}(_init_mtmd_context): Failed to load mtmd context from: {self.mmproj_path}") # Check if vision is supported self.is_support_vision = self._mtmd_cpp.mtmd_support_vision(self.mtmd_ctx) @@ -3835,7 +3841,7 @@ def from_pretrained( model_path = os.path.join(local_dir, filename) return cls( - clip_model_path=model_path, + mmproj_path=model_path, **kwargs, ) @@ -3852,13 +3858,12 @@ class GenericMTMDChatHandler(MTMDChatHandler): def __init__( self, - gguf_metadata: Dict[str, Any], - clip_model_path: str, + chat_format: str, + mmproj_path: str, verbose: bool = True, **kwargs ) -> None: - self.model_metadata = gguf_metadata - self.chat_format = self.model_metadata.get("tokenizer.chat_template", None) + self.chat_format = chat_format if verbose: print(f"Got chat template from model:\n```jinja\n{self.chat_format}\n```", flush = True) @@ -3866,7 +3871,7 @@ def __init__( if self.chat_format is None: raise ValueError("Failed to get model chat template automatically.") - super().__init__(clip_model_path = clip_model_path, verbose = verbose, **kwargs) + super().__init__(mmproj_path = mmproj_path, verbose = verbose, **kwargs) def __call__(self, **kwargs): self._chat_format_parser_tags = [tag for tag in self.KNOWN_MEDIA_TAGS if tag in self.chat_format] From 1965d5f6c3c949cab7f7ef934266c8062ebc0f45 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 14 Jun 2026 20:19:43 +0800 Subject: [PATCH 191/304] refactor(mtmd): move multimodal handlers to separate module `llama_multimodal` - Move MTMDChatHandler, GenericMTMDChatHandler, and model-specific multimodal chat handlers out of llama_chat_format.py into llama_multimodal.py. - llama_chat_format.py has grown too large and difficult to maintain, especially as MTMD support expands beyond image-only use cases. Splitting multimodal handling into its own module makes the chat formatting layer smaller and keeps media loading, MTMD tokenization, multimodal KV-cache bookkeeping, and handler implementations in a dedicated place. - This also prepares the codebase for broader multimodal support and future video frame / image batch evaluation, where the media-processing path will need to evolve independently from text-only chat formatting. - Keep backward-compatible re-exports from llama_chat_format.py so existing imports continue to work. - Also keep `clip_model_path` as a deprecated initialization alias for `mmproj_path` in the base MTMD handler. Signed-off-by: JamePeng --- llama_cpp/llama.py | 8 +- llama_cpp/llama_chat_format.py | 3811 ++------------------------------ llama_cpp/llama_multimodal.py | 3473 +++++++++++++++++++++++++++++ 3 files changed, 3690 insertions(+), 3602 deletions(-) create mode 100644 llama_cpp/llama_multimodal.py diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index ec202568f1..dbc60eaf76 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -45,6 +45,7 @@ from .llama_tokenizer import BaseLlamaTokenizer, LlamaTokenizer import llama_cpp.llama_cpp as llama_cpp_lib import llama_cpp.llama_chat_format as llama_chat_format +import llama_cpp.llama_multimodal as llama_multimodal from llama_cpp.llama_speculative import LlamaDraftModel @@ -711,20 +712,19 @@ def __init__( self.metadata = {} if self.verbose: print(f"Failed to load metadata: {e}", file=sys.stderr) - - if self.verbose: - print(f"Model metadata: {self.metadata}", file=sys.stderr) if mmproj_path is not None: if self.chat_handler is not None and self.verbose: print("Warning: Both `chat_handler` and `mmproj_path` are not null. Chat handler will be overwritten.", flush = True) - self.chat_handler = llama_chat_format.GenericMTMDChatHandler( + self.chat_handler = llama_multimodal.GenericMTMDChatHandler( chat_format = self.metadata.get("tokenizer.chat_template", None), mmproj_path = mmproj_path, verbose = self.verbose, **chat_handler_kwargs ) + + if self.verbose: print(f"Model desc: {self.model_desc}, " f"Model size: {self.model_size / (1024 * 1024):.2f} MB, " f"Model metadata: {self.metadata}", diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 0e5c9d4906..6ffe68e5e3 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1,7 +1,5 @@ from __future__ import annotations -import base64 -import ctypes import dataclasses import datetime import json @@ -9,9 +7,7 @@ import random import string import sys -import zlib -from contextlib import ExitStack from typing import ( Any, Dict, @@ -32,16 +28,11 @@ import numpy as np import numpy.typing as npt -import urllib.request -from urllib.error import URLError, HTTPError - -import llama_cpp.llama_cpp as llama_cpp_lib import llama_cpp.llama as llama_core import llama_cpp.llama_types as llama_types import llama_cpp.llama_grammar as llama_grammar -from ._ggml import GGMLLogLevel -from ._logger import logger, ggml_log_callback +from ._logger import logger from ._utils import suppress_stdout_stderr, Singleton ### Common Chat Templates and Special Tokens ### @@ -3037,3612 +3028,204 @@ def generate_streaming(tools, functions, function_call, prompt): ) -class MTMDChatHandler: - DEFAULT_SYSTEM_MESSAGE: Optional[str] = ( -"You are an exceptionally capable, precise, and helpful multimodal AI assistant that excels at deeply understanding and richly describing images, charts, diagrams, text in images, scenes, and any visual content, " -"while also answering every question accurately, clearly, and step-by-step when appropriate ā€” always responding in the same language as the user's question, remaining polite, professional, and maximally helpful." - ) - - CHAT_FORMAT = ( - "{{ bos_token if bos_token is defined else '' }}" +@register_chat_completion_handler("chatml-function-calling") +def chatml_function_calling( + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + temperature: float = 0.2, + top_p: float = 0.95, + top_k: int = 40, + min_p: float = 0.05, + typical_p: float = 1.0, + stream: bool = False, + stop: Optional[Union[str, List[str]]] = [], + response_format: Optional[llama_types.ChatCompletionRequestResponseFormat] = None, + max_tokens: Optional[int] = None, + present_penalty: float = 0.0, + frequency_penalty: float = 0.0, + repeat_penalty: float = 1.1, + top_n_sigma: float = -1.00, + mirostat_mode: int = 0, + mirostat_tau: float = 5.0, + mirostat_eta: float = 0.1, + xtc_threshold: float = 0.1, + xtc_probability: float = 0.0, + dry_multiplier: float = 0.0, + dry_base: float = 1.75, + dry_allowed_length: int = 2, + dry_penalty_last_n:int = 0, + dry_seq_breakers: list[str] = ["\n", ":", "\"", "*"], + adaptive_target : float = -1.0, + adaptive_decay : float = 0.9, + use_infill: bool = False, + model: Optional[str] = None, + logits_processor: Optional[llama_core.LogitsProcessorList] = None, + grammar: Optional[llama_grammar.LlamaGrammar] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, + **kwargs, # type: ignore +) -> Union[ + llama_types.CreateChatCompletionResponse, + Iterator[llama_types.CreateChatCompletionStreamResponse], +]: + function_calling_template = ( "{% for message in messages %}" - "{% if message.role == 'system' %}" - "{{ message.content }}" - "{% elif message.role == 'user' %}" - "USER: " - "{% if message.content is string %}" - "{{ message.content }}" - "{% elif message.content is iterable %}" - "{% for content in message.content %}" - "{% if content.type == 'image_url' %}" - "{{ content.image_url if content.image_url is string else content.image_url.url }}" - "{% elif content.type == 'audio_url' %}" - "{{ content.audio_url if content.audio_url is string else content.audio_url.url }}" - "{% elif content.type == 'input_audio' %}" - "{% if content.input_audio is string %}" - "{{ content.input_audio }}" - "{% else %}" - "data:audio/{{ content.input_audio.format }};base64,{{ content.input_audio.data }}" - "{% endif %}" - "{% elif content.type == 'video_url' %}" - "{{ content.video_url if content.video_url is string else content.video_url.url }}" - "{% elif content.type == 'text' %}" - "{{ content.text }}" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - - "{% elif message.role == 'assistant' and message.content is not none %}" - "ASSISTANT: {{ message.content }}" - "{% endif %}" - "{{ \"\n\" }}" + "<|im_start|>{{ message.role }}\n" + # System message + "{% if message.role == 'system' %}" + "{{ message.content }}" + "{% if tool_calls %}" + "\n\nYou have access to the following functions:\n" + "{% for tool in tools %}" + "\nfunctions.{{ tool.function.name }}:\n" + "{{ tool.function.parameters | tojson }}" + "\n{% endfor %}" + "\n\nYou can respond to users messages with either a single message or one or more function calls." + "\n\nTo respond with a message begin the message with 'message:', use the following format:" + "\n\nmessage:" + "\n" + "\n\nTo respond with one or more function calls begin the message with 'functions.:', use the following format:" + "\n\nfunctions.:" + '\n{ "arg1": "value1", "arg2": "value2" }' + "\nfunctions.:" + '\n{ "arg1": "value1", "arg2": "value2" }' + "{% endif %}" + "<|im_end|>\n" + "{% endif %}" + # User message + "{% if message.role == 'user' %}" + "{{ message.content }}" + "<|im_end|>\n" + "{% endif %}" + # Assistant message + "{% if message.role == 'assistant' %}" + ## Reglar message + "{% if message.content and message.content | length > 0 %}" + "{% if tool_calls %}" + "message:\n" + "{% endif %}" + "{{ message.content }}" + "<|im_end|>\n" + "{% endif %}" + ## Function calls + "{% if 'tool_calls' in message %}" + "{% for tool_call in message.tool_calls %}" + "functions.{{ tool_call.function.name }}:\n" + "{{ tool_call.function.arguments }}" "{% endfor %}" - - "{% if eos_token is defined %}" - "{{ eos_token }}" + "<|im_end|>\n" "{% endif %}" - - "{% if add_generation_prompt %}" - "ASSISTANT: " "{% endif %}" + "{% endfor %}" + "{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" ) + template_renderer = ImmutableSandboxedEnvironment( + autoescape=jinja2.select_autoescape(["html", "xml"]), + undefined=jinja2.StrictUndefined, + ).from_string(function_calling_template) - def __init__( - self, - mmproj_path: str, - verbose: bool = True, - use_gpu: bool = True, - image_min_tokens: int = -1, - image_max_tokens: int = -1, - chat_template_override: Optional[str] = None, - batch_max_tokens: int = 1024, - **kwargs - ): - - self.log_prefix = self.__class__.__name__ - if kwargs: - unexpected_args = ", ".join(f"'{k}'" for k in kwargs.keys()) - raise TypeError( - f"Initialization Error in {self.log_prefix}: Received unexpected keyword argument(s) {unexpected_args}.\n" - f"If you are passing model-specific parameters, ensure they are supported by {self.log_prefix}." - ) - - self.mmproj_path = mmproj_path - self.image_min_tokens = image_min_tokens - self.image_max_tokens = image_max_tokens - self.batch_max_tokens = batch_max_tokens - self.use_gpu = use_gpu - self.verbose = verbose - - import llama_cpp.mtmd_cpp as mtmd_cpp - self._mtmd_cpp = mtmd_cpp - self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None - self.extra_template_arguments: dict[str, Any] = {} - - self.is_support_vision = False - self.is_support_audio = False - self.is_support_video = False - - if not os.path.exists(mmproj_path): - raise ValueError(f"{self.log_prefix}(__init__): Clip model path does not exist: {mmproj_path}") - - # Pre-compile Jinja template - if (not hasattr(self, "chat_format") or self.chat_format is None) and chat_template_override is None: - self.chat_format = self.CHAT_FORMAT - elif chat_template_override is not None: - self.chat_format = chat_template_override - - self._chat_format_parser_tags = [] - self.change_chat_template(self.chat_format) - - self._exit_stack = ExitStack() - - def change_chat_template(self, new_template: str): - self.chat_template = ImmutableSandboxedEnvironment( - trim_blocks=True, - lstrip_blocks=True - ).from_string(new_template) - - def _init_mtmd_context(self, llama_model: llama_core.Llama): - """Initialize mtmd context with the llama model.""" - if self.mtmd_ctx is not None: - return # Already initialized - - self._mtmd_cpp.mtmd_helper_log_set(ggml_log_callback, ctypes.c_void_p(0)) - - # Get default parameters - self.mctx_params = self._mtmd_cpp.mtmd_context_params_default() - self.mctx_params.use_gpu = self.use_gpu - self.mctx_params.print_timings = self.verbose - self.mctx_params.n_threads = llama_model.n_threads - self.mctx_params.flash_attn_type = self._mtmd_cpp.clip_flash_attn_type.CLIP_FLASH_ATTN_TYPE_AUTO - self.mctx_params.warmup = True - if self.image_min_tokens > 0: - self.mctx_params.image_min_tokens = self.image_min_tokens - if self.image_max_tokens > 0: - self.mctx_params.image_max_tokens = self.image_max_tokens - if (self.image_max_tokens < self.image_min_tokens) and self.image_max_tokens > 0: - raise ValueError(f"{self.log_prefix}(_init_mtmd_context): Configuration Error! image_max_tokens ({self.image_max_tokens}) " - f"cannot be less than image_min_tokens ({self.image_min_tokens}).") - self.mctx_params.batch_max_tokens = self.batch_max_tokens - - # Cache the model's eos token and bos token - self.mtmd_eos_token=llama_model.detokenize([llama_model.token_eos()]).decode('utf-8', errors='ignore') - self.mtmd_bos_token=llama_model.detokenize([llama_model.token_bos()]).decode('utf-8', errors='ignore') - - # Cache the mtmd_default_marker - self.media_marker = self._mtmd_cpp.mtmd_default_marker().decode('utf-8') - - # Initialize mtmd context - self.mtmd_ctx = self._mtmd_cpp.mtmd_init_from_file( - self.mmproj_path.encode(), - llama_model.model, - self.mctx_params - ) - - if self.mtmd_ctx is None: - raise ValueError(f"{self.log_prefix}(_init_mtmd_context): Failed to load mtmd context from: {self.mmproj_path}") - - # Check if vision is supported - self.is_support_vision = self._mtmd_cpp.mtmd_support_vision(self.mtmd_ctx) - if self.is_support_vision: - if self.verbose: - print(f"{self.log_prefix}(_init_mtmd_context): Vision support detected.", file=sys.stderr) - else: - if self.verbose: - print(f"{self.log_prefix}(_init_mtmd_context): Vision is NOT supported by this mmproj model backend.", file=sys.stderr) - - # Check if audio is supported - self.is_support_audio = self._mtmd_cpp.mtmd_support_audio(self.mtmd_ctx) - if self.is_support_audio: - if self.verbose: - print(f"{self.log_prefix}(_init_mtmd_context): Audio support detected.", file=sys.stderr) - else: - if self.verbose: - print(f"{self.log_prefix}(_init_mtmd_context): Audio is NOT supported by this mmproj model backend.", file=sys.stderr) - - # Check if video is supported - self.is_support_video = self._mtmd_cpp.mtmd_helper_support_video(self.mtmd_ctx) - if self.is_support_video: - if self.verbose: - print(f"{self.log_prefix}(_init_mtmd_context): Video support detected.", file=sys.stderr) - else: - if self.verbose: - print(f"{self.log_prefix}(_init_mtmd_context): Video support is NOT available in this build.", file=sys.stderr) - - def close(self) -> None: - """Explicitly free the mtmd context and vision model resources.""" - if getattr(self, "mtmd_ctx", None) is not None: - try: - self._mtmd_cpp.mtmd_free(self.mtmd_ctx) - except Exception: - pass - self.mtmd_ctx = None - self.mctx_params = None - self.chat_template = None - - if getattr(self, "_exit_stack", None) is not None and hasattr(self._exit_stack, "close"): - self._exit_stack.close() - self._exit_stack = None - - def __del__(self) -> None: - self.close() - - def _get_media_items(self, messages: List[llama_types.ChatCompletionRequestMessage]) -> List[Dict[str, str]]: - """ - Extracts all media payloads (images, audio) sequentially to maintain exact chronological order. - Strictly enforces capability checks, raising exceptions if unsupported media is passed. - - Returns: - media_items: A list of dictionaries containing the media 'url' and its 'type' (image or audio). - """ - media_items: List[Dict[str, str]] = [] - for message in messages: - if isinstance(message.get("content"), list): - for content in message["content"]: - content_type = content.get("type", "") - - # 1. Vision Processing - if content_type == "image_url": - if not self.is_support_vision: - raise ValueError(f"{self.log_prefix}: This mmproj model instance does not support image inputs.") - - url = content["image_url"] if isinstance(content["image_url"], str) else content["image_url"]["url"] - media_items.append({"url": url, "type": "image"}) - - # 2. Audio Processing - elif content_type in ["audio", "audio_url", "input_audio"]: - if not self.is_support_audio: - raise ValueError(f"{self.log_prefix}: This mmproj model instance does not support audio inputs.") - - # Case A: Handle custom/forward-compatible audio_url format - if content_type == "audio_url" or content_type == "audio": - audio_url = content[content_type] - url = audio_url if isinstance(audio_url, str) else audio_url["url"] - media_items.append({"url": url, "type": "audio"}) - # Case B: Handle OpenAI standard input_audio format - elif content_type == "input_audio": - input_audio = content.get("input_audio", {}) - if isinstance(input_audio, dict) and "data" in input_audio: - # It might just be raw base64 data, we can format it as a data URI to reuse load_audio logic - # input_audio: { - # data: audio.base64Data, - # format: audio.mimeType.includes('wav') ? 'wav' : 'mp3' - # } - audio_data = input_audio.get("data", "") - audio_format = input_audio.get("format", "") - - # Strictly align with llama.cpp (require wav/mp3) - if audio_format not in ["wav", "mp3"]: - raise ValueError(f"{self.log_prefix}: input_audio.format must be either 'wav' or 'mp3'") - - # Format as a Data URI to reuse the unified load_media logic - media_items.append({ - "url": f"data:audio/{audio_format};base64,{audio_data}", - "type": "audio" - }) - else: - # Just a raw base64 data - url = input_audio if isinstance(input_audio, str) else "" - if url: - media_items.append({"url": url, "type": "audio"}) - - # 3. Video Processing - elif content_type == "video_url": - if not self.is_support_video: - raise ValueError(f"{self.log_prefix}: This libmtmd build does not support video inputs.") - - video_url = content["video_url"] - url = video_url if isinstance(video_url, str) else video_url["url"] - media_items.append({"url": url, "type": "video"}) - - # 4. Text & Unknown Types - elif content_type == "text": - continue - else: - if self.verbose: - print(f"{self.log_prefix}: Ignored unknown content type '{content_type}'.", file=sys.stderr) - return media_items - - def _create_bitmap_from_bytes(self, media_bytes: bytes): - """ - Constructs an mtmd_bitmap structure from a raw byte buffer containing media data. - - Supported formats: - - Images (via stb_image): jpg, png, bmp, etc. - - Audio (via miniaudio): wav, mp3, flac. - - Video: depends on whether MTMD_VIDEO was enabled at build time. - - Note: - - Media types (Image vs. Audio) are auto-detected by the C++ backend using magic bytes. - - The underlying C++ helper function is thread-safe, making it suitable for concurrent preprocessing. - - Args: - media_bytes (bytes): The raw byte content of the media file. - - Returns: - bitmap: mtmd_bitmap * - video_ctx: mtmd_helper_video * or NULL - """ - if self.mtmd_ctx is None: - raise ValueError(f"{self.log_prefix}(_create_bitmap_from_bytes): mtmd context not initialized.") + # Convert legacy functions to tools + if functions is not None: + tools = [ + { + "type": "function", + "function": function, + } + for function in functions + ] - if not media_bytes: - raise ValueError(f"{self.log_prefix}(_create_bitmap_from_bytes): empty media bytes.") + # Convert legacy function_call to tool_choice + if function_call is not None: + if isinstance(function_call, str) and ( + function_call == "none" or function_call == "auto" + ): + tool_choice = function_call + if isinstance(function_call, dict) and "name" in function_call: + tool_choice = { + "type": "function", + "function": { + "name": function_call["name"], + }, + } - buf = (ctypes.c_uint8 * len(media_bytes)).from_buffer_copy(media_bytes) + stop = ( + [stop, "<|im_end|>"] + if isinstance(stop, str) + else stop + ["<|im_end|>"] if stop else ["<|im_end|>"] + ) - wrapper = self._mtmd_cpp.mtmd_helper_bitmap_init_from_buf( - self.mtmd_ctx, - buf, - len(media_bytes), - False, + # Case 1: No tool choice by user + if ( + tool_choice is None + or (isinstance(tool_choice, str) and tool_choice == "none") + or tools is None + or len(tools) == 0 + ): + prompt = template_renderer.render( + messages=messages, + tools=[], + tool_calls=None, + add_generation_prompt=True, ) - if not wrapper.bitmap: - if wrapper.video_ctx: - self._mtmd_cpp.mtmd_helper_video_free(wrapper.video_ctx) - - raise ValueError( - f"{self.log_prefix}(_create_bitmap_from_bytes): " - "Failed to load media from bytes " - "(unsupported media format, corrupted data, or missing helper support)." - ) - - return wrapper.bitmap, wrapper.video_ctx - - def _is_text_chunk(self, chunk_type: int) -> bool: - """Return True if `chunk_type` is the MTMD text chunk type enum value.""" - return ( - chunk_type - == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_TEXT - ) + if response_format is not None and response_format["type"] == "json_object": + grammar = _grammar_for_response_format(response_format) - def _is_image_chunk(self, chunk_type: int) -> bool: - """Return True if `chunk_type` is the MTMD image chunk type enum value.""" - return ( - chunk_type - == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE + return _convert_completion_to_chat( + llama.create_completion( + prompt=prompt, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + typical_p=typical_p, + stream=stream, + stop=stop, + max_tokens=max_tokens, + present_penalty=present_penalty, + frequency_penalty=frequency_penalty, + repeat_penalty=repeat_penalty, + top_n_sigma=top_n_sigma, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + xtc_threshold=xtc_threshold, + xtc_probability=xtc_probability, + dry_multiplier=dry_multiplier, + dry_base=dry_base, + dry_allowed_length=dry_allowed_length, + dry_penalty_last_n=dry_penalty_last_n, + dry_seq_breakers=dry_seq_breakers, + adaptive_target=adaptive_target, + adaptive_decay=adaptive_decay, + use_infill=use_infill, + model=model, + logits_processor=logits_processor, + grammar=grammar, + logprobs=top_logprobs if logprobs else None, + ), + stream=stream, ) - def _is_audio_chunk(self, chunk_type: int) -> bool: - """Return True if `chunk_type` is the MTMD audio chunk type enum value.""" - return ( - chunk_type - == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO + # Case 2: Tool choice by user + if isinstance(tool_choice, dict): + tool_name = tool_choice["function"]["name"] + tool = next( + (tool for tool in tools if tool["function"]["name"] == tool_name), None ) - - def _process_mtmd_prompt( - self, - llama: llama_core.Llama, - messages: List[llama_types.ChatCompletionRequestMessage], - functions: Optional[List[llama_types.ChatCompletionFunction]] = None, - function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, - tools: Optional[List[llama_types.ChatCompletionTool]] = None, - tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, - add_generation_prompt: bool = True, - ) -> Tuple[List[int], List[tuple], Any, List[Any]]: - """ - Core multimodal preprocessing pipeline. - Converts raw chat messages into C++ MTMD chunk structures and a virtual token ledger. - - Features: - - Thread-safe concurrent media decoding to eliminate I/O bottlenecks. - - "Negative Reverse Vocabulary" mapping for O(1) prefix matching of media tokens. - - Strict RAII-style C++ memory management to prevent leaks on failure. - - Returns: - full_prompt_ids: Ledger of text tokens and negative media IDs for prefix matching. - chunk_token_spans: Tuples of (start_idx, end_idx, chunk_ptr, chunk_type, media_id). - chunks: Allocated C++ mtmd_input_chunks pointer (must be freed by the caller). - bitmap_cleanup: List of C++ bitmap pointers to be freed after evaluation. - """ - # 1. Inject default system prompt if omitted by the user - system_prompt = next((msg["content"] for msg in messages if msg.get("role") == "system"), "") - if system_prompt == "" and self.DEFAULT_SYSTEM_MESSAGE is not None: - messages = [{"role": "system", "content": self.DEFAULT_SYSTEM_MESSAGE}] + messages - - media_items = self._get_media_items(messages) - media_marker = self.media_marker - - # 2. Render the chat template and replace actual URLs with C++ media markers - text = self.chat_template.render( + if tool is None: + raise ValueError(f"Tool with name '{tool_name}' not found in tools") + prompt = template_renderer.render( messages=messages, - add_generation_prompt=add_generation_prompt, - eos_token=self.mtmd_eos_token, - bos_token=self.mtmd_bos_token, - functions=functions, - function_call=function_call, tools=tools, - tool_choice=tool_choice, - **getattr(self, 'extra_template_arguments', {}) + tool_calls=True, + add_generation_prompt=True, ) - - for tag in self._chat_format_parser_tags: - if tag not in text: - continue - - text = text.replace(tag, media_marker) - - # Replace image_url by media_marker in text - for item in media_items: - text = text.replace(item["url"], media_marker) - - if self.verbose: - print(f"{self.log_prefix}(_process_mtmd_prompt): Rendered prompt length: {len(text)} chars, Media count: {len(media_items)}.", file=sys.stderr) - print(f"{self.log_prefix}(_process_mtmd_prompt): Rendered prompt: {text}", file=sys.stderr) - - # 3. Pre-allocate bitmap array to guarantee chronological order during concurrent decoding - bitmaps = [None] * len(media_items) - bitmap_cleanup = [] - video_cleanup = [] - chunks = None - - try: - # Concurrent Media Decoding - import concurrent.futures - if media_items: - def _create_bitmap_func(idx: int, item: dict): - media_bytes = self.load_media(item["url"], item["type"]) - bitmap, video_ctx = self._create_bitmap_from_bytes(media_bytes) - return idx, bitmap, video_ctx - # This method uses multi-threaded parallel processing to convert images or audio to bitmaps, - # which can be used in the future to process large numbers of video frames. - max_workers = min(llama.n_threads, len(media_items)) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [executor.submit(_create_bitmap_func, i, item) for i, item in enumerate(media_items)] - - for future in concurrent.futures.as_completed(futures): - idx, bitmap, video_ctx = future.result() - - bitmaps[idx] = bitmap - bitmap_cleanup.append(bitmap) - - if video_ctx: - video_cleanup.append(video_ctx) - - # Strict validation: Abort if any thread failed to decode its assigned media - if any(b is None for b in bitmaps): - raise RuntimeError(f"{self.log_prefix}(_create_bitmap_func): Failed to decode one or more media files.") - else: - if self.verbose: - print(f"{self.log_prefix}(_create_bitmap_func with {max_workers} threads): {len(media_items)} bitmaps were successfully created.") - else: - # If there are no images, set the bitmaps to empty. - bitmaps = [] - - # 4. Initialize mtmd_input_chunks - input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode('utf-8') - input_text.add_special = (llama.n_tokens == 0) - input_text.parse_special = True - - chunks = self._mtmd_cpp.mtmd_input_chunks_init() - if chunks is None: - raise ValueError(f"{self.log_prefix}(mtmd_input_chunks_init): Failed to initialize mtmd_input_chunks.") - - # 5. Hybrid Tokenization (Text + Media binding) - if len(bitmaps) > 0: - bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) - result = self._mtmd_cpp.mtmd_tokenize( - self.mtmd_ctx, chunks, ctypes.byref(input_text), bitmap_array, len(bitmaps) - ) - else: - result = self._mtmd_cpp.mtmd_tokenize( - self.mtmd_ctx, chunks, ctypes.byref(input_text), None, 0 - ) - - if result != 0: - raise ValueError(f"{self.log_prefix}(mtmd_tokenize): Unable to tokenize prompt, res = {result}.") - - # Video helper contexts only need to stay alive until mtmd_tokenize() completes. - if video_cleanup: - for video_ctx in video_cleanup: - self._mtmd_cpp.mtmd_helper_video_free(video_ctx) - video_cleanup.clear() - - # 6. Virtual Token Ledger Construction - full_prompt_ids = [] - chunk_token_spans = [] - current_idx = 0 - n_chunks = self._mtmd_cpp.mtmd_input_chunks_size(chunks) - - # Cursor to track the actual media contents (URLs or base64 data) provided by the user - media_items_count = len(media_items) - media_items_cur = 0 - last_media_id = None - - for i in range(n_chunks): - chunk = self._mtmd_cpp.mtmd_input_chunks_get(chunks, i) - if chunk is None: continue - chunk_type = self._mtmd_cpp.mtmd_input_chunk_get_type(chunk) - - if self._is_text_chunk(chunk_type): - # Extract standard text token IDs - n_tokens_out = ctypes.c_size_t() - tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text(chunk, ctypes.byref(n_tokens_out)) - if tokens_ptr and n_tokens_out.value > 0: - tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)] - chunk_token_spans.append((current_idx, current_idx + len(tokens), chunk, chunk_type, None)) - full_prompt_ids.extend(tokens) - current_idx += len(tokens) - elif self._is_image_chunk(chunk_type) or self._is_audio_chunk(chunk_type): - # Extract media properties - # Note(JamePeng): - # The M-RoPE model is based on `n_pos` instead of `n_tokens` (of course, there's no difference in non-M-RoPE models). - # However, I still keep `n_tokens` because if `n_pos` is used, the underlying system will assume it is a full-match and will skip eval and sample. - # chunk_n_pos = self._mtmd_cpp.mtmd_input_chunk_get_n_pos(chunk) # equals to max(t,h,w) for M-RoPE; equals to `n_tokens` otherwise - chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk) - - if media_items_cur < media_items_count: - # The C++ parser only sees identical placeholders (e.g., "<__media__>"). - # We MUST inject the actual media content's identity here. - real_media_url = media_items[media_items_cur]["url"] - # Vocabulary Positive forward: 0 to 248,319 (Qwen3.5) - # Generate a deterministic, unique negative ID for this specific image/audio. - # - zlib.crc32 ensures cross-platform and cross-run consistency (unlike Python's hash()). - # - We map it to a negative space (-100 to -16,777,316) to avoid colliding with - # positive text token IDs (e.g., Qwen3.5 vocab goes up to ~152k). - # This empowers `longest_token_prefix` to correctly identify and reuse cached images, - # while instantly breaking the match if the image content changes. - # media_id = - (zlib.crc32(real_media_url.encode('utf-8')) % (2**24)) - 100 - media_id = - (zlib.crc32(real_media_url.encode('utf-8')) & 0xFFFFFF) - 100 - last_media_id = media_id - media_items_cur += 1 - elif last_media_id is not None: - # video may expand into multiple image chunks from one media marker - media_id = last_media_id - else: - # Magic Negative Number as fallback :) - media_id = -314159 - - if self.verbose: - print(f"{self.log_prefix}(mtmd_input_chunk_media_id): chunk_n_tokens: {chunk_n_tokens}, media_id: {media_id}, ") - - chunk_token_spans.append((current_idx, current_idx + chunk_n_tokens, chunk, chunk_type, media_id)) - - # Pad the ledger with the pseudo-ID to mimic the physical space taken in the KV cache - full_prompt_ids.extend([media_id] * chunk_n_tokens) - current_idx += chunk_n_tokens - else: - raise TypeError(f"{self.log_prefix}(mtmd_input_chunk_get_type): Invalid chunk type, chunk_type = {chunk_type}.") - - return full_prompt_ids, chunk_token_spans, chunks, bitmap_cleanup - - except Exception as e: - # Ensure no useless pointers remain upon any failure - # Free chunks - if chunks is not None: - self._mtmd_cpp.mtmd_input_chunks_free(chunks) - chunks = None - # Free bitmaps - if len(bitmap_cleanup) > 0: - for bitmap in bitmap_cleanup: - self._mtmd_cpp.mtmd_bitmap_free(bitmap) - bitmap_cleanup = None - # Free videos - if len(video_cleanup) > 0: - for video_ctx in video_cleanup: - self._mtmd_cpp.mtmd_helper_video_free(video_ctx) - video_cleanup = None - - bitmaps = None - - raise e - - def __call__( - self, - *, - llama: llama_core.Llama, - messages: List[llama_types.ChatCompletionRequestMessage], - functions: Optional[List[llama_types.ChatCompletionFunction]] = None, - function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, - tools: Optional[List[llama_types.ChatCompletionTool]] = None, - tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, - temperature: float = 0.2, - top_p: float = 0.95, - top_k: int = 40, - min_p: float = 0.05, - typical_p: float = 1.0, - stream: bool = False, - stop: Optional[Union[str, List[str]]] = [], - seed: Optional[int] = None, - response_format: Optional[ - llama_types.ChatCompletionRequestResponseFormat - ] = None, - max_tokens: Optional[int] = None, - present_penalty: float = 0.0, - frequency_penalty: float = 0.0, - repeat_penalty: float = 1.1, - top_n_sigma: float = -1.00, - mirostat_mode: int = 0, - mirostat_tau: float = 5.0, - mirostat_eta: float = 0.1, - xtc_threshold: float = 0.1, - xtc_probability: float = 0.0, - dry_multiplier: float = 0.0, - dry_base: float = 1.75, - dry_allowed_length: int = 2, - dry_penalty_last_n:int = 0, - dry_seq_breakers: list[str] = ["\n", ":", "\"", "*"], - adaptive_target : float = -1.0, - adaptive_decay : float = 0.9, - use_infill: bool = False, - model: Optional[str] = None, - logits_processor: Optional[llama_core.LogitsProcessorList] = None, - grammar: Optional[llama_grammar.LlamaGrammar] = None, - logit_bias: Optional[Dict[str, float]] = None, - logprobs: Optional[bool] = None, - top_logprobs: Optional[int] = None, - add_generation_prompt: bool = True, - reasoning_budget: int = -1, - reasoning_start: str = "", - reasoning_end: str = "", - reasoning_budget_message: Optional[str] = None, - reasoning_start_in_prompt: bool = False, - reasoning_start_max_tokens: Optional[int] = 32, - **kwargs, # type: ignore - ) -> Union[ - llama_types.CreateChatCompletionResponse, - Iterator[llama_types.CreateChatCompletionStreamResponse], - ]: - # 1. Initialize mtmd context - self._init_mtmd_context(llama) - assert self.mtmd_ctx is not None - - # 2. Concurrent Preprocessing & Ledger Construction - full_prompt_ids, chunk_token_spans, chunks, bitmap_cleanup = self._process_mtmd_prompt( - llama=llama, - messages=messages, - functions=functions, - function_call=function_call, - tools=tools, - tool_choice=tool_choice, - add_generation_prompt=add_generation_prompt, - ) - - if self.verbose: - print(f"{self.log_prefix}(__call__): Prepared virtual token ledger of length {len(full_prompt_ids)}.", file=sys.stderr) - - try: - # 3. KV Cache Synchronization & State Rollback - # Compares the virtual ledger with physical history to prevent Cache Poisoning. - current_history = llama.input_ids[:llama.n_tokens].tolist() - longest_prefix = llama.longest_token_prefix(current_history, full_prompt_ids, self.verbose) - - if longest_prefix < llama.n_tokens: - if llama.is_hybrid and llama._hybrid_cache_mgr is not None: - if llama._hybrid_cache_mgr.max_checkpoints > 0: - if self.verbose: - print(f"{self.log_prefix}(__call__): Hybrid prefix mismatch (matched {longest_prefix}/{llama.n_tokens}). " - f"Searching for nearest checkpoint...", file=sys.stderr) - - best_ckpt = llama._hybrid_cache_mgr.find_best_checkpoint(full_prompt_ids, seq_id=0) - if best_ckpt and llama._hybrid_cache_mgr.restore_checkpoint(best_ckpt, seq_id=0): - llama.n_tokens = best_ckpt.pos - if self.verbose: - print(f"{self.log_prefix}(__call__): Successfully rolled back to checkpoint at pos {llama.n_tokens}.", file=sys.stderr) - else: - if self.verbose: - print(f"{self.log_prefix}(__call__): No suitable checkpoint found or restore failed. Clearing hybrid cache entirely.", file=sys.stderr) - llama._hybrid_cache_mgr.clear() - llama._ctx.memory_clear(True) - llama.n_tokens = 0 - else: - if self.verbose: - print(f"{self.log_prefix}(__call__): Hybrid cache enabled but max_checkpoints is 0. Clearing cache entirely.", file=sys.stderr) - llama._hybrid_cache_mgr.clear() - llama._ctx.memory_clear(True) - llama.n_tokens = 0 - else: - if self.verbose: - print(f"{self.log_prefix}(__call__): Prefix mismatch. Truncating KV cache from {llama.n_tokens} to {longest_prefix}.", file=sys.stderr) - llama._ctx.memory_seq_rm(0, longest_prefix, -1) - llama.n_tokens = longest_prefix - - n_past = llama.n_tokens - - for start_idx, end_idx, chunk_ptr, chunk_type, media_id in chunk_token_spans: - # Skip previously matched chunks - if end_idx <= n_past: - continue - - if self._is_text_chunk(chunk_type): - unprocessed_start = max(start_idx, n_past) - start_idx - n_tokens_out = ctypes.c_size_t() - tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text(chunk_ptr, ctypes.byref(n_tokens_out)) - - if tokens_ptr and n_tokens_out.value > 0: - all_tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)] - tokens_to_eval = all_tokens[unprocessed_start:] - - if tokens_to_eval: - if self.verbose: - print(f"{self.log_prefix}(__call__): Evaluating TEXT chunk ({len(tokens_to_eval)} tokens) at pos {llama.n_tokens}...", file=sys.stderr) - # Text evaluation delegates shift and chunking to native llama.eval - llama.eval(tokens_to_eval) - n_past = llama.n_tokens - - elif self._is_image_chunk(chunk_type) or self._is_audio_chunk(chunk_type): - chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk_ptr) - - if self.verbose: - media_str = "IMAGE" if self._is_image_chunk(chunk_type) else "AUDIO" - print(f"{self.log_prefix}(__call__): Evaluating {media_str} chunk ({chunk_n_tokens} tokens) at pos {llama.n_tokens}...", file=sys.stderr) - - # Stage 5: Multimodal Physical OOM Defense - if n_past + chunk_n_tokens > llama.n_ctx(): - if not llama._ctx.memory_can_shift(): - raise RuntimeError( - f"{self.log_prefix}(__call__): Context Shift is explicitly disabled by the C++ backend " - f"(n_pos_per_embd > 1 or incompatible M-RoPE). " - f"Multimodal chunk exceeded context limit(currently n_ctx={llama._n_ctx}), " - f"You MUST increase n_ctx to fit the dialogue." - ) - else: - # Safely discard oldest tokens while preserving system prompts - n_discard = (n_past + chunk_n_tokens) - llama.n_ctx() + llama.n_batch - n_keep = min(llama.n_keep, n_past) - n_discard = min(n_discard, n_past - n_keep) - - if n_discard <= 0: - raise RuntimeError(f"{self.log_prefix}(__call__): Critical Overflow. Not enough unpinned tokens to discard for Context Shift.") - - if self.verbose: - print(f"{self.log_prefix}(__call__): OOM risk detected. Shifting multimodal context: keeping {n_keep}, discarding {n_discard}...", file=sys.stderr) - - # Execute physical memory shift - llama._ctx.memory_seq_rm(0, n_keep, n_keep + n_discard) - llama._ctx.memory_seq_add(0, n_keep + n_discard, n_past, -n_discard) - - # Shift python virtual array to match - remaining_len = n_past - (n_keep + n_discard) - if remaining_len > 0: - llama.input_ids[n_keep : n_keep + remaining_len] = llama.input_ids[n_keep + n_discard : n_past] - - n_past -= n_discard - llama.n_tokens = n_past - - # Execute C++ Multimodal Black-box Extraction - new_n_past = llama_cpp_lib.llama_pos(0) - result = self._mtmd_cpp.mtmd_helper_eval_chunk_single( - self.mtmd_ctx, - llama._ctx.ctx, - chunk_ptr, - llama_cpp_lib.llama_pos(n_past), - llama_cpp_lib.llama_seq_id(0), - llama.n_batch, - True, # logits_last = True, drastically saves computational overhead - ctypes.byref(new_n_past) - ) - - if result != 0: - raise ValueError(f"{self.log_prefix}(mtmd_helper_eval_chunk_single): Media evaluation failed with error code {result}.") - - # Update Ledger with "Negative Reverse Vocabulary" IDs - llama.input_ids[n_past : new_n_past.value] = media_id - n_past = new_n_past.value - llama.n_tokens = n_past - - # Extract the final, perfectly synchronized prompt sequence - prompt = llama.input_ids[: llama.n_tokens].tolist() - - # End-of-Turn Checkpoint - # Anchors the state ONLY after the entire multi-modal turn is processed - if ( - llama.is_hybrid - and llama._hybrid_cache_mgr is not None - and llama._hybrid_cache_mgr.max_checkpoints > 0 - ): - if self.verbose: - print(f"{self.log_prefix}(__call__): [End-of-Turn Checkpoint] Anchoring full prompt state at pos {llama.n_tokens}.", file=sys.stderr) - - llama._hybrid_cache_mgr.save_checkpoint( - current_pos=llama.n_tokens, - tokens=prompt, - seq_id=0 - ) - finally: - # Cleanup chunks - if chunks is not None: - self._mtmd_cpp.mtmd_input_chunks_free(chunks) - chunks = None - # Cleanup bitmaps - if bitmap_cleanup: - for bitmap in bitmap_cleanup: - self._mtmd_cpp.mtmd_bitmap_free(bitmap) - bitmap_cleanup.clear() - bitmap_array = None - - # Handle response format and tools (same as before) - if response_format is not None and response_format["type"] == "json_object": - grammar = _grammar_for_response_format(response_format) - - # Convert legacy functions to tools - if functions is not None: - tools = [ - { - "type": "function", - "function": function, - } - for function in functions - ] - - # Convert legacy function_call to tool_choice - if function_call is not None: - if isinstance(function_call, str) and ( - function_call == "none" or function_call == "auto" - ): - tool_choice = function_call - if isinstance(function_call, dict) and "name" in function_call: - tool_choice = { - "type": "function", - "function": { - "name": function_call["name"], - }, - } - - tool = None - if ( - tool_choice is not None - and isinstance(tool_choice, dict) - and tools is not None - ): - name = tool_choice["function"]["name"] - tool = next((t for t in tools if t["function"]["name"] == name), None) - if tool is None: - raise ValueError(f"Tool choice '{name}' not found in tools.") - schema = tool["function"]["parameters"] - try: - # create grammar from json schema - grammar = llama_grammar.LlamaGrammar.from_json_schema( - json.dumps(schema), verbose=llama.verbose - ) - except Exception as e: - if llama.verbose: - print(str(e), file=sys.stderr) - grammar = llama_grammar.LlamaGrammar.from_string( - llama_grammar.JSON_GBNF, verbose=llama.verbose - ) - - completion_or_chunks = llama.create_completion( - prompt=prompt, - temperature=temperature, - top_p=top_p, - top_k=top_k, - min_p=min_p, - typical_p=typical_p, - logprobs=top_logprobs if logprobs else None, - stream=stream, - stop=stop, - seed=seed, - max_tokens=max_tokens, - present_penalty=present_penalty, - frequency_penalty=frequency_penalty, - repeat_penalty=repeat_penalty, - top_n_sigma=top_n_sigma, - mirostat_mode=mirostat_mode, - mirostat_tau=mirostat_tau, - mirostat_eta=mirostat_eta, - xtc_threshold=xtc_threshold, - xtc_probability=xtc_probability, - dry_multiplier=dry_multiplier, - dry_base=dry_base, - dry_allowed_length=dry_allowed_length, - dry_penalty_last_n=dry_penalty_last_n, - dry_seq_breakers=dry_seq_breakers, - adaptive_target=adaptive_target, - adaptive_decay=adaptive_decay, - use_infill=use_infill, - model=model, - logits_processor=logits_processor, - grammar=grammar, - logit_bias=logit_bias, - reasoning_budget=reasoning_budget, - reasoning_start=reasoning_start, - reasoning_end=reasoning_end, - reasoning_budget_message=reasoning_budget_message, - reasoning_start_in_prompt=reasoning_start_in_prompt, - reasoning_start_max_tokens=reasoning_start_max_tokens, - ) - - if tool is not None: - tool_name = tool["function"]["name"] - return _convert_completion_to_chat_function( - tool_name, completion_or_chunks, stream - ) - return _convert_completion_to_chat(completion_or_chunks, stream=stream) - - def load_media(self, media_url: str, media_type: str) -> bytes: - """ - Unified dispatcher for loading media payloads. - Routes the URL/URI to the specific image, audio, or video processor based on the media_type. - """ - if media_type == "image": - return self._load_image(media_url) - - elif media_type == "audio": - audio_bytes = self._load_bytes(media_url, timeout=15, kind="audio") - try: - self.detect_audio_format(audio_bytes) - except ValueError as e: - raise ValueError(f"{self.log_prefix}(load_media): {e}") - return audio_bytes - - elif media_type == "video": - return self._load_bytes(media_url, timeout=30, kind="video") - - else: - raise ValueError(f"{self.log_prefix}(load_media): Unknown media type '{media_type}'") - - @staticmethod - def detect_audio_format(audio_bytes: bytes) -> str: - """ - Pure utility function: Detects the audio format from magic bytes. - Strictly translated from llama.cpp's `is_audio_file` to ensure 100% compatibility - and avoid false positives (e.g., AVI files disguised as RIFF). - """ - length = len(audio_bytes) - - if length < 12: - raise ValueError("Audio data is corrupted or too small (less than 12 bytes).") - - # RIFF & WAVE magic bytes verification - is_wav = audio_bytes.startswith(b"RIFF") and audio_bytes[8:12] == b"WAVE" - - # ID3 metadata or MPEG sync word verification - is_mp3 = length >= 3 and ( - audio_bytes.startswith(b"ID3") or - (audio_bytes[0] == 0xFF and (audio_bytes[1] & 0xE0) == 0xE0) - ) - - # FLAC magic bytes verification - is_flac = audio_bytes.startswith(b"fLaC") - - if is_wav: - return "wav" - elif is_mp3: - return "mp3" - elif is_flac: - return "flac" - else: - raise ValueError( - "Unsupported audio format detected via magic bytes. " - "The underlying C++ miniaudio backend ONLY supports WAV, MP3, and FLAC." - ) - - DEFAULT_HTTP_HEADERS = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/148.0.0.0 Safari/537.36" - ), - } - - @staticmethod - def _load_bytes(media_url: str, timeout: int = 15, kind: str = "media") -> bytes: - """ - Load raw bytes from a data URI, local file path, or remote HTTP/HTTPS URL. - """ - media_bytes = b"" - - # 1. Handle data URI - if media_url.strip().startswith("data:"): - comma_pos = media_url.find(",") - if comma_pos == -1: - raise ValueError("Invalid data URI: missing comma separator") - - base64_data = media_url[comma_pos + 1:] - media_bytes = base64.b64decode(base64_data) - - # 2. Handle local file path - elif os.path.exists(media_url): - with open(media_url, "rb") as f: - media_bytes = f.read() - - # 3. Handle remote URL via HTTP/HTTPS - else: - req = urllib.request.Request( - media_url, - headers=MTMDChatHandler.DEFAULT_HTTP_HEADERS, - ) - try: - with urllib.request.urlopen(req, timeout=timeout) as f: - media_bytes = f.read() - except (URLError, HTTPError) as e: - raise ConnectionError(f"Failed to download {kind} from {media_url}: {e}") - - if not media_bytes: - raise ValueError(f"Empty {kind} data received") - - return media_bytes - - @staticmethod - def _load_image(image_url: str) -> bytes: - """ - Load an image from either a URL or a data URI and return it as JPEG bytes. - - Supports: - - Remote images via HTTP/HTTPS (with proper User-Agent) - - Data URIs (base64-encoded, e.g., data:image/png;base64,...) - - Images with alpha channel (PNG, WebP, etc.) ā†’ automatically composites on white/black background - - Any format that Pillow can open. See: https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html - - Returns: - JPEG-encoded bytes (quality=95) in RGB mode, suitable for most vision models. - """ - # 1. Load image bytes from image_url - image_bytes = MTMDChatHandler._load_bytes( - image_url, - timeout=15, - kind="image", - ) - - # 2. Check if image_bytes is empty. - if not image_bytes: - raise ValueError("Empty image data received") - - # 3. Open image with Pillow - try: - from PIL import Image, ImageStat - except ImportError: - raise ImportError("Pillow is required for image processing. Install with: pip install pillow") - - import io - image = Image.open(io.BytesIO(image_bytes)) - - # 4. Handle transparency (RGBA, LA, P with transparency, etc.) - if image.mode in ("RGBA", "LA", "PA") or (image.mode == "P" and "transparency" in image.info): - # Use alpha channel as mask - if image.mode == "P": - image = image.convert("RGBA") - - alpha = image.split()[-1] # Last channel is alpha - # Compute average brightness of visible (non-transparent) pixels - stat = ImageStat.Stat(image.convert("L"), mask=alpha) - - # Choose background: white for dark content, black for bright content - bg_color = (255, 255, 255) # white - if stat.count[0] > 0 and stat.mean[0] > 127: - bg_color = (0, 0, 0) # black - - background = Image.new("RGB", image.size, bg_color) - background.paste(image, mask=alpha) - image = background - - # 5. Ensure RGB mode for formats like CMYK, palette, etc. - elif image.mode != "RGB": - image = image.convert("RGB") - - # 6. Save as high-quality JPEG, suitable for most vision models. - output = io.BytesIO() - image.save(output, format="JPEG", quality=95, optimize=True, progressive=True) - return output.getvalue() - - @classmethod - def from_pretrained( - cls, - repo_id: str, - filename: Optional[str], - local_dir: Optional[Union[str, os.PathLike[str]]] = None, - local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", - cache_dir: Optional[Union[str, os.PathLike[str]]] = None, - **kwargs: Any, - ) -> "MTMDChatHandler": - import fnmatch - from pathlib import Path - - try: - from huggingface_hub import hf_hub_download, HfFileSystem # type: ignore - from huggingface_hub.utils import validate_repo_id # type: ignore - except ImportError: - raise ImportError( - "Llama.from_pretrained requires the huggingface_hub package. " - "You can install it with `pip install --upgrade huggingface_hub`." - ) - - validate_repo_id(repo_id) - - hffs = HfFileSystem() - - files = [ - file["name"] if isinstance(file, dict) else file - for file in hffs.ls(repo_id) # type: ignore - ] - - # split each file into repo_id, subfolder, filename - file_list: List[str] = [] - for file in files: - rel_path = Path(file).relative_to(repo_id) - file_list.append(str(rel_path)) - - matching_files = [file for file in file_list if fnmatch.fnmatch(file, filename)] # type: ignore - - if len(matching_files) == 0: - raise ValueError( - f"No file found in {repo_id} that match {filename}\n\n" - f"Available Files:\n{json.dumps(file_list)}" - ) - - if len(matching_files) > 1: - raise ValueError( - f"Multiple files found in {repo_id} matching {filename}\n\n" - f"Available Files:\n{json.dumps(files)}" - ) - - (matching_file,) = matching_files - - subfolder = str(Path(matching_file).parent) - filename = Path(matching_file).name - - # download the file - hf_hub_download( - repo_id=repo_id, - filename=filename, - subfolder=subfolder, - local_dir=cast(Union[str, Path, None], local_dir), - local_dir_use_symlinks=local_dir_use_symlinks, - cache_dir=cast(Union[str, Path, None], cache_dir), - ) - - if local_dir is None: - model_path = hf_hub_download( - repo_id=repo_id, - filename=filename, - subfolder=subfolder, - local_dir=local_dir, - local_dir_use_symlinks=local_dir_use_symlinks, - cache_dir=cast(Union[str, Path, None], cache_dir), - local_files_only=True, - ) - else: - model_path = os.path.join(local_dir, filename) - - return cls( - mmproj_path=model_path, - **kwargs, - ) - -class GenericMTMDChatHandler(MTMDChatHandler): - KNOWN_MEDIA_TAGS = [ - "<|image_pad|>", - "<|audio_pad|>", - "<|video_pad|>", - "<|image|>", - "<|audio|>", - "<|video|>", - "[IMG]" - ] - - def __init__( - self, - chat_format: str, - mmproj_path: str, - verbose: bool = True, - **kwargs - ) -> None: - self.chat_format = chat_format - - if verbose: - print(f"Got chat template from model:\n```jinja\n{self.chat_format}\n```", flush = True) - - if self.chat_format is None: - raise ValueError("Failed to get model chat template automatically.") - - super().__init__(mmproj_path = mmproj_path, verbose = verbose, **kwargs) - - def __call__(self, **kwargs): - self._chat_format_parser_tags = [tag for tag in self.KNOWN_MEDIA_TAGS if tag in self.chat_format] - - if self.verbose: - print(f"{self.log_prefix} - Start processing") - - # Use parent implementation - return super().__call__(**kwargs) - -class Llava15ChatHandler(MTMDChatHandler): - CHAT_FORMAT = ( - "{% for message in messages %}" - "{% if message.role == 'system' %}" - "{{ message.content }}" - "{% endif %}" - - "{% if message.role == 'user' %}" - "{% if message.content is string %}" - "\nUSER: {{ message.content }}" - "{% elif message.content is iterable %}" - "\nUSER: " - "{% for content in message.content %}" - "{% if content.type == 'image_url' %}" - "{{ content.image_url if content.image_url is string else content.image_url.url }}" - "{% endif %}" - "{% endfor %}" - "{% for content in message.content %}" - "{% if content.type == 'text' %}" - "{{ content.text }}" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - "{% endif %}" - - "{% if message.role == 'assistant' and message.content is not none %}" - "\nASSISTANT: {{ message.content }}" - "{% endif %}" - "{% endfor %}" - - "{% if add_generation_prompt %}" - "\nASSISTANT: " - "{% endif %}" - ) - - -class ObsidianChatHandler(MTMDChatHandler): - # Prompt Format - # The model followed ChatML format. However, with ### as the seperator - - # <|im_start|>user - # What is this sign about?\n - # ### - # <|im_start|>assistant - # The sign is about bullying, and it is placed on a black background with a red background. - # ### - - CHAT_FORMAT = ( - "{% for message in messages %}" - # System message - "{% if message.role == 'system' %}" - "<|im_start|>system\n" - "{{ message.content }}\n" - "###\n" - "{% endif %}" - # User message - "{% if message.role == 'user' %}" - "<|im_start|>user\n" - "{% if message.content is string %}" - "{{ message.content }}" - "{% endif %}" - "{% if message.content is iterable %}" - "{% for content in message.content %}" - "{% if content.type == 'image_url' and content.image_url is string %}" - "{{ content.image_url }}" - "{% endif %}" - "{% if content.type == 'image_url' and content.image_url is mapping %}" - "{{ content.image_url.url }}" - "{% endif %}" - "{% endfor %}" - "{% for content in message.content %}" - "{% if content.type == 'text' %}" - "{{ content.text }}" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - "###\n" - "{% endif %}" - # Assistant message - "{% if message.role == 'assistant' %}" - "<|im_start|>assistant\n" - "{{ message.content }}" - "###\n" - "{% endif %}" - "{% endfor %}" - # Generation prompt - "{% if add_generation_prompt %}" - "<|im_start|>assistant\n" - "{% endif %}" - ) - - -class MoondreamChatHandler(MTMDChatHandler): - # Chat Format: - # f"\n\n{chat_history}Question: {question}\n\nAnswer:" - CHAT_FORMAT = ( - "{% for message in messages %}" - "{% if message.role == 'user' %}" - "{% if message.content is iterable %}" - # - "{% for content in message.content %}" - "{% if content.type == 'image_url' %}" - "{% if content.image_url is string %}" - "{{ content.image_url }}\n\n" - "{% endif %}" - "{% if content.image_url is mapping %}" - "{{ content.image_url.url }}\n\n" - "{% endif %}" - "{% endif %}" - "{% endfor %}" - # Question: - "{% for content in message.content %}" - "{% if content.type == 'text' %}" - "Question: {{ content.text }}\n\n" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - # Question: - "{% if message.content is string %}" - "Question: {{ message.content }}\n\n" - "{% endif %}" - "{% endif %}" - # Answer: - "{% if message.role == 'assistant' %}" - "Answer:{{ message.content }}\n\n" - "{% endif %}" - "{% endfor %}" - # Generation prompt - "{% if add_generation_prompt %}" - "Answer:" - "{% endif %}" - ) - - -class Llava16ChatHandler(MTMDChatHandler): - # Example prompt - # "DEFAULT_SYSTEM_MESSAGE + USER: \nWhat is shown in this image? ASSISTANT:" - - CHAT_FORMAT = ( - "{% for message in messages %}" - "{% if message.role == 'system' %}" - "{{ message.content }}" - "{% endif %}" - "{% if message.role == 'user' %}" - "{% if message.content is iterable %}" - # - "{% for content in message.content %}" - "{% if content.type == 'image_url' %}" - "{% if content.image_url is string %}" - "{{ content.image_url }}\n" - "{% endif %}" - "{% if content.image_url is mapping %}" - "{{ content.image_url.url }}\n" - "{% endif %}" - "{% endif %}" - "{% endfor %}" - # Question: - "{% for content in message.content %}" - "{% if content.type == 'text' %}" - "{{ content.text }}" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - # Question: - "{% if message.content is string %}" - "{{ message.content }}" - "{% endif %}" - "{% endif %}" - # Answer: - "{% if message.role == 'assistant' %}" - "{{ message.content }}" - "{% endif %}" - "{% endfor %}" - # Generation prompt - "{% if add_generation_prompt %}" - "Answer:" - "{% endif %}" - ) - - -class NanoLlavaChatHandler(MTMDChatHandler): - # Prompt Format - # The model follow the ChatML standard, however, without \n at the end of <|im_end|>: - - # <|im_start|>system - # Answer the question<|im_end|><|im_start|>user - # - # What is the picture about?<|im_end|><|im_start|>assistant - DEFAULT_SYSTEM_MESSAGE = "Answer the question" - - CHAT_FORMAT = ( - "{% for message in messages %}" - # System message - "{% if message.role == 'system' %}" - "<|im_start|>system\n" - "{{ message.content }}" - "<|im_end|>" - "{% endif %}" - # User message - "{% if message.role == 'user' %}" - "<|im_start|>user\n" - "{% if message.content is string %}" - "{{ message.content }}" - "{% endif %}" - "{% if message.content is iterable %}" - "{% for content in message.content %}" - "{% if content.type == 'image_url' and content.image_url is string %}" - "{{ content.image_url }}" - "{% endif %}" - "{% if content.type == 'image_url' and content.image_url is mapping %}" - "{{ content.image_url.url }}" - "{% endif %}" - "{% endfor %}" - "{% for content in message.content %}" - "{% if content.type == 'text' %}" - "{{ content.text }}" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - "<|im_end|>" - "{% endif %}" - # Assistant message - "{% if message.role == 'assistant' %}" - "<|im_start|>assistant\n" - "{{ message.content }}" - "<|im_end|>" - "{% endif %}" - "{% endfor %}" - # Generation prompt - "{% if add_generation_prompt %}" - "<|im_start|>assistant\n" - "{% endif %}" - ) - - -class Llama3VisionAlphaChatHandler(MTMDChatHandler): - # question = "" + q - - # prompt = f"<|start_header_id|>user<|end_header_id|>\n\n{question}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" - - CHAT_FORMAT = ( - "{% for message in messages %}" - "<|start_header_id|>" - "{% if message.role == 'user' %}" - "user<|end_header_id|>\n\n" - "{% if message.content is iterable %}" - # - "{% for content in message.content %}" - "{% if content.type == 'image_url' %}" - "{% if content.image_url is string %}" - "{{ content.image_url }}" - "{% endif %}" - "{% if content.image_url is mapping %}" - "{{ content.image_url.url }}" - "{% endif %}" - "{% endif %}" - "{% endfor %}" - # Question: - "{% for content in message.content %}" - "{% if content.type == 'text' %}" - "{{ content.text }}" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - # Question: - "{% if message.content is string %}" - "{{ message.content }}" - "{% endif %}" - "{% endif %}" - # Answer: - "{% if message.role == 'assistant' %}" - "assistant<|end_header_id|>\n\n" - "{{ message.content }}" - "{% endif %}" - "<|eot_id|>" - "{% endfor %}" - # Generation prompt - "{% if add_generation_prompt %}" - "<|start_header_id|>assistant<|end_header_id|>\n\n" - "{% endif %}" - ) - - -# alias -Llama3VisionAlpha = Llama3VisionAlphaChatHandler - - -class MiniCPMv26ChatHandler(MTMDChatHandler): - - CHAT_FORMAT = ( - "{% set image_count = namespace(value=0) %}" - "{% for message in messages %}" - "{% if loop.first and messages[0]['role'] != 'system' %}" - "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" - "{% endif %}" - "<|im_start|>{{ message['role'] }}\n" - "{% if message['content'] is iterable %}" - "{% for content in message['content'] %}" - "{% if content.type == 'image_url' %}" - "{% if content.image_url is string %}" - "{% set image_count.value = image_count.value + 1 %}" - "{{ image_count.value }}: {{ content.image_url }}" - "{% endif %}" - "{% if content.image_url is mapping %}" - "{% set image_count.value = image_count.value + 1 %}" - "{{ image_count.value }}: {{ content.image_url.url }}" - "{% endif %}" - "{% endif %}" - "{% endfor %}" - - "{% for content in message['content'] %}" - "{% if content.type == 'text' %}" - "{{ content.text }}" - "{% endif %}" - "{% endfor %}" - "{% endif %}" - "{% if message['content'] is string %}" - "{{ message['content'] }}" - "{% endif %}" - "<|im_end|>\n" - "{% endfor %}" - "{% if add_generation_prompt %}" - "<|im_start|>assistant\n" - "{% endif %}" - ) - - -class MiniCPMv45ChatHandler(MTMDChatHandler): - """ - Handler for MiniCPM-V 4.5 models. - - Supports: - - Multi-step tool calls with and XML tags. - - Integrated reasoning (thinking) process with tags. - - Specialized system prompt handling with tool definitions. - - Global image numbering for multi-image processing. - """ - - # Model specific control tokens - MINICPMV_BOS_TOKEN = "<|im_start|>" - MINICPMV_EOS_TOKEN = "<|im_end|>" - MINICPMV_PAD_TOKEN = "<|endoftext|>" - - # Image placeholder tags - MINICPMV_IMAGE_START_TOKEN = "" - MINICPMV_IMAGE_END_TOKEN = "" - MINICPMV_IMAGE_ID_START_TOKEN = "" - MINICPMV_IMAGE_ID_END_TOKEN = "" - - CHAT_FORMAT = ( - # --- 1. First System Message & Tools Definitions --- - "{%- if tools %}" - "{{- '" + MINICPMV_BOS_TOKEN + "system\\n' }}" - "{%- if messages[0].role == 'system' %}{{- messages[0].content + '\\n\\n' }}{%- endif %}" - "{{- '# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\n' }}" - "{{- 'You are provided with function signatures within XML tags:\\n' }}" - "{%- for tool in tools %}{{- '\\n' + (tool | tojson) }}{%- endfor %}" - "{{- '\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\"name\": , \"arguments\": }\\n" + MINICPMV_EOS_TOKEN + "\\n' }}" - "{%- elif messages[0].role == 'system' %}" - "{{- '" + MINICPMV_BOS_TOKEN + "system\\n' + messages[0].content + '" + MINICPMV_EOS_TOKEN + "\\n' }}" - "{%- endif %}" - - # --- 2. Message Stream Processing --- - "{% set image_count = namespace(value=0) %}" - "{%- for message in messages %}" - # --- Unified Role Handling (User, Assistant, and subsequent Systems) --- - "{%- if message.role in ['user', 'assistant'] or (message.role == 'system' and not loop.first) %}" - "{{- '" + MINICPMV_BOS_TOKEN + "' + message.role + '\\n' }}" - - "{%- set content = message.content %}" - "{%- if content is not string %}" - "{%- set ns = namespace(content_str='') %}" - "{%- for item in content %}" - # --- Explicit image_url type and value checking --- - "{%- if item.type == 'image_url' %}" - "{%- set image_url = item.image_url if item.image_url is string else item.image_url.url %}" - "{%- set image_count.value = image_count.value + 1 %}" - # Format: N: IMAGE_URL - "{%- set ns.content_str = ns.content_str + '' + (image_count.value | string) + ': ' + image_url + '' %}" - "{%- elif item.type == 'text' %}" - "{%- set ns.content_str = ns.content_str + item.text %}" - "{%- endif %}" - "{%- endfor %}" - "{%- set content = ns.content_str %}" - "{%- endif %}" - - "{{- content -}}" - - # Append tool_calls to assistant messages if they exist - "{%- if message.role == 'assistant' and message.tool_calls %}" - "{%- for tool_call in message.tool_calls %}" - "{%- set tc = tool_call.function if tool_call.function else tool_call %}" - "{{- '\\n\\n{\"name\": \"' + tc.name + '\", \"arguments\": ' }}" - "{{- tc.arguments if tc.arguments is string else tc.arguments | tojson }}" - "{{- '}\\n' }}" - "{%- endfor %}" - "{%- endif %}" - "{{- '" + MINICPMV_EOS_TOKEN + "\\n' }}" - - # --- Specialized Tool Response Handling --- - # Group consecutive tool responses under a single user-like block - "{%- elif message.role == 'tool' %}" - "{%- if loop.first or (messages[loop.index0 - 1].role != 'tool') %}" - "{{- '" + MINICPMV_BOS_TOKEN + "user' }}" - "{%- endif %}" - "{{- '\\n\\n' + message.content + '\\n' }}" - "{%- if loop.last or (messages[loop.index0 + 1].role != 'tool') %}" - "{{- '" + MINICPMV_EOS_TOKEN + "\\n' }}" - "{%- endif %}" - "{%- endif %}" - "{%- endfor %}" - - # --- 3. Generation Prompt --- - "{%- if add_generation_prompt %}" - "{{- '" + MINICPMV_BOS_TOKEN + "assistant\\n' }}" - # Handle thinking/reasoning block visibility based on configuration - "{%- if enable_thinking is defined and enable_thinking is false %}" - "{{- '\\n\\n\\n\\n' }}" - "{%- elif enable_thinking is defined and enable_thinking is true %}" - "{{- '\\n' }}" - "{%- endif %}" - "{%- endif %}" - ) - - def __init__(self, enable_thinking: bool = True, **kwargs): - """ - Initializes the MiniCPM-V 4.5 Handler. - - Args: - enable_thinking (bool): If True, model generates reasoning before the final answer. - **kwargs: Additional arguments for the base MTMDChatHandler. - """ - self.enable_thinking = enable_thinking - super().__init__(**kwargs) - - def __call__(self, **kwargs): - # Inject thinking control flag into the template - self.extra_template_arguments["enable_thinking"] = self.enable_thinking - - # Set stop token patch - kwargs['stop'] = [self.MINICPMV_EOS_TOKEN, self.MINICPMV_PAD_TOKEN] - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") - return super().__call__(**kwargs) - - -class MiniCPMV46ChatHandler(MTMDChatHandler): - """ - Handler for MiniCPM-V-4.6 models. - - Features: - - Aligned with official tokenizer_config.json special tokens. - - Custom `<|image_pad|>` and `<|video_pad|>` multimodal tokens. - - Integrated MTMD-style URL and Base64 injection for visual content. - - Specialized `` and `` block generation. - - Autonomously folds previous reasoning paths using `last_query_index`. - - Toggles `` block generation via `enable_thinking` (Defaults to False). - """ - - # Core tokens - MINICPM_BOS_TOKEN = "<|im_start|>" - MINICPM_EOS_TOKEN = "<|im_end|>" - MINICPM_PAD_TOKEN = "<|endoftext|>" - - # Vision tokens - MINICPM_VISION_BOS_TOKEN = "<|vision_start|>" - MINICPM_VISION_EOS_TOKEN = "<|vision_end|>" - MINICPM_IMAGE_TOKEN = "<|image_pad|>" - MINICPM_VIDEO_TOKEN = "<|video_pad|>" - - CHAT_FORMAT = ( - "{%- if enable_thinking is not defined -%}\n" - " {%- set enable_thinking = false -%}\n" - "{%- endif -%}\n" - "{%- macro render_content(content, is_system_content=false) -%}\n" - " {%- if content is string -%}\n" - " {{- content -}}\n" - " {%- elif content is iterable and content is not mapping -%}\n" - " {%- set ns = namespace(parts=[]) -%}\n" - " {%- for item in content -%}\n" - " {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}\n" - " {%- if is_system_content -%}\n" - " {{- raise_exception('System message cannot contain images.') -}}\n" - " {%- endif -%}\n" - " {%- set url_val = '' -%}\n" - " {%- if item.type == 'image_url' -%}\n" - " {%- set url_val = item.image_url if item.image_url is string else item.image_url.url -%}\n" - " {%- endif -%}\n" - " {%- set ns.parts = ns.parts + ['<|image_pad|>' + url_val] -%}\n" - # " {%- elif 'video' in item or 'video_url' in item or item.type == 'video' -%}\n" - # " {%- if is_system_content -%}\n" - # " {{- raise_exception('System message cannot contain videos.') -}}\n" - # " {%- endif -%}\n" - # " {%- set url_val = '' -%}\n" - # " {%- if item.type == 'video_url' -%}\n" - # " {%- set url_val = item.video_url if item.video_url is string else item.video_url.url -%}\n" - # " {%- endif -%}\n" - # " {%- set ns.parts = ns.parts + ['<|video_pad|>' + url_val] -%}\n" - " {%- elif 'text' in item -%}\n" - " {%- set ns.parts = ns.parts + [item.text] -%}\n" - " {%- else -%}\n" - " {{- raise_exception('Unexpected item type in content.') -}}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {{- ns.parts | join('\\n') -}}\n" - " {%- elif content is none or content is undefined -%}\n" - " {{- '' -}}\n" - " {%- else -%}\n" - " {{- raise_exception('Unexpected content type.') -}}\n" - " {%- endif -%}\n" - "{%- endmacro -%}\n" - "{%- if not messages %}\n" - " {{- raise_exception('No messages provided.') }}\n" - "{%- endif %}\n" - "{%- if tools and tools is iterable and tools is not mapping %}\n" - " {{- '<|im_start|>system\\n' }}\n" - " {{- '# Tools\\n\\nYou have access to the following functions:\\n\\n' }}\n" - " {%- for tool in tools %}\n" - " {{- '\\n' }}\n" - " {{- tool | tojson }}\n" - " {%- endfor %}\n" - " {{- '\\n' }}\n" - " {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n' }}\n" - " {%- if messages[0].role == 'system' %}\n" - " {%- set content = render_content(messages[0].content, true)|trim %}\n" - " {%- if content %}\n" - " {{- '\\n\\n' + content }}\n" - " {%- endif %}\n" - " {%- endif %}\n" - " {{- '<|im_end|>\\n' }}\n" - "{%- else %}\n" - " {%- if messages[0].role == 'system' %}\n" - " {%- set content = render_content(messages[0].content, true)|trim %}\n" - " {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n" - " {%- endif %}\n" - "{%- endif %}\n" - "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n" - "{%- for message in messages[::-1] %}\n" - " {%- set index = (messages|length - 1) - loop.index0 %}\n" - " {%- if ns.multi_step_tool and message.role == 'user' %}\n" - " {%- set content = render_content(message.content)|trim %}\n" - " {%- if not(content.startswith('') and content.endswith('')) %}\n" - " {%- set ns.multi_step_tool = false %}\n" - " {%- set ns.last_query_index = index %}\n" - " {%- endif %}\n" - " {%- endif %}\n" - "{%- endfor %}\n" - "{%- if ns.multi_step_tool %}\n" - " {{- raise_exception('No user query found in messages.') }}\n" - "{%- endif %}\n" - "{%- for message in messages %}\n" - " {%- set content = render_content(message.content)|trim %}\n" - " {%- if message.role == 'system' %}\n" - " {%- if not loop.first %}\n" - " {{- raise_exception('System message must be at the beginning.') }}\n" - " {%- endif %}\n" - " {%- elif message.role == 'user' %}\n" - " {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n" - " {%- elif message.role == 'assistant' %}\n" - " {%- set reasoning_content = '' %}\n" - " {%- if message.reasoning_content is string %}\n" - " {%- set reasoning_content = message.reasoning_content %}\n" - " {%- else %}\n" - " {%- if '' in content %}\n" - " {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n" - " {%- set content = content.split('')[-1].lstrip('\\n') %}\n" - " {%- endif %}\n" - " {%- endif %}\n" - " {%- set reasoning_content = reasoning_content|trim %}\n" - " {%- if loop.index0 > ns.last_query_index %}\n" - " {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n\\n' + content }}\n" - " {%- else %}\n" - " {{- '<|im_start|>' + message.role + '\\n' + content }}\n" - " {%- endif %}\n" - " {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n" - " {%- for tool_call in message.tool_calls %}\n" - " {%- if tool_call.function is defined %}\n" - " {%- set tool_call = tool_call.function %}\n" - " {%- endif %}\n" - " {%- if loop.first %}\n" - " {%- if content|trim %}\n" - " {{- '\\n\\n\\n\\n' }}\n" - " {%- else %}\n" - " {{- '\\n\\n' }}\n" - " {%- endif %}\n" - " {%- else %}\n" - " {{- '\\n\\n\\n' }}\n" - " {%- endif %}\n" - " {%- if tool_call.arguments is defined %}\n" - " {%- for args_name, args_value in tool_call.arguments|items %}\n" - " {{- '\\n' }}\n" - " {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n" - " {{- args_value }}\n" - " {{- '\\n\\n' }}\n" - " {%- endfor %}\n" - " {%- endif %}\n" - " {{- '\\n' }}\n" - " {%- endfor %}\n" - " {%- endif %}\n" - " {{- '<|im_end|>\\n' }}\n" - " {%- elif message.role == 'tool' %}\n" - " {%- if loop.previtem and loop.previtem.role != 'tool' %}\n" - " {{- '<|im_start|>user' }}\n" - " {%- endif %}\n" - " {{- '\\n\\n' }}\n" - " {{- content }}\n" - " {{- '\\n' }}\n" - " {%- if not loop.last and loop.nextitem.role != 'tool' %}\n" - " {{- '<|im_end|>\\n' }}\n" - " {%- elif loop.last %}\n" - " {{- '<|im_end|>\\n' }}\n" - " {%- endif %}\n" - " {%- else %}\n" - " {{- raise_exception('Unexpected message role.') }}\n" - " {%- endif %}\n" - "{%- endfor %}\n" - "{%- if add_generation_prompt %}\n" - " {{- '<|im_start|>assistant\\n' }}\n" - " {%- if enable_thinking is defined and enable_thinking is false %}\n" - " {{- '\\n\\n\\n\\n' }}\n" - " {%- else %}\n" - " {{- '\\n' }}\n" - " {%- endif %}\n" - "{%- endif %}\n" - ) - - def __init__(self, enable_thinking: bool = True, **kwargs): - """ - Initializes the MiniCPM-V-4.6 Handler. - - Args: - enable_thinking (bool): Controls whether to open a `` block for reasoning. - Defaults to False as per the standard template logic. - """ - self.enable_thinking = enable_thinking - super().__init__(**kwargs) - - def __call__(self, **kwargs): - # Inject the thinking variable into the Jinja environment - self.extra_template_arguments["enable_thinking"] = self.enable_thinking - - # MiniCPM uses standard <|im_end|> ChatML stop formatting - kwargs['stop'] = [self.MINICPM_PAD_TOKEN, self.MINICPM_EOS_TOKEN] - - if self.verbose: - print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") - - return super().__call__(**kwargs) - - -class Gemma3ChatHandler(MTMDChatHandler): - - GEMMA3_BOI_TOKEN = "" - GEMMA3_EOI_TOKEN = "" - GEMMA3_BOS_TOKEN = "" - GEMMA3_EOS_TOKEN = "" - - CHAT_FORMAT = ( - "{% if messages[0]['role'] == 'system' %}" - "{% set loop_messages = messages[1:] %}" - "{% if messages[0]['content'] is string %}" - "{% set first_user_prefix = messages[0]['content'] + '\n\n' %}" - "{% else %}" - "{% set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' %}" - "{% endif %}" - "{% else %}" - "{% set loop_messages = messages %}" - "{% set first_user_prefix = '' %}" - "{% endif %}" - - "{% for message in loop_messages %}" - "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}" - "{{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}" - "{% endif %}" - - "{% if message['role'] == 'assistant' %}" - "{% set role = 'model' %}" - "{% else %}" - "{% set role = message['role'] %}" - "{% endif %}" - - "{{ '' + role + '\n' + (first_user_prefix if loop.first else '') }}" - - "{% if message['content'] is string %}" - "{{ message['content'] | trim }}" - "{% elif message['content'] is iterable %}" - "{% for item in message['content'] %}" - "{% if item['type'] == 'image_url' and item['image_url'] is string %}" - "{{ '' + item['image_url'] + '' }}" - "{% elif item['type'] == 'image_url' and item['image_url'] is mapping %}" - "{{ '' + item['image_url']['url'] + '' }}" - "{% elif item['type'] == 'text' %}" - "{{ item['text'] | trim }}" - "{% endif %}" - "{% endfor %}" - "{% else %}" - "{{ raise_exception('Invalid content type') }}" - "{% endif %}" - - "\n" - "{% endfor %}" - - "{% if add_generation_prompt %}" - "model\n" - "{% endif %}" - ) - - -class Gemma4ChatHandler(MTMDChatHandler): - """ - Handler for Gemma 4 models. - - Note on `enable_thinking`: - The `enable_thinking` toggle is currently ONLY supported by Gemma4 31B and 26BA4B models. - It is NOT supported by Gemma4 E2B and E4B models. - - [Important Note for Audio Processing!] - It is recommended to use BF16 mmproj for Gemma4 E2B and E4B models. - Other quantizations are known to have degraded performance; - ref comment: https://github.com/ggml-org/llama.cpp/pull/21421#issuecomment-4230306463 - """ - - # The special token in Gemma 4 - GEMMA4_BOI_TOKEN = "<|image>" - GEMMA4_EOI_TOKEN = "" - GEMMA4_BOA_TOKEN = "<|audio>" - GEMMA4_EOA_TOKEN = "" - GEMMA4_BOS_TOKEN = "" - GEMMA4_EOS_TOKEN = "" - GEMMA4_SOT_TOKEN = "<|turn>" - GEMMA4_EOT_TOKEN = "" - GEMMA4_SOC_TOKEN = "<|channel>" - GEMMA4_EOC_TOKEN = "" - GEMMA4_STC_TOKEN = "<|tool_call>" - GEMMA4_ETC_TOKEN = "" - GEMMA4_STD_TOKEN = "<|tool>" - GEMMA4_ETD_TOKEN = "" - GEMMA4_STR_TOKEN = "<|tool_response>" - GEMMA4_ETR_TOKEN = "" - - CHAT_FORMAT = ( - "{%- macro format_parameters(properties, required, filter_keys=false) -%}\n" - " {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n" - " {%- set ns = namespace(found_first=false) -%}\n" - " {%- for key, value in properties | dictsort -%}\n" - " {%- set add_comma = false -%}\n" - " {%- if not filter_keys or key not in standard_keys -%}\n" - " {%- if ns.found_first %},{% endif -%}\n" - " {%- set ns.found_first = true -%}\n" - " {{ key }}:{\n" - " {%- if value['description'] -%}\n" - " description:<|\"|>{{ value['description'] }}<|\"|>\n" - " {%- set add_comma = true -%}\n" - " {%- endif -%}\n" - " {%- if value['type'] | upper == 'STRING' -%}\n" - " {%- if value['enum'] -%}\n" - " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" - " enum:{{ format_argument(value['enum']) }}\n" - " {%- endif -%}\n" - " {%- elif value['type'] | upper == 'ARRAY' -%}\n" - " {%- if value['items'] is mapping and value['items'] -%}\n" - " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" - " items:{\n" - " {%- set ns_items = namespace(found_first=false) -%}\n" - " {%- for item_key, item_value in value['items'] | dictsort -%}\n" - " {%- if item_value is not none -%}\n" - " {%- if ns_items.found_first %},{% endif -%}\n" - " {%- set ns_items.found_first = true -%}\n" - " {%- if item_key == 'properties' -%}\n" - " properties:{\n" - " {%- if item_value is mapping -%}\n" - " {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n" - " {%- endif -%}\n" - " }\n" - " {%- elif item_key == 'required' -%}\n" - " required:[\n" - " {%- for req_item in item_value -%}\n" - " <|\"|>{{- req_item -}}<|\"|>\n" - " {%- if not loop.last %},{% endif -%}\n" - " {%- endfor -%}\n" - " ]\n" - " {%- elif item_key == 'type' -%}\n" - " {%- if item_value is string -%}\n" - " type:{{ format_argument(item_value | upper) }}\n" - " {%- else -%}\n" - " type:{{ format_argument(item_value | map('upper') | list) }}\n" - " {%- endif -%}\n" - " {%- else -%}\n" - " {{ item_key }}:{{ format_argument(item_value) }}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " }\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- if value['nullable'] %}\n" - " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" - " nullable:true\n" - " {%- endif -%}\n" - " {%- if value['type'] | upper == 'OBJECT' -%}\n" - " {%- if value['properties'] is defined and value['properties'] is mapping -%}\n" - " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" - " properties:{\n" - " {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n" - " }\n" - " {%- elif value is mapping -%}\n" - " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" - " properties:{\n" - " {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}\n" - " }\n" - " {%- endif -%}\n" - " {%- if value['required'] -%}\n" - " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" - " required:[\n" - " {%- for item in value['required'] | default([]) -%}\n" - " <|\"|>{{- item -}}<|\"|>\n" - " {%- if not loop.last %},{% endif -%}\n" - " {%- endfor -%}\n" - " ]\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" - " type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - "{%- endmacro -%}\n" - "{%- macro format_function_declaration(tool_data) -%}\n" - " declaration:{{- tool_data['function']['name'] -}}{description:<|\"|>{{- tool_data['function']['description'] -}}<|\"|>\n" - " {%- set params = tool_data['function']['parameters'] -%}\n" - " {%- if params -%}\n" - " ,parameters:{\n" - " {%- if params.get('properties') -%}\n" - " properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n" - " {%- endif -%}\n" - " {%- if params.get('required') -%}\n" - " required:[\n" - " {%- for item in params['required'] -%}\n" - " <|\"|>{{- item -}}<|\"|>\n" - " {{- ',' if not loop.last -}}\n" - " {%- endfor -%}\n" - " ],\n" - " {%- endif -%}\n" - " {%- if params.get('type') -%}\n" - " type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- if 'response' in tool_data['function'] -%}\n" - " {%- set response_declaration = tool_data['function']['response'] -%}\n" - " ,response:{\n" - " {%- if response_declaration['description'] -%}\n" - " description:<|\"|>{{- response_declaration['description'] -}}<|\"|>,\n" - " {%- endif -%}\n" - " {%- if response_declaration['type'] | upper == 'OBJECT' -%}\n" - " type:<|\"|>{{- response_declaration['type'] | upper -}}<|\"|>}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " }\n" - "{%- endmacro -%}\n" - "{%- macro format_argument(argument, escape_keys=True) -%}\n" - " {%- if argument is string -%}\n" - " {{- '<|\"|>' + argument + '<|\"|>' -}}\n" - " {%- elif argument is boolean -%}\n" - " {{- 'true' if argument else 'false' -}}\n" - " {%- elif argument is mapping -%}\n" - " {{- '{' -}}\n" - " {%- set ns = namespace(found_first=false) -%}\n" - " {%- for key, value in argument | dictsort -%}\n" - " {%- if ns.found_first %},{% endif -%}\n" - " {%- set ns.found_first = true -%}\n" - " {%- if escape_keys -%}\n" - " {{- '<|\"|>' + key + '<|\"|>' -}}\n" - " {%- else -%}\n" - " {{- key -}}\n" - " {%- endif -%}\n" - " :{{- format_argument(value, escape_keys=escape_keys) -}}\n" - " {%- endfor -%}\n" - " {{- '}' -}}\n" - " {%- elif argument is sequence -%}\n" - " {{- '[' -}}\n" - " {%- for item in argument -%}\n" - " {{- format_argument(item, escape_keys=escape_keys) -}}\n" - " {%- if not loop.last %},{% endif -%}\n" - " {%- endfor -%}\n" - " {{- ']' -}}\n" - " {%- else -%}\n" - " {{- argument -}}\n" - " {%- endif -%}\n" - "{%- endmacro -%}\n" - "{%- macro strip_thinking(text) -%}\n" - " {%- set ns = namespace(result='') -%}\n" - " {%- for part in text.split('') -%}\n" - " {%- if '<|channel>' in part -%}\n" - " {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n" - " {%- else -%}\n" - " {%- set ns.result = ns.result + part -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {{- ns.result | trim -}}\n" - "{%- endmacro -%}\n" - "\n" - "{%- macro format_tool_response_block(tool_name, response) -%}\n" - " {{- '<|tool_response>' -}}\n" - " {%- if response is mapping -%}\n" - " {{- 'response:' + tool_name + '{' -}}\n" - " {%- for key, value in response | dictsort -%}\n" - " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" - " {%- if not loop.last %},{% endif -%}\n" - " {%- endfor -%}\n" - " {{- '}' -}}\n" - " {%- else -%}\n" - " {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}\n" - " {%- endif -%}\n" - " {{- '' -}}\n" - "{%- endmacro -%}\n" - "\n" - "{%- set ns = namespace(prev_message_type=None) -%}\n" - "{%- set loop_messages = messages -%}\n" - "{{- bos_token -}}\n" - "{#- Handle System/Tool Definitions Block -#}\n" - "{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n" - " {{- '<|turn>system\\n' -}}\n" - " {#- Inject Thinking token at the very top of the FIRST system turn -#}\n" - " {%- if enable_thinking is defined and enable_thinking -%}\n" - " {{- '<|think|>\\n' -}}\n" - " {%- set ns.prev_message_type = 'think' -%}\n" - " {%- endif -%}\n" - " {%- if messages[0]['role'] in ['system', 'developer'] -%}\n" - " {%- if messages[0]['content'] is string -%}\n" - " {{- messages[0]['content'] | trim -}}\n" - " {%- elif messages[0]['content'] is sequence -%}\n" - " {%- for item in messages[0]['content'] -%}\n" - " {{- item['text'] | trim + ' '-}}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - " {%- set loop_messages = messages[1:] -%}\n" - " {%- endif -%}\n" - " {%- if tools -%}\n" - " {%- for tool in tools %}\n" - " {{- '<|tool>' -}}\n" - " {{- format_function_declaration(tool) | trim -}}\n" - " {{- '' -}}\n" - " {%- endfor %}\n" - " {%- set ns.prev_message_type = 'tool' -%}\n" - " {%- endif -%}\n" - " {{- '\\n' -}}\n" - "{%- endif %}\n" - "\n" - "{#- Pre-scan: find last user message index for reasoning guard -#}\n" - "{%- set ns_turn = namespace(last_user_idx=-1) -%}\n" - "{%- for i in range(loop_messages | length) -%}\n" - " {%- if loop_messages[i]['role'] == 'user' -%}\n" - " {%- set ns_turn.last_user_idx = i -%}\n" - " {%- endif -%}\n" - "{%- endfor -%}\n" - "\n" - "{#- Loop through messages -#}\n" - "{%- for message in loop_messages -%}\n" - " {%- if message['role'] != 'tool' -%}\n" - " {%- set ns.prev_message_type = None -%}\n" - " {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n" - " {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n" - " {%- set prev_nt = namespace(role=None, found=false) -%}\n" - " {%- if loop.index0 > 0 -%}\n" - " {%- for j in range(loop.index0 - 1, -1, -1) -%}\n" - " {%- if not prev_nt.found -%}\n" - " {%- if loop_messages[j]['role'] != 'tool' -%}\n" - " {%- set prev_nt.role = loop_messages[j]['role'] -%}\n" - " {%- set prev_nt.found = true -%}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - " {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n" - " {%- if not continue_same_model_turn -%}\n" - " {{- '<|turn>' + role + '\\n' }}\n" - " {%- endif -%}\n" - "\n" - " {#- Render reasoning/reasoning_content as thinking channel -#}\n" - " {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n" - " {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n" - " {{- '<|channel>thought\\n' + thinking_text + '\\n' -}}\n" - " {%- endif -%}\n" - "\n" - " {%- if message.get('tool_calls') -%}\n" - " {%- for tool_call in message['tool_calls'] -%}\n" - " {%- set function = tool_call['function'] -%}\n" - " {{- '<|tool_call>call:' + function['name'] + '{' -}}\n" - " {%- if function['arguments'] is mapping -%}\n" - " {%- set ns_args = namespace(found_first=false) -%}\n" - " {%- for key, value in function['arguments'] | dictsort -%}\n" - " {%- if ns_args.found_first %},{% endif -%}\n" - " {%- set ns_args.found_first = true -%}\n" - " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" - " {%- endfor -%}\n" - " {%- elif function['arguments'] is string -%}\n" - " {{- function['arguments'] -}}\n" - " {%- endif -%}\n" - " {{- '}' -}}\n" - " {%- endfor -%}\n" - " {%- set ns.prev_message_type = 'tool_call' -%}\n" - " {%- endif -%}\n" - "\n" - " {%- set ns_tr_out = namespace(flag=false) -%}\n" - " {%- if message.get('tool_responses') -%}\n" - " {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n" - " {%- for tool_response in message['tool_responses'] -%}\n" - " {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n" - " {%- set ns_tr_out.flag = true -%}\n" - " {%- set ns.prev_message_type = 'tool_response' -%}\n" - " {%- endfor -%}\n" - " {%- elif message.get('tool_calls') -%}\n" - " {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}\n" - " {%- set ns_tool_scan = namespace(stopped=false) -%}\n" - " {%- for k in range(loop.index0 + 1, loop_messages | length) -%}\n" - " {%- if ns_tool_scan.stopped -%}\n" - " {%- elif loop_messages[k]['role'] != 'tool' -%}\n" - " {%- set ns_tool_scan.stopped = true -%}\n" - " {%- else -%}\n" - " {%- set follow = loop_messages[k] -%}\n" - " {#- Resolve tool_call_id to function name -#}\n" - " {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n" - " {%- for tc in message['tool_calls'] -%}\n" - " {%- if tc.get('id') == follow.get('tool_call_id') -%}\n" - " {%- set ns_tname.name = tc['function']['name'] -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {#- Handle content as string or content-parts array -#}\n" - " {%- set tool_body = follow.get('content') -%}\n" - " {%- if tool_body is string -%}\n" - " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n" - " {%- elif tool_body is sequence and tool_body is not string -%}\n" - " {%- set ns_txt = namespace(s='') -%}\n" - " {%- for part in tool_body -%}\n" - " {%- if part.get('type') == 'text' -%}\n" - " {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n" - " {%- for part in tool_body -%}\n" - " {%- if part.get('type') == 'image_url' -%}\n" - " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" - " {{- '<|image|>' + url_val -}}\n" - " {%- elif part.get('type') in ['audio_url', 'input_audio'] -%}\n" - " {%- if part.get('type') == 'audio_url' -%}\n" - " {%- set audio_val = part['audio_url'] if part['audio_url'] is string else part['audio_url']['url'] -%}\n" - " {{- '<|audio|>' + audio_val -}}\n" - " {%- elif part.get('type') == 'input_audio' -%}\n" - " {%- set audio_val = part['input_audio'] if part['input_audio'] is string else ('data:audio/' + part['input_audio']['format'] + ';base64,' + part['input_audio']['data']) -%}\n" - " {{- '<|audio|>' + audio_val -}}\n" - " {%- endif -%}\n" - # " {%- elif part.get('type') == 'video_url' -%}\n" - # " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" - # " {{- '<|video|>' + video_val -}}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- else -%}\n" - " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n" - " {%- endif -%}\n" - " {%- set ns_tr_out.flag = true -%}\n" - " {%- set ns.prev_message_type = 'tool_response' -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - "\n" - " {%- set captured_content -%}\n" - " {%- if message['content'] is string -%}\n" - " {%- if role == 'model' -%}\n" - " {{- strip_thinking(message['content']) -}}\n" - " {%- else -%}\n" - " {{- message['content'] | trim -}}\n" - " {%- endif -%}\n" - " {%- elif message['content'] is sequence -%}\n" - " {%- for item in message['content'] -%}\n" - " {%- if item['type'] == 'text' -%}\n" - " {%- if role == 'model' -%}\n" - " {{- strip_thinking(item['text']) -}}\n" - " {%- else -%}\n" - " {{- item['text'] | trim -}}\n" - " {%- endif -%}\n" - " {%- elif item['type'] == 'image_url' -%}\n" - " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" - " {{- '<|image|>' + url_val -}}\n" - " {%- set ns.prev_message_type = 'image' -%}\n" - " {%- elif item['type'] in ['audio_url', 'input_audio'] -%}\n" - " {%- if item['type'] == 'audio_url' -%}\n" - " {%- set audio_val = item['audio_url'] if item['audio_url'] is string else item['audio_url']['url'] -%}\n" - " {{- '<|audio|>' + audio_val -}}\n" - " {%- elif item['type'] == 'input_audio' -%}\n" - " {%- set audio_val = item['input_audio'] if item['input_audio'] is string else ('data:audio/' + item['input_audio']['format'] + ';base64,' + item['input_audio']['data']) -%}\n" - " {{- '<|audio|>' + audio_val -}}\n" - " {%- endif -%}\n" - " {%- set ns.prev_message_type = 'audio' -%}\n" - " {%- endif -%}\n" - # " {%- elif item['type'] == 'video_url' -%}\n" - # " {%- set video_val = item['video_url'] if item['video_url'] is string else item['video_url']['url'] -%}\n" - # " {{- '<|video|>' + video_val -}}\n" - # " {%- set ns.prev_message_type = 'video' -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - " {%- endset -%}\n" - "\n" - " {{- captured_content -}}\n" - " {%- set has_content = captured_content | trim | length > 0 -%}\n" - "\n" - " {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n" - " {{- '<|tool_response>' -}}\n" - " {%- elif not (ns_tr_out.flag and not has_content) -%}\n" - " {{- '\\n' -}}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - "{%- endfor -%}\n" - "\n" - "{%- if add_generation_prompt -%}\n" - " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n" - " {{- '<|turn>model\\n' -}}\n" - " {%- if not enable_thinking | default(false) -%}\n" - " {{- '<|channel>thought\\n' -}}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - "{%- endif -%}\n" - ) - - def __init__(self, enable_thinking: bool = True, **kwargs): - """ - Initializes the Gemma 4 Handler. - - Args: - enable_thinking (bool): Controls whether the <|think|> tag is injected and - manages <|channel>thought behavior. - Note: ONLY supported on Gemma4 31B and 26BA4B models. - NOT supported on Gemma4 E2B and E4B models. - """ - self.enable_thinking = enable_thinking - super().__init__(**kwargs) - - def __call__(self, **kwargs): - # Inject the thinking variable into the Jinja environment - self.extra_template_arguments["enable_thinking"] = self.enable_thinking - - # Set the stop token based on Gemma 4's format () - # generation_config.json: "eos_token_id": [1, 106, 50] - kwargs['stop'] = [self.GEMMA4_EOS_TOKEN, self.GEMMA4_EOT_TOKEN, self.GEMMA4_STR_TOKEN] - - if self.verbose: - print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") - - return super().__call__(**kwargs) - - -class GLM41VChatHandler(MTMDChatHandler): - # Note: Make sure the GGUF files of your converted model and mmproj are F16 or F32. - - GLM41V_EOS_TOKEN = "<|endoftext|>" - GLM41V_PAD_TOKEN = "<|endoftext|>" - GLM41V_IMAGE_START_TOKEN = "<|begin_of_image|>" - GLM41V_IMAGE_END_TOKEN = "<|end_of_image|>" - - CHAT_FORMAT = ( - "[gMASK]\n" - "{%- for msg in messages -%}" - "{%- if msg.role == 'system' -%}" - "<|system|>\n{{ msg.content }}{{ GLM41V_EOS_TOKEN }}" - "{%- elif msg.role == 'user' -%}" - "<|user|>\n" - "{%- if msg.content is string -%}" - "{{ msg.content }}" - "{%- else -%}" - "{%- for item in msg.content -%}" - "{%- if item.type == 'image_url' or 'image_url' in item -%}" - "<|begin_of_image|>" - "{%- if item.image_url is string -%}" - "{{- item.image_url -}}" - "{%- else -%}" - "{{- item.image_url.url -}}" - "{%- endif -%}" - "<|end_of_image|>" - "{%- elif item.type == 'text' -%}" - "{{ item.text }}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}{{ GLM41V_EOS_TOKEN }}" - "{%- elif msg.role == 'assistant' -%}" - "{%- if msg.metadata -%}" - "<|assistant|>{{ msg.metadata }}\n{{ msg.content }}{{ GLM41V_EOS_TOKEN }}" - "{%- else -%}" - "<|assistant|>\n{{ msg.content }}{{ GLM41V_EOS_TOKEN }}" - "{%- endif -%}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- if add_generation_prompt -%}" - "<|assistant|>\n" - "{%- endif -%}" - ) - - def __call__(self, **kwargs): - self.extra_template_arguments["GLM41V_EOS_TOKEN"] = self.GLM41V_EOS_TOKEN - # https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking/blob/main/generation_config.json - stop_tokens = [self.GLM41V_EOS_TOKEN, "<|user|>", "<|observation|>", ""] # Stop token patch - kwargs['stop'] = stop_tokens - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix} - Start processing") - - # Use parent implementation - return super().__call__(**kwargs) - - -class GLM46VChatHandler(MTMDChatHandler): - GLM46V_EOS_TOKEN = "<|endoftext|>" - GLM46V_PAD_TOKEN = "<|endoftext|>" - GLM46V_IMAGE_START_TOKEN = "<|begin_of_image|>" - GLM46V_IMAGE_END_TOKEN = "<|end_of_image|>" - - CHAT_FORMAT = ( - "[gMASK]" - "{%- if tools -%}" - "<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n" - "You are provided with function signatures within XML tags:\n\n" - "{%- for tool in tools -%}" - "{{ tool | tojson(ensure_ascii=False) }}\n" - "{%- endfor -%}" - "\n\nFor each function call, output the function name and arguments within the following XML format:\n" - "{function-name}\n{arg-key-1}\n{arg-value-1}\n...\n" - "{%- endif -%}" - - "{%- for m in messages -%}" - "{%- if m.role == 'system' -%}" - "<|system|>\n{{ m.content }}" - "{%- elif m.role == 'user' -%}" - "<|user|>\n" - "{%- if m.content is string -%}" - "{{ m.content }}" - "{%- else -%}" - "{%- for item in m.content -%}" - "{%- if item.type == 'image_url' or 'image_url' in item -%}" - "<|begin_of_image|>" - "{%- if item.image_url is string -%}" - "{{- item.image_url -}}" - "{%- else -%}" - "{{- item.image_url.url -}}" - "{%- endif -%}" - "<|end_of_image|>" - "{%- elif item.type == 'text' -%}" - "{{ item.text }}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - # If enable_thinking is disabled, insert `/nothink` according to the source code logic. - "{{ '/nothink' if not enable_thinking else '' }}" - "{%- elif m.role == 'assistant' -%}" - "<|assistant|>" - "{%- if enable_thinking -%}" - "{%- set reasoning = m.reasoning_content if m.reasoning_content is string else '' -%}" - "\n{{ reasoning.strip() }}" - "{%- else -%}" - "\n" - "{%- endif -%}" - "{{ '\n' + m.content.strip() if m.content.strip() else '' }}" - "{%- endif -%}" - "{{ GLM46V_EOS_TOKEN }}" - "{%- endfor -%}" - - "{%- if add_generation_prompt -%}" - "<|assistant|>\n" - "{{ '' if enable_thinking else '\n' }}" - "{%- endif -%}" - ) - - def __init__(self, enable_thinking: bool = True, **kwargs): - """ - GLM-4.6V Handler - Parameters: - - enable_thinking (bool): Whether to enable the model's think process. The default is True. - """ - self.enable_thinking = enable_thinking - super().__init__(**kwargs) - - def __call__(self, **kwargs): - self.extra_template_arguments["enable_thinking"] = self.enable_thinking - self.extra_template_arguments["GLM46V_EOS_TOKEN"] = self.GLM46V_EOS_TOKEN - - # https://huggingface.co/zai-org/GLM-4.6V-Flash/blob/main/generation_config.json - kwargs['stop'] = [self.GLM46V_EOS_TOKEN, "<|user|>", "<|observation|>", "<|code_middle|>"] # Stop token patch - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") - - return super().__call__(**kwargs) - - -class GraniteDoclingChatHandler(MTMDChatHandler): - """ - Handler for Granite-Docling models. - - Format(512x512): Content - - Note(JamePeng): The GGUF files for Model and MMPROJ should be BF16 version !!! - Since the model does not have special tokens for the start and end of an image, - it is recommended to process only one image at a time. - You can iterate through the images individually for recognition. - - """ - GRANITE_BOS_TOKEN = "<|start_of_role|>" - GRANITE_EOS_TOKEN = "<|end_of_text|>" - GRANITE_PAD_TOKEN = "<|end_of_text|>" - GRANITE_IMAGE_TOKEN = "" - - CHAT_FORMAT = ( - "{%- for message in messages -%}" - "{{- '<|start_of_role|>' + message['role'] + '<|end_of_role|>' -}}" - "{%- if message['content'] is string -%}" - "{{- message['content'] -}}" - "{%- else -%}" - "{%- for part in message['content'] -%}" - "{%- if part['type'] == 'text' -%}" - "{{- part['text'] -}}" - "{%- elif part['type'] == 'image_url' -%}" - "{%- if part.image_url is string -%}" - "{{- part.image_url -}}" - "{%- else -%}" - "{{- part.image_url.url -}}" - "{%- endif -%}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - "{{- '<|end_of_text|>\n' -}}" - "{%- endfor -%}" - "{%- if add_generation_prompt -%}" - "{{- '<|start_of_role|>assistant' -}}" - # Support the 'controls' parameter if present in generation arguments - "{%- if controls -%}{{- ' ' + controls | tojson() -}}{%- endif -%}" - "{{- '<|end_of_role|>' -}}" - "{%- endif -%}" - ) - - def __init__(self, controls: dict = None, **kwargs): - """ - Granite-Docling Handler - Args: - controls (dict, optional): Operational parameters passed to the assistant role. - - The 'controls' parameter is used to guide the model's behavior or output format. - Common examples for 'controls' include: - - Document Parsing: {"mode": "document_parsing", "format": "json"} - """ - self.controls = controls - super().__init__(**kwargs) - - def __call__(self, **kwargs): - # Inject controls into the template environment - self.extra_template_arguments["controls"] = self.controls - self.DEFAULT_SYSTEM_MESSAGE = None - kwargs['stop'] = [self.GRANITE_EOS_TOKEN] - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix} - Start processing") - - - return super().__call__(**kwargs) - - -class LFM2VLChatHandler(MTMDChatHandler): - LFM2VL_BOS_TOKEN = "<|startoftext|>" - LFM2VL_EOS_TOKEN = "<|im_end|>" - LFM2VL_IMAGE_START_TOKEN = "<|image_start|>" - LFM2VL_IMAGE_END_TOKEN = "<|image_end|>" - - CHAT_FORMAT = ( - "{%- for message in messages -%}" - "{{ '<|im_start|>' + message['role'] + '\n' }}" - "{%- if message['content'] is string -%}" - "{{ message['content'] }}" - "{%- else -%}" - "{%- for content in message['content'] -%}" - "{%- if 'image_url' in content -%}" - "{%- if content.image_url is string -%}" - "<|image_start|>{{ content.image_url }}<|image_end|>" - "{%- else -%}" - "<|image_start|>{{ content.image_url.url }}<|image_end|>" - "{%- endif -%}" - "{%- elif content['type'] == 'text' -%}" - "{{ content['text'] }}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - "{{ '<|im_end|>\n' }}" - "{%- endfor -%}" - "{%- if add_generation_prompt -%}" - "{{ '<|im_start|>assistant\n' }}" - "{%- endif -%}" - ) - - def __init__(self, image_min_tokens: int = -1, image_max_tokens: int = -1, **kwargs): - """ - LFM2-VL Handler - LiquidAI officially recommends configuring LFM2-VL with the following Vision parameters: min_image_tokens=64, max_image_tokens=256 - """ - self.image_min_tokens = image_min_tokens - self.image_max_tokens = image_max_tokens - super().__init__(image_min_tokens=self.image_min_tokens, image_max_tokens=self.image_max_tokens, **kwargs) - - def __call__(self, **kwargs): - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix} - Start processing") - - return super().__call__(**kwargs) - - -class LFM25VLChatHandler(MTMDChatHandler): - """ - Handler for LFM2.5-VL multimodal models. - - Note(JamePeng): The suggestion is to compress the input image to 512x512 pixels to achieve native resolution processing. - """ - # Aligned with LFM2.5-VL tokenizer_config - LFM25VL_BOS_TOKEN = "<|startoftext|>" - LFM25VL_EOS_TOKEN = "<|im_end|>" - LFM25VL_PAD_TOKEN = "<|pad|>" - - # Image specific tokens - LFM25VL_IMAGE_TOKEN = "" - LFM25VL_IMAGE_START_TOKEN = "<|image_start|>" - LFM25VL_IMAGE_END_TOKEN = "<|image_end|>" - LFM25VL_IMAGE_THUMBNAIL = "<|img_thumbnail|>" - - CHAT_FORMAT = ( - "{{- bos_token -}}\n" - "{%- set keep_past_thinking = keep_past_thinking | default(false) -%}\n" - "{%- set ns = namespace(system_prompt='', content='') -%}\n" - "{%- if messages[0]['role'] == 'system' -%}\n" - " {%- set ns.system_prompt = messages[0]['content'] -%}\n" - " {%- set messages = messages[1:] -%}\n" - "{%- endif -%}\n" - "{%- if tools -%}\n" - " {%- set ns.system_prompt = ns.system_prompt + ('\\n' if ns.system_prompt else '') + 'List of tools: [' -%}\n" - " {%- for tool in tools -%}\n" - " {%- if tool is not string -%}\n" - " {%- set tool = tool | tojson -%}\n" - " {%- endif -%}\n" - " {%- set ns.system_prompt = ns.system_prompt + tool -%}\n" - " {%- if not loop.last -%}\n" - " {%- set ns.system_prompt = ns.system_prompt + ', ' -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- set ns.system_prompt = ns.system_prompt + ']' -%}\n" - "{%- endif -%}\n" - "{%- if ns.system_prompt -%}\n" - " {{- '<|im_start|>system\\n' + ns.system_prompt + '<|im_end|>\\n' -}}\n" - "{%- endif -%}\n" - "{%- set ns.last_assistant_index = -1 -%}\n" - "{%- for message in messages -%}\n" - " {%- if message['role'] == 'assistant' -%}\n" - " {%- set ns.last_assistant_index = loop.index0 -%}\n" - " {%- endif -%}\n" - "{%- endfor -%}\n" - "{%- for message in messages -%}\n" - " {{- '<|im_start|>' + message['role'] + '\\n' -}}\n" - " {%- set content = message['content'] -%}\n" - " {%- if content is not string -%}\n" - " {%- set ns.content = '' -%}\n" - " {#- MTMD-style Multimodal Injection (Audio stripped for VL model) -#}\n" - " {%- for item in content -%}\n" - " {%- if item['type'] == 'image_url' -%}\n" - " {%- set img_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" - " {%- set ns.content = ns.content + img_val -%}\n" - " {%- elif item['type'] == 'text' -%}\n" - " {%- set ns.content = ns.content + item['text'] -%}\n" - " {%- else -%}\n" - " {%- set ns.content = ns.content + (item | tojson) -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- set content = ns.content -%}\n" - " {%- endif -%}\n" - " {%- if message['role'] == 'assistant' and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}\n" - " {%- if '' in content -%}\n" - " {%- set content = content.split('')[-1] | trim -%}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {{- content + '<|im_end|>\\n' -}}\n" - "{%- endfor -%}\n" - "{%- if add_generation_prompt -%}\n" - " {{- '<|im_start|>assistant\\n' -}}\n" - "{%- endif -%}\n" - ) - - def __init__(self, keep_past_thinking: bool = False, **kwargs): - self.keep_past_thinking = keep_past_thinking - super().__init__(**kwargs) - - - def __call__(self, **kwargs): - if self.image_min_tokens > 256: - if self.verbose: - print(f"{self.log_prefix}: For LFM2.5-VL, using values higher than 256 for `image_min_tokens` could cause errors. Please reset it to between 64 and 256.") - self.image_min_tokens = -1 - - self.extra_template_arguments["keep_past_thinking"] = self.keep_past_thinking - - kwargs['stop'] = [self.LFM25VL_EOS_TOKEN] - - if self.verbose: - print(f"{self.log_prefix}(keep_past_thinking={self.keep_past_thinking}) - Start processing") - return super().__call__(**kwargs) - - -class PaddleOCRChatHandler(MTMDChatHandler): - """ - Handler for PaddleOCR 1.5/1.6 multimodal models. - """ - - PADDLEOCR_CLS_TOKEN = "<|begin_of_sentence|>" - PADDLEOCR_BOS_TOKEN = "" - PADDLEOCR_EOS_TOKEN = "" - PADDLEOCR_SEP_TOKEN = "<|end_of_sentence|>" - PADDLEOCR_IMAGE_BOS_TOKEN = "<|IMAGE_START|>" - PADDLEOCR_IMAGE_EOS_TOKEN = "<|IMAGE_END|>" - - CHAT_FORMAT = ( - "{%- if not add_generation_prompt is defined -%}{%- set add_generation_prompt = true -%}{%- endif -%}" - "{%- if not cls_token is defined -%}{%- set cls_token = '" + PADDLEOCR_CLS_TOKEN + "' -%}{%- endif -%}" - "{%- if not eos_token is defined -%}{%- set eos_token = '" + PADDLEOCR_EOS_TOKEN + "' -%}{%- endif -%}" - - "{{- cls_token -}}" - "{%- for message in messages -%}" - "{%- if message['role'] == 'user' -%}" - "{{- 'User: ' -}}" - - # Robust parsing: Check if content is string or list - "{%- if message['content'] is string -%}" - "{{- message['content'] -}}" - "{%- else -%}" - # Pass 1: Render all images first - "{%- for content in message['content'] -%}" - "{%- if content['type'] == 'image_url' and 'image_url' in content -%}" - "{{- '<|IMAGE_START|>' -}}" - "{%- if content.image_url is string -%}" - "{{- content.image_url -}}" - "{%- else -%}" - "{{- content.image_url.url -}}" - "{%- endif -%}" - "{{- '<|IMAGE_END|>' -}}" - "{%- endif -%}" - "{%- endfor -%}" - - # Pass 2: Render all text second - "{%- for content in message['content'] -%}" - "{%- if content['type'] == 'text' -%}" - "{{- content['text'] -}}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - "{{- '\\n' -}}" - - "{%- elif message['role'] == 'assistant' -%}" - "{{- 'Assistant:\\n' -}}" - "{%- if message['content'] is string -%}" - "{{- message['content'] -}}" - "{%- else -%}" - "{%- for content in message['content'] -%}" - "{%- if content['type'] == 'text' -%}" - "{{- content['text'] -}}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - "{{- eos_token -}}" - - "{%- elif message['role'] == 'system' -%}" - "{%- if message['content'] is string -%}" - "{{- message['content'] + '\\n' -}}" - "{%- else -%}" - "{%- for content in message['content'] -%}" - "{%- if content['type'] == 'text' -%}" - "{{- content['text'] + '\\n' -}}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - "{%- endif -%}" - "{%- endfor -%}" - - "{%- if add_generation_prompt -%}" - "{{- 'Assistant:\\n' -}}" - "{%- endif -%}" - ) - - def __init__( - self, - image_min_tokens: int = -1, - image_max_tokens: int = -1, - **kwargs - ): - self.image_min_tokens = image_min_tokens - self.image_max_tokens = image_max_tokens - super().__init__( - image_min_tokens=self.image_min_tokens, - image_max_tokens=self.image_max_tokens, - **kwargs - ) - - def __call__(self, **kwargs): - # Set the specific stop token defined in the PaddleOCR template - kwargs['stop'] = [self.PADDLEOCR_EOS_TOKEN] - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix} - Start processing") - - return super().__call__(**kwargs) - - -class Qwen25VLChatHandler(MTMDChatHandler): - - QWEN25_VL_BOS_TOKEN = "<|endoftext|>" - QWEN25_VL_PAD_TOKEN = "<|endoftext|>" - QWEN25_VL_EOS_TOKEN = "<|im_end|>" - - CHAT_FORMAT = ( - "{% set image_count = namespace(value=0) %}" - "{% for message in messages %}" - "{% if loop.first and message['role'] != 'system' %}" - "<|im_start|>system\n" - "{{ self.DEFAULT_SYSTEM_MESSAGE }}<|im_end|>\n" - "{% endif %}" - "<|im_start|>{{ message['role'] }}\n" - "{% if message['content'] is string %}" - "{{ message['content'] }}<|im_end|>\n" - "{% else %}" - "{% for content in message['content'] %}" - "{% if content['type'] == 'image_url' %}" - "{% if content.image_url is string %}" - "{% set image_count.value = image_count.value + 1 %}" - "Picture {{ image_count.value }}: <|vision_start|> {{ content.image_url }} <|vision_end|>" - "{% else %}" - "{% set image_count.value = image_count.value + 1 %}" - "Picture {{ image_count.value }}: <|vision_start|> {{ content.image_url.url }} <|vision_end|>" - "{% endif %}" - "{% elif content['type'] == 'text' %}" - "{{ content['text'] }}" - "{% endif %}" - "{% endfor %}" - "<|im_end|>\n" - "{% endif %}" - "{% endfor %}" - "<|im_start|>assistant\n" - ) - - def __call__(self, **kwargs): - kwargs['stop'] = [self.QWEN25_VL_EOS_TOKEN, self.QWEN25_VL_PAD_TOKEN] - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix} - Start processing") - - # Use parent implementation - return super().__call__(**kwargs) - -class Qwen3ASRChatHandler(MTMDChatHandler): - """ - Handler for Qwen 3 ASR (Automatic Speech Recognition) models. - - Features: - - Highly specialized for Speech-to-Text tasks. - - Aggregates all system text into a single cohesive system block. - - Drops user text entirely, extracting ONLY audio data into a unified user turn. - - Wraps audio with <|audio_start|><|audio_pad|>[DATA]<|audio_end|>. - - Integrated MTMD-style URL and Base64 injection for input_audio and audio_url. - """ - - DEFAULT_SYSTEM_MESSAGE = """ - You are an advanced multilingual Speech-to-Text model. Accurately transcribe the audio into text in its original spoken language. - You should ignore background noise, filler words, and stutters where possible, and format the final output with correct grammar and capitalization. - """ - - QWEN3_ASR_BOS_TOKEN = "<|im_start|>" - QWEN3_ASR_PAD_TOKEN = "<|endoftext|>" - QWEN3_ASR_EOS_TOKEN = "<|im_end|>" - - - QWEN3_ASR_AUDIO_BOS_TOKEN = "<|audio_start|>" - QWEN3_ASR_AUDIO_PAD_TOKEN = "<|audio_pad|>" - QWEN3_ASR_AUDIO_EOS_TOKEN = "<|audio_end|>" - - CHAT_FORMAT = ( - "{%- set ns = namespace(system_text='') -%}\n" - "{%- for m in messages -%}\n" - " {%- if m.role == 'system' -%}\n" - " {%- if m.content is string -%}\n" - " {%- set ns.system_text = ns.system_text + m.content -%}\n" - " {%- else -%}\n" - " {%- for c in m.content -%}\n" - " {%- if c.type == 'text' and (c.text is defined) -%}\n" - " {%- set ns.system_text = ns.system_text + c.text -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - "{%- endfor -%}\n" - "\n" - "{%- set ns2 = namespace(audio_tokens='') -%}\n" - "{%- for m in messages -%}\n" - " {%- if m.content is not string -%}\n" - " {%- for c in m.content -%}\n" - " {%- if c.type == 'audio' or ('audio' in c) or ('audio_url' in c) or c.type == 'input_audio' -%}\n" - " {#- MTMD Audio Injection -#}\n" - " {%- set audio_val = '' -%}\n" - " {%- if c.type == 'audio_url' or 'audio_url' in c -%}\n" - " {%- set audio_val = c.audio_url if c.audio_url is string else c.audio_url.url -%}\n" - " {%- elif c.type == 'input_audio' or 'input_audio' in c -%}\n" - " {%- set audio_val = c.input_audio if c.input_audio is string else ('data:audio/' + c.input_audio.format + ';base64,' + c.input_audio.data) -%}\n" - " {%- endif -%}\n" - " {%- set ns2.audio_tokens = ns2.audio_tokens + '<|audio_start|><|audio_pad|>' + audio_val + '<|audio_end|>' -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - "{%- endfor -%}\n" - "\n" - "{{- '<|im_start|>system\\n' + (ns.system_text if ns.system_text is string else '') + '<|im_end|>\\n' -}}\n" - "{{- '<|im_start|>user\\n' + ns2.audio_tokens + '<|im_end|>\\n' -}}\n" - "{%- if add_generation_prompt -%}\n" - " {{- '<|im_start|>assistant\\n' -}}\n" - "{%- endif -%}\n" - ) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def __call__(self, **kwargs): - # Qwen3 models universally use `<|endoftext|>` and `<|im_end|>` as the stop token - kwargs['stop'] = [self.QWEN3_ASR_AUDIO_PAD_TOKEN, self.QWEN3_ASR_AUDIO_EOS_TOKEN] - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix} - Start processing Qwen3-ASR (Audio Only)") - - return super().__call__(**kwargs) - -class Qwen3VLChatHandler(MTMDChatHandler): - - QWEN3_VL_BOS_TOKEN = "<|endoftext|>" - QWEN3_VL_PAD_TOKEN = "<|endoftext|>" - QWEN3_VL_EOS_TOKEN = "<|im_end|>" - - CHAT_FORMAT = ( - "{{- '<|im_start|>system\n' -}}" - "{%- if messages[0].content is string and messages[0].role == 'system' -%}" - "{{- messages[0].content -}}" - "{%- elif messages[0].role == 'system' -%}" - "{%- if 'text' in messages[0].content -%}" - "{{- messages[0].content.text -}}" - "{%- else -%}" - "{{- 'You are a helpful assistant.' -}}" - "{%- endif -%}" - "{%- endif -%}" - "{%- if tools -%}" - "{{- '\n\n' -}}" - "{{- '# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n' -}}" - "{%- for tool in tools -%}" - "{{- '\n' -}}" - "{{- tool | tojson -}}" - "{%- endfor -%}" - "{{- '\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n\n\nYou can also return a response for the user alongside a function call:\nRESPONSE FOR THE USER HERE\n\n{\"name\": , \"arguments\": }\n' -}}" - "{%- endif -%}" - "{{- '<|im_end|>\n' -}}" - "{%- set image_count = namespace(value=0) -%}" - #"{%- set video_count = namespace(value=0) -%}" - "{%- for message in messages -%}" - "{%- if message.role == 'tool' -%}" - "{{- '<|im_start|>user\n\n' -}}" - "{%- elif message.role != 'system' -%}" - "{{- '<|im_start|>' + message.role + '\n' -}}" - "{%- endif -%}" - "{%- if message.content is string and message.role != 'system' -%}" - "{{- message.content -}}" - "{%- elif message.role != 'system' -%}" - "{%- for content in message.content -%}" - "{%- if 'image_url' in content -%}" - "{%- set image_count.value = image_count.value + 1 -%}" - "{%- if add_vision_id -%}" - "{{- 'Picture ' -}}" - "{{- image_count.value | string -}}" - "{{- ': ' -}}" - "{%- endif -%}" - "{{- '<|vision_start|>' -}}" - "{%- if content.image_url is string -%}" - "{{- content.image_url -}}" - "{%- else -%}" - "{{- content.image_url.url -}}" - "{%- endif -%}" - "{{- '<|vision_end|>' -}}" - "{%- endif -%}" - # Video not supported yet - "{%- if 'text' in content -%}" - "{{- content.text -}}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- endif -%}" - "{%- if message.role == 'assistant' -%}" - "{%- if message.tool_calls -%}" - "{%- for tool_call in message.tool_calls -%}" - "{%- if (loop.first and message.content) or (not loop.first) -%}" - "{{- '\n' -}}" - "{%- endif -%}" - "{%- if tool_call.function -%}" - "{%- set tool_call = tool_call.function -%}" - "{%- endif -%}" - "{{- '\n{\"name\": \"' + tool_call.name + '\", \"arguments\": ' -}}" - "{%- if tool_call.arguments is string -%}" - "{{- tool_call.arguments -}}" - "{%- else -%}" - "{{- tool_call.arguments | tojson -}}" - "{%- endif -%}" - "{{- '}\n' -}}" - "{%- endfor -%}" - "{%- endif -%}" - "{%- elif message.role == 'tool' -%}" - "{{- '' -}}" - "{%- endif -%}" - "{%- if message.role != 'system' -%}" - "{{- '<|im_end|>\n' -}}" - "{%- endif -%}" - "{%- endfor -%}" - "{%- if add_generation_prompt -%}" - "{{- '<|im_start|>assistant\n' -}}" - "{%- if force_reasoning -%}" - "{{- '\n' -}}" - "{%- endif -%}" - "{%- endif -%}" - ) - - def __init__( - self, - force_reasoning: bool = False, - add_vision_id: bool = True, - **kwargs, - ): - """ - Parameters: - - force_reasoning (bool): - - True: Force the reasoning in the model by adding to the chat template. - - False (default): Don't force the reasoning. - - add_vision_id (bool): - - True (default): Count all the images. Recommended for multi-image. - - False: Doesn't count the images. Can save tokens with single-image. - """ - super().__init__(**kwargs) - self.force_reasoning = force_reasoning - self.extra_template_arguments["force_reasoning"] = force_reasoning - self.extra_template_arguments["add_vision_id"] = add_vision_id - - def __call__(self, **kwargs): - kwargs['stop'] = [self.QWEN3_VL_EOS_TOKEN, self.QWEN3_VL_PAD_TOKEN] - - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix}(force_reasoning={self.force_reasoning}) - Start processing") - - # Use parent implementation - return super().__call__(**kwargs) - -class Qwen35ChatHandler(MTMDChatHandler): - """ - Handler for Qwen3.5/Qwen3.6 models. - """ - CHAT_FORMAT = ( - "{%- set image_count = namespace(value=0) -%}" - "{%- set video_count = namespace(value=0) -%}" - "{%- macro render_content(content, do_vision_count, is_system_content=false) -%}" - " {%- if content is string -%}" - " {{- content -}}" - " {%- elif content is iterable and content is not mapping -%}" - " {%- for item in content -%}" - " {%- if 'image_url' in item or item.type == 'image_url' -%}" - " {%- if is_system_content -%}" - " {{- raise_exception('System message cannot contain images.') -}}" - " {%- endif -%}" - " {%- if do_vision_count -%}" - " {%- set image_count.value = image_count.value + 1 -%}" - " {%- endif -%}" - " {%- if add_vision_id -%}" - " {{- 'Picture ' -}}" - " {{- image_count.value | string -}}" - " {{- ': ' -}}" - " {%- endif -%}" - " {{- '<|vision_start|>' -}}" - " {%- if item.image_url is string -%}" - " {{- item.image_url -}}" - " {%- else -%}" - " {{- item.image_url.url -}}" - " {%- endif -%}" - " {{- '<|vision_end|>' -}}" - " {%- elif 'video' in item -%}" - " {{- raise_exception('llama.cpp does not currently support video.') -}}" # Video not supported, raise exception - " {%- if is_system_content -%}" - " {{- raise_exception('System message cannot contain videos.') -}}" - " {%- endif -%}" - " {%- if do_vision_count -%}" - " {%- set video_count.value = video_count.value + 1 -%}" - " {%- endif -%}" - " {%- if add_vision_id -%}" - " {{- 'Video ' ~ video_count.value ~ ': ' -}}" - " {%- endif -%}" - " {{- '<|vision_start|>' -}}" - " {{- item.video -}}" - " {{- '<|vision_end|>' -}}" - " {%- elif 'text' in item -%}" - " {{- item.text -}}" - " {%- else -%}" - " {{- raise_exception('Unexpected item type in content.') -}}" - " {%- endif -%}" - " {%- endfor -%}" - " {%- elif content is none or content is undefined -%}" - " {{- '' -}}" - " {%- else -%}" - " {{- raise_exception('Unexpected content type.') -}}" - " {%- endif -%}" - "{%- endmacro -%}" - "{%- if not messages -%}" - " {{- raise_exception('No messages provided.') -}}" - "{%- endif -%}" - "{%- if tools and tools is iterable and tools is not mapping -%}" - " {{- '<|im_start|>system\n' -}}" - " {{- '# Tools\n\nYou have access to the following functions:\n\n' -}}" - " {%- for tool in tools -%}" - " {{- '\n' -}}" - " {{- tool | tojson -}}" - " {%- endfor -%}" - " {{- '\n' -}}" - " {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' -}}" - " {%- if messages[0].role == 'system' -%}" - " {%- set content = render_content(messages[0].content, false, true) | trim -%}" - " {%- if content -%}" - " {{- '\n\n' + content -}}" - " {%- endif -%}" - " {%- endif -%}" - " {{- '<|im_end|>\n' -}}" - "{%- elif messages[0].role == 'system' -%}" - " {%- set content = render_content(messages[0].content, false, true) -%}" - " {{- '<|im_start|>system\n' + content + '<|im_end|>\n' -}}" - "{%- endif -%}" - "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages | length - 1) -%}" - "{%- for message in messages[::-1] -%}" - " {%- set index = messages | length - 1 - loop.index0 -%}" - " {%- if ns.multi_step_tool and message.role == 'user' -%}" - " {%- set content = render_content(message.content, false) | trim -%}" - " {%- if not (content.startswith('') and content.endswith('')) -%}" - " {%- set ns.multi_step_tool = false -%}" - " {%- set ns.last_query_index = index -%}" - " {%- endif -%}" - " {%- endif -%}" - "{%- endfor -%}" - "{%- if ns.multi_step_tool -%}" - " {{- raise_exception('No user query found in messages.') -}}" - "{%- endif -%}" - "{%- for message in messages -%}" - " {%- set content = render_content(message.content, true) | trim -%}" - " {%- if message.role == 'system' -%}" - " {%- if not loop.first -%}" - " {{- raise_exception('System message must be at the beginning.') -}}" - " {%- endif -%}" - " {%- elif message.role == 'user' -%}" - " {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' -}}" - " {%- elif message.role == 'assistant' -%}" - " {%- set reasoning_content = '' -%}" - " {%- if message.reasoning_content is string -%}" - " {%- set reasoning_content = message.reasoning_content -%}" - " {%- elif '' in content -%}" - " {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') -%}" - " {%- set content = content.split('')[-1].lstrip('\n') -%}" - " {%- endif -%}" - " {%- set reasoning_content = reasoning_content | trim -%}" - " {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) -%}" - " {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content -}}" - " {%- else -%}" - " {{- '<|im_start|>' + message.role + '\n' + content -}}" - " {%- endif -%}" - " {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping -%}" - " {%- for tool_call in message.tool_calls -%}" - " {%- if tool_call.function is defined -%}" - " {%- set tool_call = tool_call.function -%}" - " {%- endif -%}" - " {%- if loop.first -%}" - " {%- if content | trim -%}" - " {{- '\n\n\n\n' -}}" - " {%- else -%}" - " {{- '\n\n' -}}" - " {%- endif -%}" - " {%- else -%}" - " {{- '\n\n\n' -}}" - " {%- endif -%}" - " {%- if tool_call.arguments is defined -%}" - " {%- for (args_name, args_value) in tool_call.arguments | items -%}" - " {{- '\n' -}}" - " {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}" - " {{- args_value -}}" - " {{- '\n' -}}" - " {%- endfor -%}" - " {%- endif -%}" - " {{- '\n' -}}" - " {%- endfor -%}" - " {%- endif -%}" - " {{- '<|im_end|>\n' -}}" - " {%- elif message.role == 'tool' -%}" - " {%- if loop.previtem and loop.previtem.role != 'tool' -%}" - " {{- '<|im_start|>user' -}}" - " {%- endif -%}" - " {{- '\n\n' -}}" - " {{- content -}}" - " {{- '\n' -}}" - " {%- if not loop.last and loop.nextitem.role != 'tool' -%}" - " {{- '<|im_end|>\n' -}}" - " {%- elif loop.last -%}" - " {{- '<|im_end|>\n' -}}" - " {%- endif -%}" - " {%- else -%}" - " {{- raise_exception('Unexpected message role.') -}}" - " {%- endif -%}" - "{%- endfor -%}" - "{%- if add_generation_prompt -%}" - " {{- '<|im_start|>assistant\n' -}}" - " {%- if enable_thinking is defined and enable_thinking is false -%}" - " {{- '\n\n\n\n' -}}" - " {%- else -%}" - " {{- '\n' -}}" - " {%- endif -%}" - "{%- endif -%}" - ) - - def __init__( - self, - add_vision_id: bool = True, - enable_thinking: bool = True, - preserve_thinking: bool = False, - **kwargs, - ): - """ - Parameters: - - add_vision_id (bool): - - True (default): Count all the images. Recommended for multi-image. - - False: Doesn't count the images. Can save tokens with single-image. - - enable_thinking (bool): - - True (default): Enables reasoning for better results. - - False: Disables reasoning for faster results. - - preserve_thinking (bool): - - True: Keeps reasoning process for ALL historical conversational turns. - - False (default): Only keeps for the latest assistant reply to save tokens. - """ - super().__init__(**kwargs) - self.enable_thinking = enable_thinking - self.preserve_thinking = preserve_thinking - self.extra_template_arguments["add_vision_id"] = add_vision_id - self.extra_template_arguments["enable_thinking"] = enable_thinking - self.extra_template_arguments["preserve_thinking"] = preserve_thinking - - def __call__(self, **kwargs): - llama = kwargs['llama'] - - if hasattr(llama, 'input_ids'): - llama.input_ids.fill(0) - - if self.verbose: - print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}, preserve_thinking={self.preserve_thinking}) - Start processing") - - # Use parent implementation - return super().__call__(**kwargs) - - -class Step3VLChatHandler(MTMDChatHandler): - """ - Handler for Step3-VL models. - """ - - STEP3VL_BOS_TOKEN = "<|im_start|>" - STEP3VL_EOS_TOKEN = "<|im_end|>" - STEP3VL_PAD_TOKEN = "<|endoftext|>" - STEP3VL_IMAGE_TOKEN = "" - - CHAT_FORMAT = ( - "{%- macro render_content(content) -%}\n" - " {%- if content is none -%}{{- '' -}}\n" - " {%- elif content is string -%}{{- content -}}\n" - " {%- elif content is mapping -%}{{- content['value'] if 'value' in content else content['text'] -}}\n" - " {%- elif content is iterable -%}\n" - " {%- for item in content -%}\n" - " {%- if item.type == 'text' -%}\n" - " {{- item['value'] if 'value' in item else item['text'] -}}\n" - " {%- elif item.type in ['image', 'image_url'] -%}\n" - " {%- set url_val = '' -%}\n" - " {%- if item.image_url -%}\n" - " {%- set url_val = item.image_url if item.image_url is string else item.image_url.url -%}\n" - " {%- endif -%}\n" - " {{- '' + url_val -}}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - "{%- endmacro -%}\n" - "\n" - "{%- if tools -%}\n" - " {{- '<|im_start|>system\\n' -}}\n" - " {%- if messages[0].role == 'system' -%}\n" - " {{- render_content(messages[0].content) + '\\n\\n' -}}\n" - " {%- endif -%}\n" - " {{- '# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n' -}}\n" - " {%- for tool in tools -%}\n" - " {{- '\\n' -}}\n" - " {{- tool | tojson -}}\n" - " {%- endfor -%}\n" - " {{- '\\n\\n\\nAlways adhere to this exact format for tool use:\\n\\n\\n{\"name\": , \"arguments\": }\\n\\n{additional_tool_calls}\\n\\nNote:\\n- For each function call, return a json object with function name and arguments within XML tags.\\n- `` must be an exact match to one of the available tools.\\n- `` must be valid JSON that strictly follows the tool\\'s parameters schema.<|im_end|>\\n' -}}\n" - "{%- else -%}\n" - " {%- if messages[0].role == 'system' -%}\n" - " {{- '<|im_start|>system\\n' + render_content(messages[0].content) + '<|im_end|>\\n' -}}\n" - " {%- endif -%}\n" - "{%- endif -%}\n" - "\n" - "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) -%}\n" - "{%- for message in messages[::-1] -%}\n" - " {%- set index = (messages|length - 1) - loop.index0 -%}\n" - " {%- if ns.multi_step_tool and message.role == 'user' and render_content(message.content) is string and not(render_content(message.content).startswith('') and render_content(message.content).endswith('')) -%}\n" - " {%- set ns.multi_step_tool = false -%}\n" - " {%- set ns.last_query_index = index -%}\n" - " {%- endif -%}\n" - "{%- endfor -%}\n" - "\n" - "{%- for message in messages -%}\n" - " {%- set content = render_content(message.content) -%}\n" - " {%- if (message.role == 'user') or (message.role == 'system' and not loop.first) -%}\n" - " {%- set role_name = 'observation' if (message.role == 'system' and not loop.first and message.name == 'observation') else message.role -%}\n" - " {{- '<|im_start|>' + role_name + '\\n' + content + '<|im_end|>' + '\\n' -}}\n" - " {%- elif message.role == 'assistant' -%}\n" - " {%- if message.reasoning_content is string -%}\n" - " {%- set reasoning_content = render_content(message.reasoning_content) -%}\n" - " {%- else -%}\n" - " {%- if '' in content -%}\n" - " {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') -%}\n" - " {%- set content = content.split('')[-1].lstrip('\\n') -%}\n" - " {%- else -%}\n" - " {%- set reasoning_content = '' -%}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- if loop.index0 > ns.last_query_index -%}\n" - " {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n' + content -}}\n" - " {%- else -%}\n" - " {{- '<|im_start|>' + message.role + '\\n' + content -}}\n" - " {%- endif -%}\n" - " {%- if message.tool_calls -%}\n" - " {{- '\\n' -}}\n" - " {%- for tool_call in message.tool_calls -%}\n" - " {{- '\\n' -}}\n" - " {%- if tool_call.function -%}\n" - " {%- set tool_call = tool_call.function -%}\n" - " {%- endif -%}\n" - " {{- '\\n{\"name\": \"' -}}\n" - " {{- tool_call.name -}}\n" - " {{- '\", \"arguments\": ' -}}\n" - " {%- if tool_call.arguments is string -%}\n" - " {{- tool_call.arguments -}}\n" - " {%- else -%}\n" - " {{- tool_call.arguments | tojson -}}\n" - " {%- endif -%}\n" - " {{- '}\\n' -}}\n" - " {%- endfor -%}\n" - " {{- '\\n' -}}\n" - " {%- endif -%}\n" - " {{- '<|im_end|>\\n' -}}\n" - " {%- elif message.role == 'tool' -%}\n" - " {%- if loop.first or (messages[loop.index0 - 1].role != 'tool') -%}\n" - " {{- '<|im_start|>tool_response' -}}\n" - " {%- endif -%}\n" - " {{- '\\n\\n' -}}\n" - " {{- content -}}\n" - " {{- '\\n' -}}\n" - " {%- if loop.last or (messages[loop.index0 + 1].role != 'tool') -%}\n" - " {{- '<|im_end|>\\n' -}}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - "{%- endfor -%}\n" - "{%- if add_generation_prompt -%}\n" - " {{- '<|im_start|>assistant\\n\\n\\n\\n' if (enable_thinking is defined and not enable_thinking) else '<|im_start|>assistant\\n' -}}\n" - "{%- endif -%}\n" - ) - - def __init__(self, enable_thinking: bool = True, **kwargs): - """ - Initializes the Step3-VL Handler. - - Args: - enable_thinking (bool): If False, injects an empty block to bypass reasoning. - """ - self.enable_thinking = enable_thinking - super().__init__(**kwargs) - - def __call__(self, **kwargs): - # Pass thinking toggle into Jinja - self.extra_template_arguments["enable_thinking"] = self.enable_thinking - - # Step3 uses standard <|im_end|> ChatML stop formatting - kwargs['stop'] = [self.STEP3VL_PAD_TOKEN, self.STEP3VL_EOS_TOKEN] - - if self.verbose: - print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") - - return super().__call__(**kwargs) - - -@register_chat_completion_handler("chatml-function-calling") -def chatml_function_calling( - llama: llama_core.Llama, - messages: List[llama_types.ChatCompletionRequestMessage], - functions: Optional[List[llama_types.ChatCompletionFunction]] = None, - function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, - tools: Optional[List[llama_types.ChatCompletionTool]] = None, - tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, - temperature: float = 0.2, - top_p: float = 0.95, - top_k: int = 40, - min_p: float = 0.05, - typical_p: float = 1.0, - stream: bool = False, - stop: Optional[Union[str, List[str]]] = [], - response_format: Optional[llama_types.ChatCompletionRequestResponseFormat] = None, - max_tokens: Optional[int] = None, - present_penalty: float = 0.0, - frequency_penalty: float = 0.0, - repeat_penalty: float = 1.1, - top_n_sigma: float = -1.00, - mirostat_mode: int = 0, - mirostat_tau: float = 5.0, - mirostat_eta: float = 0.1, - xtc_threshold: float = 0.1, - xtc_probability: float = 0.0, - dry_multiplier: float = 0.0, - dry_base: float = 1.75, - dry_allowed_length: int = 2, - dry_penalty_last_n:int = 0, - dry_seq_breakers: list[str] = ["\n", ":", "\"", "*"], - adaptive_target : float = -1.0, - adaptive_decay : float = 0.9, - use_infill: bool = False, - model: Optional[str] = None, - logits_processor: Optional[llama_core.LogitsProcessorList] = None, - grammar: Optional[llama_grammar.LlamaGrammar] = None, - logprobs: Optional[bool] = None, - top_logprobs: Optional[int] = None, - **kwargs, # type: ignore -) -> Union[ - llama_types.CreateChatCompletionResponse, - Iterator[llama_types.CreateChatCompletionStreamResponse], -]: - function_calling_template = ( - "{% for message in messages %}" - "<|im_start|>{{ message.role }}\n" - # System message - "{% if message.role == 'system' %}" - "{{ message.content }}" - "{% if tool_calls %}" - "\n\nYou have access to the following functions:\n" - "{% for tool in tools %}" - "\nfunctions.{{ tool.function.name }}:\n" - "{{ tool.function.parameters | tojson }}" - "\n{% endfor %}" - "\n\nYou can respond to users messages with either a single message or one or more function calls." - "\n\nTo respond with a message begin the message with 'message:', use the following format:" - "\n\nmessage:" - "\n" - "\n\nTo respond with one or more function calls begin the message with 'functions.:', use the following format:" - "\n\nfunctions.:" - '\n{ "arg1": "value1", "arg2": "value2" }' - "\nfunctions.:" - '\n{ "arg1": "value1", "arg2": "value2" }' - "{% endif %}" - "<|im_end|>\n" - "{% endif %}" - # User message - "{% if message.role == 'user' %}" - "{{ message.content }}" - "<|im_end|>\n" - "{% endif %}" - # Assistant message - "{% if message.role == 'assistant' %}" - ## Reglar message - "{% if message.content and message.content | length > 0 %}" - "{% if tool_calls %}" - "message:\n" - "{% endif %}" - "{{ message.content }}" - "<|im_end|>\n" - "{% endif %}" - ## Function calls - "{% if 'tool_calls' in message %}" - "{% for tool_call in message.tool_calls %}" - "functions.{{ tool_call.function.name }}:\n" - "{{ tool_call.function.arguments }}" - "{% endfor %}" - "<|im_end|>\n" - "{% endif %}" - "{% endif %}" - "{% endfor %}" - "{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" - ) - template_renderer = ImmutableSandboxedEnvironment( - autoescape=jinja2.select_autoescape(["html", "xml"]), - undefined=jinja2.StrictUndefined, - ).from_string(function_calling_template) - - # Convert legacy functions to tools - if functions is not None: - tools = [ - { - "type": "function", - "function": function, - } - for function in functions - ] - - # Convert legacy function_call to tool_choice - if function_call is not None: - if isinstance(function_call, str) and ( - function_call == "none" or function_call == "auto" - ): - tool_choice = function_call - if isinstance(function_call, dict) and "name" in function_call: - tool_choice = { - "type": "function", - "function": { - "name": function_call["name"], - }, - } - - stop = ( - [stop, "<|im_end|>"] - if isinstance(stop, str) - else stop + ["<|im_end|>"] if stop else ["<|im_end|>"] - ) - - # Case 1: No tool choice by user - if ( - tool_choice is None - or (isinstance(tool_choice, str) and tool_choice == "none") - or tools is None - or len(tools) == 0 - ): - prompt = template_renderer.render( - messages=messages, - tools=[], - tool_calls=None, - add_generation_prompt=True, - ) - - if response_format is not None and response_format["type"] == "json_object": - grammar = _grammar_for_response_format(response_format) - - return _convert_completion_to_chat( - llama.create_completion( - prompt=prompt, - temperature=temperature, - top_p=top_p, - top_k=top_k, - min_p=min_p, - typical_p=typical_p, - stream=stream, - stop=stop, - max_tokens=max_tokens, - present_penalty=present_penalty, - frequency_penalty=frequency_penalty, - repeat_penalty=repeat_penalty, - top_n_sigma=top_n_sigma, - mirostat_mode=mirostat_mode, - mirostat_tau=mirostat_tau, - mirostat_eta=mirostat_eta, - xtc_threshold=xtc_threshold, - xtc_probability=xtc_probability, - dry_multiplier=dry_multiplier, - dry_base=dry_base, - dry_allowed_length=dry_allowed_length, - dry_penalty_last_n=dry_penalty_last_n, - dry_seq_breakers=dry_seq_breakers, - adaptive_target=adaptive_target, - adaptive_decay=adaptive_decay, - use_infill=use_infill, - model=model, - logits_processor=logits_processor, - grammar=grammar, - logprobs=top_logprobs if logprobs else None, - ), - stream=stream, - ) - - # Case 2: Tool choice by user - if isinstance(tool_choice, dict): - tool_name = tool_choice["function"]["name"] - tool = next( - (tool for tool in tools if tool["function"]["name"] == tool_name), None - ) - if tool is None: - raise ValueError(f"Tool with name '{tool_name}' not found in tools") - prompt = template_renderer.render( - messages=messages, - tools=tools, - tool_calls=True, - add_generation_prompt=True, - ) - prompt += f"functions.{tool_name}:\n" + prompt += f"functions.{tool_name}:\n" try: grammar = llama_grammar.LlamaGrammar.from_json_schema( json.dumps(tool["function"]["parameters"]), verbose=llama.verbose @@ -6956,3 +3539,35 @@ def chatml_function_calling( } raise ValueError("Automatic streaming tool choice is not supported") + +# Backward compatibility re-exports. +# These multimodal chat handlers have been moved to `llama_multimodal`. +# New code should import them from `llama_cpp.llama_multimodal` instead of +# `llama_cpp.llama_chat_format`. +from llama_cpp.llama_multimodal import ( + MTMDChatHandler, + GenericMTMDChatHandler, + Llava15ChatHandler, + ObsidianChatHandler, + MoondreamChatHandler, + Llava16ChatHandler, + NanoLlavaChatHandler, + Llama3VisionAlphaChatHandler, + Llama3VisionAlpha, + MiniCPMv26ChatHandler, + MiniCPMv45ChatHandler, + MiniCPMV46ChatHandler, + Gemma3ChatHandler, + Gemma4ChatHandler, + GLM41VChatHandler, + GLM46VChatHandler, + GraniteDoclingChatHandler, + LFM2VLChatHandler, + LFM25VLChatHandler, + PaddleOCRChatHandler, + Qwen25VLChatHandler, + Qwen3ASRChatHandler, + Qwen3VLChatHandler, + Qwen35ChatHandler, + Step3VLChatHandler +) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py new file mode 100644 index 0000000000..a055869543 --- /dev/null +++ b/llama_cpp/llama_multimodal.py @@ -0,0 +1,3473 @@ +from __future__ import annotations + +import base64 +import ctypes +import json +import os +import sys +import zlib + +from contextlib import ExitStack +from typing import ( + Any, + Dict, + Iterator, + List, + Literal, + Optional, + Tuple, + Union, + Protocol, + TYPE_CHECKING, + cast, +) + +import urllib.request +from urllib.error import URLError, HTTPError + +import llama_cpp.llama_cpp as llama_cpp_lib +import llama_cpp.llama_types as llama_types +import llama_cpp.llama_grammar as llama_grammar + +if TYPE_CHECKING: + import llama_cpp.llama as llama_core + +from ._logger import ggml_log_callback + +from llama_cpp.llama_chat_format import ( + _convert_completion_to_chat, + _convert_completion_to_chat_function, + _grammar_for_response_format, + ImmutableSandboxedEnvironment +) + +class MTMDChatHandler: + DEFAULT_SYSTEM_MESSAGE: Optional[str] = ( +"You are an exceptionally capable, precise, and helpful multimodal AI assistant that excels at deeply understanding and richly describing images, charts, diagrams, text in images, scenes, and any visual content, " +"while also answering every question accurately, clearly, and step-by-step when appropriate ā€” always responding in the same language as the user's question, remaining polite, professional, and maximally helpful." + ) + + CHAT_FORMAT = ( + "{{ bos_token if bos_token is defined else '' }}" + "{% for message in messages %}" + "{% if message.role == 'system' %}" + "{{ message.content }}" + "{% elif message.role == 'user' %}" + "USER: " + "{% if message.content is string %}" + "{{ message.content }}" + "{% elif message.content is iterable %}" + "{% for content in message.content %}" + "{% if content.type == 'image_url' %}" + "{{ content.image_url if content.image_url is string else content.image_url.url }}" + "{% elif content.type == 'audio_url' %}" + "{{ content.audio_url if content.audio_url is string else content.audio_url.url }}" + "{% elif content.type == 'input_audio' %}" + "{% if content.input_audio is string %}" + "{{ content.input_audio }}" + "{% else %}" + "data:audio/{{ content.input_audio.format }};base64,{{ content.input_audio.data }}" + "{% endif %}" + "{% elif content.type == 'video_url' %}" + "{{ content.video_url if content.video_url is string else content.video_url.url }}" + "{% elif content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + + "{% elif message.role == 'assistant' and message.content is not none %}" + "ASSISTANT: {{ message.content }}" + "{% endif %}" + "{{ \"\n\" }}" + "{% endfor %}" + + "{% if eos_token is defined %}" + "{{ eos_token }}" + "{% endif %}" + + "{% if add_generation_prompt %}" + "ASSISTANT: " + "{% endif %}" + ) + + def __init__( + self, + mmproj_path: Optional[str] = None, + verbose: bool = True, + use_gpu: bool = True, + image_min_tokens: int = -1, + image_max_tokens: int = -1, + chat_template_override: Optional[str] = None, + batch_max_tokens: int = 1024, + **kwargs + ): + + self.log_prefix = self.__class__.__name__ + self.verbose = verbose + + # Backward compatibility: `clip_model_path` was the old name for `mmproj_path`. + # Accept it for existing user code, warn during initialization, and normalize + # all internal usage to `mmproj_path`. + clip_model_path = kwargs.pop("clip_model_path", None) + if mmproj_path is None and clip_model_path is not None: + mmproj_path = clip_model_path + if self.verbose: + print( + f"{self.log_prefix}(__init__): `clip_model_path` is deprecated; " + "please use `mmproj_path` instead.", + file=sys.stderr, + ) + + if kwargs: + unexpected_args = ", ".join(f"'{k}'" for k in kwargs.keys()) + raise TypeError( + f"Initialization Error in {self.log_prefix}: Received unexpected keyword argument(s) {unexpected_args}.\n" + f"If you are passing model-specific parameters, ensure they are supported by {self.log_prefix}." + ) + + if mmproj_path is None: + raise ValueError( + f"{self.log_prefix}(__init__): `mmproj_path` is required. " + "`clip_model_path` is accepted only as a deprecated compatibility alias." + ) + + self.mmproj_path = mmproj_path + if not os.path.exists(self.mmproj_path): + raise ValueError( + f"{self.log_prefix}(__init__): mmproj path does not exist: {self.mmproj_path}" + ) + + self.image_min_tokens = image_min_tokens + self.image_max_tokens = image_max_tokens + self.batch_max_tokens = batch_max_tokens + self.use_gpu = use_gpu + + import llama_cpp.mtmd_cpp as mtmd_cpp + self._mtmd_cpp = mtmd_cpp + self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None + self.extra_template_arguments: dict[str, Any] = {} + + self.is_support_vision = False + self.is_support_audio = False + self.is_support_video = False + + # Pre-compile Jinja template + if (not hasattr(self, "chat_format") or self.chat_format is None) and chat_template_override is None: + self.chat_format = self.CHAT_FORMAT + elif chat_template_override is not None: + self.chat_format = chat_template_override + + self._chat_format_parser_tags = [] + self._change_chat_template(self.chat_format) + + self._exit_stack = ExitStack() + + def _change_chat_template(self, new_template: str): + self.chat_template = ImmutableSandboxedEnvironment( + trim_blocks=True, + lstrip_blocks=True + ).from_string(new_template) + + def _init_mtmd_context(self, llama_model: llama_core.Llama): + """Initialize mtmd context with the llama model.""" + if self.mtmd_ctx is not None: + return # Already initialized + + self._mtmd_cpp.mtmd_helper_log_set(ggml_log_callback, ctypes.c_void_p(0)) + + # Get default parameters + self.mctx_params = self._mtmd_cpp.mtmd_context_params_default() + self.mctx_params.use_gpu = self.use_gpu + self.mctx_params.print_timings = self.verbose + self.mctx_params.n_threads = llama_model.n_threads + self.mctx_params.flash_attn_type = self._mtmd_cpp.clip_flash_attn_type.CLIP_FLASH_ATTN_TYPE_AUTO + self.mctx_params.warmup = True + if self.image_min_tokens > 0: + self.mctx_params.image_min_tokens = self.image_min_tokens + if self.image_max_tokens > 0: + self.mctx_params.image_max_tokens = self.image_max_tokens + if (self.image_max_tokens < self.image_min_tokens) and self.image_max_tokens > 0: + raise ValueError(f"{self.log_prefix}(_init_mtmd_context): Configuration Error! image_max_tokens ({self.image_max_tokens}) " + f"cannot be less than image_min_tokens ({self.image_min_tokens}).") + self.mctx_params.batch_max_tokens = self.batch_max_tokens + + # Cache the model's eos token and bos token + self.mtmd_eos_token=llama_model.detokenize([llama_model.token_eos()]).decode('utf-8', errors='ignore') + self.mtmd_bos_token=llama_model.detokenize([llama_model.token_bos()]).decode('utf-8', errors='ignore') + + # Cache the mtmd_default_marker + self.media_marker = self._mtmd_cpp.mtmd_default_marker().decode('utf-8') + + # Initialize mtmd context + self.mtmd_ctx = self._mtmd_cpp.mtmd_init_from_file( + self.mmproj_path.encode(), + llama_model.model, + self.mctx_params + ) + + if self.mtmd_ctx is None: + raise ValueError(f"{self.log_prefix}(_init_mtmd_context): Failed to load mtmd context from: {self.mmproj_path}") + + # Check if vision is supported + self.is_support_vision = self._mtmd_cpp.mtmd_support_vision(self.mtmd_ctx) + if self.is_support_vision: + if self.verbose: + print(f"{self.log_prefix}(_init_mtmd_context): Vision support detected.", file=sys.stderr) + else: + if self.verbose: + print(f"{self.log_prefix}(_init_mtmd_context): Vision is NOT supported by this mmproj model backend.", file=sys.stderr) + + # Check if audio is supported + self.is_support_audio = self._mtmd_cpp.mtmd_support_audio(self.mtmd_ctx) + if self.is_support_audio: + if self.verbose: + print(f"{self.log_prefix}(_init_mtmd_context): Audio support detected.", file=sys.stderr) + else: + if self.verbose: + print(f"{self.log_prefix}(_init_mtmd_context): Audio is NOT supported by this mmproj model backend.", file=sys.stderr) + + # Check if video is supported + self.is_support_video = self._mtmd_cpp.mtmd_helper_support_video(self.mtmd_ctx) + if self.is_support_video: + if self.verbose: + print(f"{self.log_prefix}(_init_mtmd_context): Video support detected.", file=sys.stderr) + else: + if self.verbose: + print(f"{self.log_prefix}(_init_mtmd_context): Video support is NOT available in this build.", file=sys.stderr) + + def close(self) -> None: + """Explicitly free the mtmd context and vision model resources.""" + if getattr(self, "mtmd_ctx", None) is not None: + try: + self._mtmd_cpp.mtmd_free(self.mtmd_ctx) + except Exception: + pass + self.mtmd_ctx = None + self.mctx_params = None + self.chat_template = None + + if getattr(self, "_exit_stack", None) is not None and hasattr(self._exit_stack, "close"): + self._exit_stack.close() + self._exit_stack = None + + def __del__(self) -> None: + self.close() + + def _get_media_items(self, messages: List[llama_types.ChatCompletionRequestMessage]) -> List[Dict[str, str]]: + """ + Extracts all media payloads (images, audio) sequentially to maintain exact chronological order. + Strictly enforces capability checks, raising exceptions if unsupported media is passed. + + Returns: + media_items: A list of dictionaries containing the media 'url' and its 'type' (image or audio). + """ + media_items: List[Dict[str, str]] = [] + for message in messages: + if isinstance(message.get("content"), list): + for content in message["content"]: + content_type = content.get("type", "") + + # 1. Vision Processing + if content_type == "image_url": + if not self.is_support_vision: + raise ValueError(f"{self.log_prefix}: This mmproj model instance does not support image inputs.") + + url = content["image_url"] if isinstance(content["image_url"], str) else content["image_url"]["url"] + media_items.append({"url": url, "type": "image"}) + + # 2. Audio Processing + elif content_type in ["audio", "audio_url", "input_audio"]: + if not self.is_support_audio: + raise ValueError(f"{self.log_prefix}: This mmproj model instance does not support audio inputs.") + + # Case A: Handle custom/forward-compatible audio_url format + if content_type == "audio_url" or content_type == "audio": + audio_url = content[content_type] + url = audio_url if isinstance(audio_url, str) else audio_url["url"] + media_items.append({"url": url, "type": "audio"}) + # Case B: Handle OpenAI standard input_audio format + elif content_type == "input_audio": + input_audio = content.get("input_audio", {}) + if isinstance(input_audio, dict) and "data" in input_audio: + # It might just be raw base64 data, we can format it as a data URI to reuse load_audio logic + # input_audio: { + # data: audio.base64Data, + # format: audio.mimeType.includes('wav') ? 'wav' : 'mp3' + # } + audio_data = input_audio.get("data", "") + audio_format = input_audio.get("format", "") + + # Strictly align with llama.cpp (require wav/mp3) + if audio_format not in ["wav", "mp3"]: + raise ValueError(f"{self.log_prefix}: input_audio.format must be either 'wav' or 'mp3'") + + # Format as a Data URI to reuse the unified load_media logic + media_items.append({ + "url": f"data:audio/{audio_format};base64,{audio_data}", + "type": "audio" + }) + else: + # Just a raw base64 data + url = input_audio if isinstance(input_audio, str) else "" + if url: + media_items.append({"url": url, "type": "audio"}) + + # 3. Video Processing + elif content_type == "video_url": + if not self.is_support_video: + raise ValueError(f"{self.log_prefix}: This libmtmd build does not support video inputs.") + + video_url = content["video_url"] + url = video_url if isinstance(video_url, str) else video_url["url"] + media_items.append({"url": url, "type": "video"}) + + # 4. Text & Unknown Types + elif content_type == "text": + continue + else: + if self.verbose: + print(f"{self.log_prefix}: Ignored unknown content type '{content_type}'.", file=sys.stderr) + return media_items + + def _create_bitmap_from_bytes(self, media_bytes: bytes): + """ + Constructs an mtmd_bitmap structure from a raw byte buffer containing media data. + + Supported formats: + - Images (via stb_image): jpg, png, bmp, etc. + - Audio (via miniaudio): wav, mp3, flac. + - Video: depends on whether MTMD_VIDEO was enabled at build time. + + Note: + - Media types (Image vs. Audio) are auto-detected by the C++ backend using magic bytes. + - The underlying C++ helper function is thread-safe, making it suitable for concurrent preprocessing. + + Args: + media_bytes (bytes): The raw byte content of the media file. + + Returns: + bitmap: mtmd_bitmap * + video_ctx: mtmd_helper_video * or NULL + """ + if self.mtmd_ctx is None: + raise ValueError(f"{self.log_prefix}(_create_bitmap_from_bytes): mtmd context not initialized.") + + if not media_bytes: + raise ValueError(f"{self.log_prefix}(_create_bitmap_from_bytes): empty media bytes.") + + buf = (ctypes.c_uint8 * len(media_bytes)).from_buffer_copy(media_bytes) + + wrapper = self._mtmd_cpp.mtmd_helper_bitmap_init_from_buf( + self.mtmd_ctx, + buf, + len(media_bytes), + False, + ) + + if not wrapper.bitmap: + if wrapper.video_ctx: + self._mtmd_cpp.mtmd_helper_video_free(wrapper.video_ctx) + + raise ValueError( + f"{self.log_prefix}(_create_bitmap_from_bytes): " + "Failed to load media from bytes " + "(unsupported media format, corrupted data, or missing helper support)." + ) + + return wrapper.bitmap, wrapper.video_ctx + + def _is_text_chunk(self, chunk_type: int) -> bool: + """Return True if `chunk_type` is the MTMD text chunk type enum value.""" + return ( + chunk_type + == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_TEXT + ) + + def _is_image_chunk(self, chunk_type: int) -> bool: + """Return True if `chunk_type` is the MTMD image chunk type enum value.""" + return ( + chunk_type + == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_IMAGE + ) + + def _is_audio_chunk(self, chunk_type: int) -> bool: + """Return True if `chunk_type` is the MTMD audio chunk type enum value.""" + return ( + chunk_type + == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO + ) + + def _process_mtmd_prompt( + self, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + add_generation_prompt: bool = True, + ) -> Tuple[List[int], List[tuple], Any, List[Any]]: + """ + Core multimodal preprocessing pipeline. + Converts raw chat messages into C++ MTMD chunk structures and a virtual token ledger. + + Features: + - Thread-safe concurrent media decoding to eliminate I/O bottlenecks. + - "Negative Reverse Vocabulary" mapping for O(1) prefix matching of media tokens. + - Strict RAII-style C++ memory management to prevent leaks on failure. + + Returns: + full_prompt_ids: Ledger of text tokens and negative media IDs for prefix matching. + chunk_token_spans: Tuples of (start_idx, end_idx, chunk_ptr, chunk_type, media_id). + chunks: Allocated C++ mtmd_input_chunks pointer (must be freed by the caller). + bitmap_cleanup: List of C++ bitmap pointers to be freed after evaluation. + """ + # 1. Inject default system prompt if omitted by the user + system_prompt = next((msg["content"] for msg in messages if msg.get("role") == "system"), "") + if system_prompt == "" and self.DEFAULT_SYSTEM_MESSAGE is not None: + messages = [{"role": "system", "content": self.DEFAULT_SYSTEM_MESSAGE}] + messages + + media_items = self._get_media_items(messages) + media_marker = self.media_marker + + # 2. Render the chat template and replace actual URLs with C++ media markers + text = self.chat_template.render( + messages=messages, + add_generation_prompt=add_generation_prompt, + eos_token=self.mtmd_eos_token, + bos_token=self.mtmd_bos_token, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + **getattr(self, 'extra_template_arguments', {}) + ) + + for tag in self._chat_format_parser_tags: + if tag not in text: + continue + + text = text.replace(tag, media_marker) + + # Replace image_url by media_marker in text + for item in media_items: + text = text.replace(item["url"], media_marker) + + if self.verbose: + print(f"{self.log_prefix}(_process_mtmd_prompt): Rendered prompt length: {len(text)} chars, Media count: {len(media_items)}.", file=sys.stderr) + print(f"{self.log_prefix}(_process_mtmd_prompt): Rendered prompt: {text}", file=sys.stderr) + + # 3. Pre-allocate bitmap array to guarantee chronological order during concurrent decoding + bitmaps = [None] * len(media_items) + bitmap_cleanup = [] + video_cleanup = [] + chunks = None + + try: + # Concurrent Media Decoding + import concurrent.futures + if media_items: + def _create_bitmap_func(idx: int, item: dict): + media_bytes = self.load_media(item["url"], item["type"]) + bitmap, video_ctx = self._create_bitmap_from_bytes(media_bytes) + return idx, bitmap, video_ctx + # This method uses multi-threaded parallel processing to convert images or audio to bitmaps, + # which can be used in the future to process large numbers of video frames. + max_workers = min(llama.n_threads, len(media_items)) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(_create_bitmap_func, i, item) for i, item in enumerate(media_items)] + + for future in concurrent.futures.as_completed(futures): + idx, bitmap, video_ctx = future.result() + + bitmaps[idx] = bitmap + bitmap_cleanup.append(bitmap) + + if video_ctx: + video_cleanup.append(video_ctx) + + # Strict validation: Abort if any thread failed to decode its assigned media + if any(b is None for b in bitmaps): + raise RuntimeError(f"{self.log_prefix}(_create_bitmap_func): Failed to decode one or more media files.") + else: + if self.verbose: + print(f"{self.log_prefix}(_create_bitmap_func with {max_workers} threads): {len(media_items)} bitmaps were successfully created.") + else: + # If there are no images, set the bitmaps to empty. + bitmaps = [] + + # 4. Initialize mtmd_input_chunks + input_text = self._mtmd_cpp.mtmd_input_text() + input_text.text = text.encode('utf-8') + input_text.add_special = (llama.n_tokens == 0) + input_text.parse_special = True + + chunks = self._mtmd_cpp.mtmd_input_chunks_init() + if chunks is None: + raise ValueError(f"{self.log_prefix}(mtmd_input_chunks_init): Failed to initialize mtmd_input_chunks.") + + # 5. Hybrid Tokenization (Text + Media binding) + if len(bitmaps) > 0: + bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) + result = self._mtmd_cpp.mtmd_tokenize( + self.mtmd_ctx, chunks, ctypes.byref(input_text), bitmap_array, len(bitmaps) + ) + else: + result = self._mtmd_cpp.mtmd_tokenize( + self.mtmd_ctx, chunks, ctypes.byref(input_text), None, 0 + ) + + if result != 0: + raise ValueError(f"{self.log_prefix}(mtmd_tokenize): Unable to tokenize prompt, res = {result}.") + + # Video helper contexts only need to stay alive until mtmd_tokenize() completes. + if video_cleanup: + for video_ctx in video_cleanup: + self._mtmd_cpp.mtmd_helper_video_free(video_ctx) + video_cleanup.clear() + + # 6. Virtual Token Ledger Construction + full_prompt_ids = [] + chunk_token_spans = [] + current_idx = 0 + n_chunks = self._mtmd_cpp.mtmd_input_chunks_size(chunks) + + # Cursor to track the actual media contents (URLs or base64 data) provided by the user + media_items_count = len(media_items) + media_items_cur = 0 + last_media_id = None + + for i in range(n_chunks): + chunk = self._mtmd_cpp.mtmd_input_chunks_get(chunks, i) + if chunk is None: continue + chunk_type = self._mtmd_cpp.mtmd_input_chunk_get_type(chunk) + + if self._is_text_chunk(chunk_type): + # Extract standard text token IDs + n_tokens_out = ctypes.c_size_t() + tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text(chunk, ctypes.byref(n_tokens_out)) + if tokens_ptr and n_tokens_out.value > 0: + tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)] + chunk_token_spans.append((current_idx, current_idx + len(tokens), chunk, chunk_type, None)) + full_prompt_ids.extend(tokens) + current_idx += len(tokens) + elif self._is_image_chunk(chunk_type) or self._is_audio_chunk(chunk_type): + # Extract media properties + # Note(JamePeng): + # The M-RoPE model is based on `n_pos` instead of `n_tokens` (of course, there's no difference in non-M-RoPE models). + # However, I still keep `n_tokens` because if `n_pos` is used, the underlying system will assume it is a full-match and will skip eval and sample. + # chunk_n_pos = self._mtmd_cpp.mtmd_input_chunk_get_n_pos(chunk) # equals to max(t,h,w) for M-RoPE; equals to `n_tokens` otherwise + chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk) + + if media_items_cur < media_items_count: + # The C++ parser only sees identical placeholders (e.g., "<__media__>"). + # We MUST inject the actual media content's identity here. + real_media_url = media_items[media_items_cur]["url"] + # Vocabulary Positive forward: 0 to 248,319 (Qwen3.5) + # Generate a deterministic, unique negative ID for this specific image/audio. + # - zlib.crc32 ensures cross-platform and cross-run consistency (unlike Python's hash()). + # - We map it to a negative space (-100 to -16,777,316) to avoid colliding with + # positive text token IDs (e.g., Qwen3.5 vocab goes up to ~152k). + # This empowers `longest_token_prefix` to correctly identify and reuse cached images, + # while instantly breaking the match if the image content changes. + # media_id = - (zlib.crc32(real_media_url.encode('utf-8')) % (2**24)) - 100 + media_id = - (zlib.crc32(real_media_url.encode('utf-8')) & 0xFFFFFF) - 100 + last_media_id = media_id + media_items_cur += 1 + elif last_media_id is not None: + # video may expand into multiple image chunks from one media marker + media_id = last_media_id + else: + # Magic Negative Number as fallback :) + media_id = -314159 + + if self.verbose: + print(f"{self.log_prefix}(mtmd_input_chunk_media_id): chunk_n_tokens: {chunk_n_tokens}, media_id: {media_id}, ") + + chunk_token_spans.append((current_idx, current_idx + chunk_n_tokens, chunk, chunk_type, media_id)) + + # Pad the ledger with the pseudo-ID to mimic the physical space taken in the KV cache + full_prompt_ids.extend([media_id] * chunk_n_tokens) + current_idx += chunk_n_tokens + else: + raise TypeError(f"{self.log_prefix}(mtmd_input_chunk_get_type): Invalid chunk type, chunk_type = {chunk_type}.") + + return full_prompt_ids, chunk_token_spans, chunks, bitmap_cleanup + + except Exception as e: + # Ensure no useless pointers remain upon any failure + # Free chunks + if chunks is not None: + self._mtmd_cpp.mtmd_input_chunks_free(chunks) + chunks = None + # Free bitmaps + if len(bitmap_cleanup) > 0: + for bitmap in bitmap_cleanup: + self._mtmd_cpp.mtmd_bitmap_free(bitmap) + bitmap_cleanup = None + # Free videos + if len(video_cleanup) > 0: + for video_ctx in video_cleanup: + self._mtmd_cpp.mtmd_helper_video_free(video_ctx) + video_cleanup = None + + bitmaps = None + + raise e + + def __call__( + self, + *, + llama: llama_core.Llama, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + temperature: float = 0.2, + top_p: float = 0.95, + top_k: int = 40, + min_p: float = 0.05, + typical_p: float = 1.0, + stream: bool = False, + stop: Optional[Union[str, List[str]]] = [], + seed: Optional[int] = None, + response_format: Optional[ + llama_types.ChatCompletionRequestResponseFormat + ] = None, + max_tokens: Optional[int] = None, + present_penalty: float = 0.0, + frequency_penalty: float = 0.0, + repeat_penalty: float = 1.1, + top_n_sigma: float = -1.00, + mirostat_mode: int = 0, + mirostat_tau: float = 5.0, + mirostat_eta: float = 0.1, + xtc_threshold: float = 0.1, + xtc_probability: float = 0.0, + dry_multiplier: float = 0.0, + dry_base: float = 1.75, + dry_allowed_length: int = 2, + dry_penalty_last_n:int = 0, + dry_seq_breakers: list[str] = ["\n", ":", "\"", "*"], + adaptive_target : float = -1.0, + adaptive_decay : float = 0.9, + use_infill: bool = False, + model: Optional[str] = None, + logits_processor: Optional[llama_core.LogitsProcessorList] = None, + grammar: Optional[llama_grammar.LlamaGrammar] = None, + logit_bias: Optional[Dict[str, float]] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, + add_generation_prompt: bool = True, + reasoning_budget: int = -1, + reasoning_start: str = "", + reasoning_end: str = "", + reasoning_budget_message: Optional[str] = None, + reasoning_start_in_prompt: bool = False, + reasoning_start_max_tokens: Optional[int] = 32, + **kwargs, # type: ignore + ) -> Union[ + llama_types.CreateChatCompletionResponse, + Iterator[llama_types.CreateChatCompletionStreamResponse], + ]: + # 1. Initialize mtmd context + self._init_mtmd_context(llama) + assert self.mtmd_ctx is not None + + # 2. Concurrent Preprocessing & Ledger Construction + full_prompt_ids, chunk_token_spans, chunks, bitmap_cleanup = self._process_mtmd_prompt( + llama=llama, + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + add_generation_prompt=add_generation_prompt, + ) + + if self.verbose: + print(f"{self.log_prefix}(__call__): Prepared virtual token ledger of length {len(full_prompt_ids)}.", file=sys.stderr) + + try: + # 3. KV Cache Synchronization & State Rollback + # Compares the virtual ledger with physical history to prevent Cache Poisoning. + current_history = llama.input_ids[:llama.n_tokens].tolist() + longest_prefix = llama.longest_token_prefix(current_history, full_prompt_ids, self.verbose) + + if longest_prefix < llama.n_tokens: + if llama.is_hybrid and llama._hybrid_cache_mgr is not None: + if llama._hybrid_cache_mgr.max_checkpoints > 0: + if self.verbose: + print(f"{self.log_prefix}(__call__): Hybrid prefix mismatch (matched {longest_prefix}/{llama.n_tokens}). " + f"Searching for nearest checkpoint...", file=sys.stderr) + + best_ckpt = llama._hybrid_cache_mgr.find_best_checkpoint(full_prompt_ids, seq_id=0) + if best_ckpt and llama._hybrid_cache_mgr.restore_checkpoint(best_ckpt, seq_id=0): + llama.n_tokens = best_ckpt.pos + if self.verbose: + print(f"{self.log_prefix}(__call__): Successfully rolled back to checkpoint at pos {llama.n_tokens}.", file=sys.stderr) + else: + if self.verbose: + print(f"{self.log_prefix}(__call__): No suitable checkpoint found or restore failed. Clearing hybrid cache entirely.", file=sys.stderr) + llama._hybrid_cache_mgr.clear() + llama._ctx.memory_clear(True) + llama.n_tokens = 0 + else: + if self.verbose: + print(f"{self.log_prefix}(__call__): Hybrid cache enabled but max_checkpoints is 0. Clearing cache entirely.", file=sys.stderr) + llama._hybrid_cache_mgr.clear() + llama._ctx.memory_clear(True) + llama.n_tokens = 0 + else: + if self.verbose: + print(f"{self.log_prefix}(__call__): Prefix mismatch. Truncating KV cache from {llama.n_tokens} to {longest_prefix}.", file=sys.stderr) + llama._ctx.memory_seq_rm(0, longest_prefix, -1) + llama.n_tokens = longest_prefix + + n_past = llama.n_tokens + + for start_idx, end_idx, chunk_ptr, chunk_type, media_id in chunk_token_spans: + # Skip previously matched chunks + if end_idx <= n_past: + continue + + if self._is_text_chunk(chunk_type): + unprocessed_start = max(start_idx, n_past) - start_idx + n_tokens_out = ctypes.c_size_t() + tokens_ptr = self._mtmd_cpp.mtmd_input_chunk_get_tokens_text(chunk_ptr, ctypes.byref(n_tokens_out)) + + if tokens_ptr and n_tokens_out.value > 0: + all_tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)] + tokens_to_eval = all_tokens[unprocessed_start:] + + if tokens_to_eval: + if self.verbose: + print(f"{self.log_prefix}(__call__): Evaluating TEXT chunk ({len(tokens_to_eval)} tokens) at pos {llama.n_tokens}...", file=sys.stderr) + # Text evaluation delegates shift and chunking to native llama.eval + llama.eval(tokens_to_eval) + n_past = llama.n_tokens + + elif self._is_image_chunk(chunk_type) or self._is_audio_chunk(chunk_type): + chunk_n_tokens = self._mtmd_cpp.mtmd_input_chunk_get_n_tokens(chunk_ptr) + + if self.verbose: + media_str = "IMAGE" if self._is_image_chunk(chunk_type) else "AUDIO" + print(f"{self.log_prefix}(__call__): Evaluating {media_str} chunk ({chunk_n_tokens} tokens) at pos {llama.n_tokens}...", file=sys.stderr) + + # Stage 5: Multimodal Physical OOM Defense + if n_past + chunk_n_tokens > llama.n_ctx(): + if not llama._ctx.memory_can_shift(): + raise RuntimeError( + f"{self.log_prefix}(__call__): Context Shift is explicitly disabled by the C++ backend " + f"(n_pos_per_embd > 1 or incompatible M-RoPE). " + f"Multimodal chunk exceeded context limit(currently n_ctx={llama._n_ctx}), " + f"You MUST increase n_ctx to fit the dialogue." + ) + else: + # Safely discard oldest tokens while preserving system prompts + n_discard = (n_past + chunk_n_tokens) - llama.n_ctx() + llama.n_batch + n_keep = min(llama.n_keep, n_past) + n_discard = min(n_discard, n_past - n_keep) + + if n_discard <= 0: + raise RuntimeError(f"{self.log_prefix}(__call__): Critical Overflow. Not enough unpinned tokens to discard for Context Shift.") + + if self.verbose: + print(f"{self.log_prefix}(__call__): OOM risk detected. Shifting multimodal context: keeping {n_keep}, discarding {n_discard}...", file=sys.stderr) + + # Execute physical memory shift + llama._ctx.memory_seq_rm(0, n_keep, n_keep + n_discard) + llama._ctx.memory_seq_add(0, n_keep + n_discard, n_past, -n_discard) + + # Shift python virtual array to match + remaining_len = n_past - (n_keep + n_discard) + if remaining_len > 0: + llama.input_ids[n_keep : n_keep + remaining_len] = llama.input_ids[n_keep + n_discard : n_past] + + n_past -= n_discard + llama.n_tokens = n_past + + # Execute C++ Multimodal Black-box Extraction + new_n_past = llama_cpp_lib.llama_pos(0) + result = self._mtmd_cpp.mtmd_helper_eval_chunk_single( + self.mtmd_ctx, + llama._ctx.ctx, + chunk_ptr, + llama_cpp_lib.llama_pos(n_past), + llama_cpp_lib.llama_seq_id(0), + llama.n_batch, + True, # logits_last = True, drastically saves computational overhead + ctypes.byref(new_n_past) + ) + + if result != 0: + raise ValueError(f"{self.log_prefix}(mtmd_helper_eval_chunk_single): Media evaluation failed with error code {result}.") + + # Update Ledger with "Negative Reverse Vocabulary" IDs + llama.input_ids[n_past : new_n_past.value] = media_id + n_past = new_n_past.value + llama.n_tokens = n_past + + # Extract the final, perfectly synchronized prompt sequence + prompt = llama.input_ids[: llama.n_tokens].tolist() + + # End-of-Turn Checkpoint + # Anchors the state ONLY after the entire multi-modal turn is processed + if ( + llama.is_hybrid + and llama._hybrid_cache_mgr is not None + and llama._hybrid_cache_mgr.max_checkpoints > 0 + ): + if self.verbose: + print(f"{self.log_prefix}(__call__): [End-of-Turn Checkpoint] Anchoring full prompt state at pos {llama.n_tokens}.", file=sys.stderr) + + llama._hybrid_cache_mgr.save_checkpoint( + current_pos=llama.n_tokens, + tokens=prompt, + seq_id=0 + ) + finally: + # Cleanup chunks + if chunks is not None: + self._mtmd_cpp.mtmd_input_chunks_free(chunks) + chunks = None + # Cleanup bitmaps + if bitmap_cleanup: + for bitmap in bitmap_cleanup: + self._mtmd_cpp.mtmd_bitmap_free(bitmap) + bitmap_cleanup.clear() + bitmap_array = None + + # Handle response format and tools (same as before) + if response_format is not None and response_format["type"] == "json_object": + grammar = _grammar_for_response_format(response_format) + + # Convert legacy functions to tools + if functions is not None: + tools = [ + { + "type": "function", + "function": function, + } + for function in functions + ] + + # Convert legacy function_call to tool_choice + if function_call is not None: + if isinstance(function_call, str) and ( + function_call == "none" or function_call == "auto" + ): + tool_choice = function_call + if isinstance(function_call, dict) and "name" in function_call: + tool_choice = { + "type": "function", + "function": { + "name": function_call["name"], + }, + } + + tool = None + if ( + tool_choice is not None + and isinstance(tool_choice, dict) + and tools is not None + ): + name = tool_choice["function"]["name"] + tool = next((t for t in tools if t["function"]["name"] == name), None) + if tool is None: + raise ValueError(f"Tool choice '{name}' not found in tools.") + schema = tool["function"]["parameters"] + try: + # create grammar from json schema + grammar = llama_grammar.LlamaGrammar.from_json_schema( + json.dumps(schema), verbose=llama.verbose + ) + except Exception as e: + if llama.verbose: + print(str(e), file=sys.stderr) + grammar = llama_grammar.LlamaGrammar.from_string( + llama_grammar.JSON_GBNF, verbose=llama.verbose + ) + + completion_or_chunks = llama.create_completion( + prompt=prompt, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + typical_p=typical_p, + logprobs=top_logprobs if logprobs else None, + stream=stream, + stop=stop, + seed=seed, + max_tokens=max_tokens, + present_penalty=present_penalty, + frequency_penalty=frequency_penalty, + repeat_penalty=repeat_penalty, + top_n_sigma=top_n_sigma, + mirostat_mode=mirostat_mode, + mirostat_tau=mirostat_tau, + mirostat_eta=mirostat_eta, + xtc_threshold=xtc_threshold, + xtc_probability=xtc_probability, + dry_multiplier=dry_multiplier, + dry_base=dry_base, + dry_allowed_length=dry_allowed_length, + dry_penalty_last_n=dry_penalty_last_n, + dry_seq_breakers=dry_seq_breakers, + adaptive_target=adaptive_target, + adaptive_decay=adaptive_decay, + use_infill=use_infill, + model=model, + logits_processor=logits_processor, + grammar=grammar, + logit_bias=logit_bias, + reasoning_budget=reasoning_budget, + reasoning_start=reasoning_start, + reasoning_end=reasoning_end, + reasoning_budget_message=reasoning_budget_message, + reasoning_start_in_prompt=reasoning_start_in_prompt, + reasoning_start_max_tokens=reasoning_start_max_tokens, + ) + + if tool is not None: + tool_name = tool["function"]["name"] + return _convert_completion_to_chat_function( + tool_name, completion_or_chunks, stream + ) + return _convert_completion_to_chat(completion_or_chunks, stream=stream) + + def load_media(self, media_url: str, media_type: str) -> bytes: + """ + Unified dispatcher for loading media payloads. + Routes the URL/URI to the specific image, audio, or video processor based on the media_type. + """ + if media_type == "image": + return self._load_image(media_url) + + elif media_type == "audio": + audio_bytes = self._load_bytes(media_url, timeout=15, kind="audio") + try: + self.detect_audio_format(audio_bytes) + except ValueError as e: + raise ValueError(f"{self.log_prefix}(load_media): {e}") + return audio_bytes + + elif media_type == "video": + return self._load_bytes(media_url, timeout=30, kind="video") + + else: + raise ValueError(f"{self.log_prefix}(load_media): Unknown media type '{media_type}'") + + @staticmethod + def detect_audio_format(audio_bytes: bytes) -> str: + """ + Pure utility function: Detects the audio format from magic bytes. + Strictly translated from llama.cpp's `is_audio_file` to ensure 100% compatibility + and avoid false positives (e.g., AVI files disguised as RIFF). + """ + length = len(audio_bytes) + + if length < 12: + raise ValueError("Audio data is corrupted or too small (less than 12 bytes).") + + # RIFF & WAVE magic bytes verification + is_wav = audio_bytes.startswith(b"RIFF") and audio_bytes[8:12] == b"WAVE" + + # ID3 metadata or MPEG sync word verification + is_mp3 = length >= 3 and ( + audio_bytes.startswith(b"ID3") or + (audio_bytes[0] == 0xFF and (audio_bytes[1] & 0xE0) == 0xE0) + ) + + # FLAC magic bytes verification + is_flac = audio_bytes.startswith(b"fLaC") + + if is_wav: + return "wav" + elif is_mp3: + return "mp3" + elif is_flac: + return "flac" + else: + raise ValueError( + "Unsupported audio format detected via magic bytes. " + "The underlying C++ miniaudio backend ONLY supports WAV, MP3, and FLAC." + ) + + DEFAULT_HTTP_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/148.0.0.0 Safari/537.36" + ), + } + + @staticmethod + def _load_bytes(media_url: str, timeout: int = 15, kind: str = "media") -> bytes: + """ + Load raw bytes from a data URI, local file path, or remote HTTP/HTTPS URL. + """ + media_bytes = b"" + + # 1. Handle data URI + if media_url.strip().startswith("data:"): + comma_pos = media_url.find(",") + if comma_pos == -1: + raise ValueError("Invalid data URI: missing comma separator") + + base64_data = media_url[comma_pos + 1:] + media_bytes = base64.b64decode(base64_data) + + # 2. Handle local file path + elif os.path.exists(media_url): + with open(media_url, "rb") as f: + media_bytes = f.read() + + # 3. Handle remote URL via HTTP/HTTPS + else: + req = urllib.request.Request( + media_url, + headers=MTMDChatHandler.DEFAULT_HTTP_HEADERS, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as f: + media_bytes = f.read() + except (URLError, HTTPError) as e: + raise ConnectionError(f"Failed to download {kind} from {media_url}: {e}") + + if not media_bytes: + raise ValueError(f"Empty {kind} data received") + + return media_bytes + + @staticmethod + def _load_image(image_url: str) -> bytes: + """ + Load an image from either a URL or a data URI and return it as JPEG bytes. + + Supports: + - Remote images via HTTP/HTTPS (with proper User-Agent) + - Data URIs (base64-encoded, e.g., data:image/png;base64,...) + - Images with alpha channel (PNG, WebP, etc.) ā†’ automatically composites on white/black background + - Any format that Pillow can open. See: https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html + + Returns: + JPEG-encoded bytes (quality=95) in RGB mode, suitable for most vision models. + """ + # 1. Load image bytes from image_url + image_bytes = MTMDChatHandler._load_bytes( + image_url, + timeout=15, + kind="image", + ) + + # 2. Check if image_bytes is empty. + if not image_bytes: + raise ValueError("Empty image data received") + + # 3. Open image with Pillow + try: + from PIL import Image, ImageStat + except ImportError: + raise ImportError("Pillow is required for image processing. Install with: pip install pillow") + + import io + image = Image.open(io.BytesIO(image_bytes)) + + # 4. Handle transparency (RGBA, LA, P with transparency, etc.) + if image.mode in ("RGBA", "LA", "PA") or (image.mode == "P" and "transparency" in image.info): + # Use alpha channel as mask + if image.mode == "P": + image = image.convert("RGBA") + + alpha = image.split()[-1] # Last channel is alpha + # Compute average brightness of visible (non-transparent) pixels + stat = ImageStat.Stat(image.convert("L"), mask=alpha) + + # Choose background: white for dark content, black for bright content + bg_color = (255, 255, 255) # white + if stat.count[0] > 0 and stat.mean[0] > 127: + bg_color = (0, 0, 0) # black + + background = Image.new("RGB", image.size, bg_color) + background.paste(image, mask=alpha) + image = background + + # 5. Ensure RGB mode for formats like CMYK, palette, etc. + elif image.mode != "RGB": + image = image.convert("RGB") + + # 6. Save as high-quality JPEG, suitable for most vision models. + output = io.BytesIO() + image.save(output, format="JPEG", quality=95, optimize=True, progressive=True) + return output.getvalue() + + @classmethod + def from_pretrained( + cls, + repo_id: str, + filename: Optional[str], + local_dir: Optional[Union[str, os.PathLike[str]]] = None, + local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto", + cache_dir: Optional[Union[str, os.PathLike[str]]] = None, + **kwargs: Any, + ) -> "MTMDChatHandler": + import fnmatch + from pathlib import Path + + try: + from huggingface_hub import hf_hub_download, HfFileSystem # type: ignore + from huggingface_hub.utils import validate_repo_id # type: ignore + except ImportError: + raise ImportError( + "Llama.from_pretrained requires the huggingface_hub package. " + "You can install it with `pip install --upgrade huggingface_hub`." + ) + + validate_repo_id(repo_id) + + hffs = HfFileSystem() + + files = [ + file["name"] if isinstance(file, dict) else file + for file in hffs.ls(repo_id) # type: ignore + ] + + # split each file into repo_id, subfolder, filename + file_list: List[str] = [] + for file in files: + rel_path = Path(file).relative_to(repo_id) + file_list.append(str(rel_path)) + + matching_files = [file for file in file_list if fnmatch.fnmatch(file, filename)] # type: ignore + + if len(matching_files) == 0: + raise ValueError( + f"No file found in {repo_id} that match {filename}\n\n" + f"Available Files:\n{json.dumps(file_list)}" + ) + + if len(matching_files) > 1: + raise ValueError( + f"Multiple files found in {repo_id} matching {filename}\n\n" + f"Available Files:\n{json.dumps(files)}" + ) + + (matching_file,) = matching_files + + subfolder = str(Path(matching_file).parent) + filename = Path(matching_file).name + + # download the file + hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder, + local_dir=cast(Union[str, Path, None], local_dir), + local_dir_use_symlinks=local_dir_use_symlinks, + cache_dir=cast(Union[str, Path, None], cache_dir), + ) + + if local_dir is None: + model_path = hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder, + local_dir=local_dir, + local_dir_use_symlinks=local_dir_use_symlinks, + cache_dir=cast(Union[str, Path, None], cache_dir), + local_files_only=True, + ) + else: + model_path = os.path.join(local_dir, filename) + + return cls( + mmproj_path=model_path, + **kwargs, + ) + +# Experiments are not recommended for this purpose at this time. +class GenericMTMDChatHandler(MTMDChatHandler): + KNOWN_MEDIA_TAGS = [ + "<|image_pad|>", + "<|audio_pad|>", + "<|video_pad|>", + "<|image|>", + "<|audio|>", + "<|video|>", + "[IMG]" + ] + + def __init__( + self, + chat_format: str, + mmproj_path: str, + verbose: bool = True, + **kwargs + ) -> None: + + self.chat_format = chat_format + if self.chat_format is None: + raise ValueError("Failed to get model chat template automatically.") + + self.verbose = verbose + if self.verbose: + print(f"Got chat template from model:\n```jinja\n{self.chat_format}\n```", flush = True) + + super().__init__(mmproj_path = mmproj_path, verbose = verbose, **kwargs) + + def __call__(self, **kwargs): + self._chat_format_parser_tags = [tag for tag in self.KNOWN_MEDIA_TAGS if tag in self.chat_format] + + if self.verbose: + print(f"{self.log_prefix} - Start processing") + + # Use parent implementation + return super().__call__(**kwargs) + +class Llava15ChatHandler(MTMDChatHandler): + CHAT_FORMAT = ( + "{% for message in messages %}" + "{% if message.role == 'system' %}" + "{{ message.content }}" + "{% endif %}" + + "{% if message.role == 'user' %}" + "{% if message.content is string %}" + "\nUSER: {{ message.content }}" + "{% elif message.content is iterable %}" + "\nUSER: " + "{% for content in message.content %}" + "{% if content.type == 'image_url' %}" + "{{ content.image_url if content.image_url is string else content.image_url.url }}" + "{% endif %}" + "{% endfor %}" + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "{% endif %}" + + "{% if message.role == 'assistant' and message.content is not none %}" + "\nASSISTANT: {{ message.content }}" + "{% endif %}" + "{% endfor %}" + + "{% if add_generation_prompt %}" + "\nASSISTANT: " + "{% endif %}" + ) + + +class ObsidianChatHandler(MTMDChatHandler): + # Prompt Format + # The model followed ChatML format. However, with ### as the seperator + + # <|im_start|>user + # What is this sign about?\n + # ### + # <|im_start|>assistant + # The sign is about bullying, and it is placed on a black background with a red background. + # ### + + CHAT_FORMAT = ( + "{% for message in messages %}" + # System message + "{% if message.role == 'system' %}" + "<|im_start|>system\n" + "{{ message.content }}\n" + "###\n" + "{% endif %}" + # User message + "{% if message.role == 'user' %}" + "<|im_start|>user\n" + "{% if message.content is string %}" + "{{ message.content }}" + "{% endif %}" + "{% if message.content is iterable %}" + "{% for content in message.content %}" + "{% if content.type == 'image_url' and content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.type == 'image_url' and content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endfor %}" + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "###\n" + "{% endif %}" + # Assistant message + "{% if message.role == 'assistant' %}" + "<|im_start|>assistant\n" + "{{ message.content }}" + "###\n" + "{% endif %}" + "{% endfor %}" + # Generation prompt + "{% if add_generation_prompt %}" + "<|im_start|>assistant\n" + "{% endif %}" + ) + + +class MoondreamChatHandler(MTMDChatHandler): + # Chat Format: + # f"\n\n{chat_history}Question: {question}\n\nAnswer:" + CHAT_FORMAT = ( + "{% for message in messages %}" + "{% if message.role == 'user' %}" + "{% if message.content is iterable %}" + # + "{% for content in message.content %}" + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}\n\n" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{{ content.image_url.url }}\n\n" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + # Question: + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "Question: {{ content.text }}\n\n" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + # Question: + "{% if message.content is string %}" + "Question: {{ message.content }}\n\n" + "{% endif %}" + "{% endif %}" + # Answer: + "{% if message.role == 'assistant' %}" + "Answer:{{ message.content }}\n\n" + "{% endif %}" + "{% endfor %}" + # Generation prompt + "{% if add_generation_prompt %}" + "Answer:" + "{% endif %}" + ) + + +class Llava16ChatHandler(MTMDChatHandler): + # Example prompt + # "DEFAULT_SYSTEM_MESSAGE + USER: \nWhat is shown in this image? ASSISTANT:" + + CHAT_FORMAT = ( + "{% for message in messages %}" + "{% if message.role == 'system' %}" + "{{ message.content }}" + "{% endif %}" + "{% if message.role == 'user' %}" + "{% if message.content is iterable %}" + # + "{% for content in message.content %}" + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}\n" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{{ content.image_url.url }}\n" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + # Question: + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + # Question: + "{% if message.content is string %}" + "{{ message.content }}" + "{% endif %}" + "{% endif %}" + # Answer: + "{% if message.role == 'assistant' %}" + "{{ message.content }}" + "{% endif %}" + "{% endfor %}" + # Generation prompt + "{% if add_generation_prompt %}" + "Answer:" + "{% endif %}" + ) + + +class NanoLlavaChatHandler(MTMDChatHandler): + # Prompt Format + # The model follow the ChatML standard, however, without \n at the end of <|im_end|>: + + # <|im_start|>system + # Answer the question<|im_end|><|im_start|>user + # + # What is the picture about?<|im_end|><|im_start|>assistant + DEFAULT_SYSTEM_MESSAGE = "Answer the question" + + CHAT_FORMAT = ( + "{% for message in messages %}" + # System message + "{% if message.role == 'system' %}" + "<|im_start|>system\n" + "{{ message.content }}" + "<|im_end|>" + "{% endif %}" + # User message + "{% if message.role == 'user' %}" + "<|im_start|>user\n" + "{% if message.content is string %}" + "{{ message.content }}" + "{% endif %}" + "{% if message.content is iterable %}" + "{% for content in message.content %}" + "{% if content.type == 'image_url' and content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.type == 'image_url' and content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endfor %}" + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "<|im_end|>" + "{% endif %}" + # Assistant message + "{% if message.role == 'assistant' %}" + "<|im_start|>assistant\n" + "{{ message.content }}" + "<|im_end|>" + "{% endif %}" + "{% endfor %}" + # Generation prompt + "{% if add_generation_prompt %}" + "<|im_start|>assistant\n" + "{% endif %}" + ) + + +class Llama3VisionAlphaChatHandler(MTMDChatHandler): + # question = "" + q + + # prompt = f"<|start_header_id|>user<|end_header_id|>\n\n{question}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n" + + CHAT_FORMAT = ( + "{% for message in messages %}" + "<|start_header_id|>" + "{% if message.role == 'user' %}" + "user<|end_header_id|>\n\n" + "{% if message.content is iterable %}" + # + "{% for content in message.content %}" + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{{ content.image_url }}" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{{ content.image_url.url }}" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + # Question: + "{% for content in message.content %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + # Question: + "{% if message.content is string %}" + "{{ message.content }}" + "{% endif %}" + "{% endif %}" + # Answer: + "{% if message.role == 'assistant' %}" + "assistant<|end_header_id|>\n\n" + "{{ message.content }}" + "{% endif %}" + "<|eot_id|>" + "{% endfor %}" + # Generation prompt + "{% if add_generation_prompt %}" + "<|start_header_id|>assistant<|end_header_id|>\n\n" + "{% endif %}" + ) + + +# alias +Llama3VisionAlpha = Llama3VisionAlphaChatHandler + + +class MiniCPMv26ChatHandler(MTMDChatHandler): + + CHAT_FORMAT = ( + "{% set image_count = namespace(value=0) %}" + "{% for message in messages %}" + "{% if loop.first and messages[0]['role'] != 'system' %}" + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "{% endif %}" + "<|im_start|>{{ message['role'] }}\n" + "{% if message['content'] is iterable %}" + "{% for content in message['content'] %}" + "{% if content.type == 'image_url' %}" + "{% if content.image_url is string %}" + "{% set image_count.value = image_count.value + 1 %}" + "{{ image_count.value }}: {{ content.image_url }}" + "{% endif %}" + "{% if content.image_url is mapping %}" + "{% set image_count.value = image_count.value + 1 %}" + "{{ image_count.value }}: {{ content.image_url.url }}" + "{% endif %}" + "{% endif %}" + "{% endfor %}" + + "{% for content in message['content'] %}" + "{% if content.type == 'text' %}" + "{{ content.text }}" + "{% endif %}" + "{% endfor %}" + "{% endif %}" + "{% if message['content'] is string %}" + "{{ message['content'] }}" + "{% endif %}" + "<|im_end|>\n" + "{% endfor %}" + "{% if add_generation_prompt %}" + "<|im_start|>assistant\n" + "{% endif %}" + ) + + +class MiniCPMv45ChatHandler(MTMDChatHandler): + """ + Handler for MiniCPM-V 4.5 models. + + Supports: + - Multi-step tool calls with and XML tags. + - Integrated reasoning (thinking) process with tags. + - Specialized system prompt handling with tool definitions. + - Global image numbering for multi-image processing. + """ + + # Model specific control tokens + MINICPMV_BOS_TOKEN = "<|im_start|>" + MINICPMV_EOS_TOKEN = "<|im_end|>" + MINICPMV_PAD_TOKEN = "<|endoftext|>" + + # Image placeholder tags + MINICPMV_IMAGE_START_TOKEN = "" + MINICPMV_IMAGE_END_TOKEN = "" + MINICPMV_IMAGE_ID_START_TOKEN = "" + MINICPMV_IMAGE_ID_END_TOKEN = "" + + CHAT_FORMAT = ( + # --- 1. First System Message & Tools Definitions --- + "{%- if tools %}" + "{{- '" + MINICPMV_BOS_TOKEN + "system\\n' }}" + "{%- if messages[0].role == 'system' %}{{- messages[0].content + '\\n\\n' }}{%- endif %}" + "{{- '# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\n' }}" + "{{- 'You are provided with function signatures within XML tags:\\n' }}" + "{%- for tool in tools %}{{- '\\n' + (tool | tojson) }}{%- endfor %}" + "{{- '\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\"name\": , \"arguments\": }\\n" + MINICPMV_EOS_TOKEN + "\\n' }}" + "{%- elif messages[0].role == 'system' %}" + "{{- '" + MINICPMV_BOS_TOKEN + "system\\n' + messages[0].content + '" + MINICPMV_EOS_TOKEN + "\\n' }}" + "{%- endif %}" + + # --- 2. Message Stream Processing --- + "{% set image_count = namespace(value=0) %}" + "{%- for message in messages %}" + # --- Unified Role Handling (User, Assistant, and subsequent Systems) --- + "{%- if message.role in ['user', 'assistant'] or (message.role == 'system' and not loop.first) %}" + "{{- '" + MINICPMV_BOS_TOKEN + "' + message.role + '\\n' }}" + + "{%- set content = message.content %}" + "{%- if content is not string %}" + "{%- set ns = namespace(content_str='') %}" + "{%- for item in content %}" + # --- Explicit image_url type and value checking --- + "{%- if item.type == 'image_url' %}" + "{%- set image_url = item.image_url if item.image_url is string else item.image_url.url %}" + "{%- set image_count.value = image_count.value + 1 %}" + # Format: N: IMAGE_URL + "{%- set ns.content_str = ns.content_str + '' + (image_count.value | string) + ': ' + image_url + '' %}" + "{%- elif item.type == 'text' %}" + "{%- set ns.content_str = ns.content_str + item.text %}" + "{%- endif %}" + "{%- endfor %}" + "{%- set content = ns.content_str %}" + "{%- endif %}" + + "{{- content -}}" + + # Append tool_calls to assistant messages if they exist + "{%- if message.role == 'assistant' and message.tool_calls %}" + "{%- for tool_call in message.tool_calls %}" + "{%- set tc = tool_call.function if tool_call.function else tool_call %}" + "{{- '\\n\\n{\"name\": \"' + tc.name + '\", \"arguments\": ' }}" + "{{- tc.arguments if tc.arguments is string else tc.arguments | tojson }}" + "{{- '}\\n' }}" + "{%- endfor %}" + "{%- endif %}" + "{{- '" + MINICPMV_EOS_TOKEN + "\\n' }}" + + # --- Specialized Tool Response Handling --- + # Group consecutive tool responses under a single user-like block + "{%- elif message.role == 'tool' %}" + "{%- if loop.first or (messages[loop.index0 - 1].role != 'tool') %}" + "{{- '" + MINICPMV_BOS_TOKEN + "user' }}" + "{%- endif %}" + "{{- '\\n\\n' + message.content + '\\n' }}" + "{%- if loop.last or (messages[loop.index0 + 1].role != 'tool') %}" + "{{- '" + MINICPMV_EOS_TOKEN + "\\n' }}" + "{%- endif %}" + "{%- endif %}" + "{%- endfor %}" + + # --- 3. Generation Prompt --- + "{%- if add_generation_prompt %}" + "{{- '" + MINICPMV_BOS_TOKEN + "assistant\\n' }}" + # Handle thinking/reasoning block visibility based on configuration + "{%- if enable_thinking is defined and enable_thinking is false %}" + "{{- '\\n\\n\\n\\n' }}" + "{%- elif enable_thinking is defined and enable_thinking is true %}" + "{{- '\\n' }}" + "{%- endif %}" + "{%- endif %}" + ) + + def __init__(self, enable_thinking: bool = True, **kwargs): + """ + Initializes the MiniCPM-V 4.5 Handler. + + Args: + enable_thinking (bool): If True, model generates reasoning before the final answer. + **kwargs: Additional arguments for the base MTMDChatHandler. + """ + self.enable_thinking = enable_thinking + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Inject thinking control flag into the template + self.extra_template_arguments["enable_thinking"] = self.enable_thinking + + # Set stop token patch + kwargs['stop'] = [self.MINICPMV_EOS_TOKEN, self.MINICPMV_PAD_TOKEN] + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") + return super().__call__(**kwargs) + + +class MiniCPMV46ChatHandler(MTMDChatHandler): + """ + Handler for MiniCPM-V-4.6 models. + + Features: + - Aligned with official tokenizer_config.json special tokens. + - Custom `<|image_pad|>` and `<|video_pad|>` multimodal tokens. + - Integrated MTMD-style URL and Base64 injection for visual content. + - Specialized `` and `` block generation. + - Autonomously folds previous reasoning paths using `last_query_index`. + - Toggles `` block generation via `enable_thinking` (Defaults to False). + """ + + # Core tokens + MINICPM_BOS_TOKEN = "<|im_start|>" + MINICPM_EOS_TOKEN = "<|im_end|>" + MINICPM_PAD_TOKEN = "<|endoftext|>" + + # Vision tokens + MINICPM_VISION_BOS_TOKEN = "<|vision_start|>" + MINICPM_VISION_EOS_TOKEN = "<|vision_end|>" + MINICPM_IMAGE_TOKEN = "<|image_pad|>" + MINICPM_VIDEO_TOKEN = "<|video_pad|>" + + CHAT_FORMAT = ( + "{%- if enable_thinking is not defined -%}\n" + " {%- set enable_thinking = false -%}\n" + "{%- endif -%}\n" + "{%- macro render_content(content, is_system_content=false) -%}\n" + " {%- if content is string -%}\n" + " {{- content -}}\n" + " {%- elif content is iterable and content is not mapping -%}\n" + " {%- set ns = namespace(parts=[]) -%}\n" + " {%- for item in content -%}\n" + " {%- if 'image' in item or 'image_url' in item or item.type == 'image' -%}\n" + " {%- if is_system_content -%}\n" + " {{- raise_exception('System message cannot contain images.') -}}\n" + " {%- endif -%}\n" + " {%- set url_val = '' -%}\n" + " {%- if item.type == 'image_url' -%}\n" + " {%- set url_val = item.image_url if item.image_url is string else item.image_url.url -%}\n" + " {%- endif -%}\n" + " {%- set ns.parts = ns.parts + ['<|image_pad|>' + url_val] -%}\n" + # " {%- elif 'video' in item or 'video_url' in item or item.type == 'video' -%}\n" + # " {%- if is_system_content -%}\n" + # " {{- raise_exception('System message cannot contain videos.') -}}\n" + # " {%- endif -%}\n" + # " {%- set url_val = '' -%}\n" + # " {%- if item.type == 'video_url' -%}\n" + # " {%- set url_val = item.video_url if item.video_url is string else item.video_url.url -%}\n" + # " {%- endif -%}\n" + # " {%- set ns.parts = ns.parts + ['<|video_pad|>' + url_val] -%}\n" + " {%- elif 'text' in item -%}\n" + " {%- set ns.parts = ns.parts + [item.text] -%}\n" + " {%- else -%}\n" + " {{- raise_exception('Unexpected item type in content.') -}}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {{- ns.parts | join('\\n') -}}\n" + " {%- elif content is none or content is undefined -%}\n" + " {{- '' -}}\n" + " {%- else -%}\n" + " {{- raise_exception('Unexpected content type.') -}}\n" + " {%- endif -%}\n" + "{%- endmacro -%}\n" + "{%- if not messages %}\n" + " {{- raise_exception('No messages provided.') }}\n" + "{%- endif %}\n" + "{%- if tools and tools is iterable and tools is not mapping %}\n" + " {{- '<|im_start|>system\\n' }}\n" + " {{- '# Tools\\n\\nYou have access to the following functions:\\n\\n' }}\n" + " {%- for tool in tools %}\n" + " {{- '\\n' }}\n" + " {{- tool | tojson }}\n" + " {%- endfor %}\n" + " {{- '\\n' }}\n" + " {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n' }}\n" + " {%- if messages[0].role == 'system' %}\n" + " {%- set content = render_content(messages[0].content, true)|trim %}\n" + " {%- if content %}\n" + " {{- '\\n\\n' + content }}\n" + " {%- endif %}\n" + " {%- endif %}\n" + " {{- '<|im_end|>\\n' }}\n" + "{%- else %}\n" + " {%- if messages[0].role == 'system' %}\n" + " {%- set content = render_content(messages[0].content, true)|trim %}\n" + " {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n" + " {%- endif %}\n" + "{%- endif %}\n" + "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n" + "{%- for message in messages[::-1] %}\n" + " {%- set index = (messages|length - 1) - loop.index0 %}\n" + " {%- if ns.multi_step_tool and message.role == 'user' %}\n" + " {%- set content = render_content(message.content)|trim %}\n" + " {%- if not(content.startswith('') and content.endswith('')) %}\n" + " {%- set ns.multi_step_tool = false %}\n" + " {%- set ns.last_query_index = index %}\n" + " {%- endif %}\n" + " {%- endif %}\n" + "{%- endfor %}\n" + "{%- if ns.multi_step_tool %}\n" + " {{- raise_exception('No user query found in messages.') }}\n" + "{%- endif %}\n" + "{%- for message in messages %}\n" + " {%- set content = render_content(message.content)|trim %}\n" + " {%- if message.role == 'system' %}\n" + " {%- if not loop.first %}\n" + " {{- raise_exception('System message must be at the beginning.') }}\n" + " {%- endif %}\n" + " {%- elif message.role == 'user' %}\n" + " {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n" + " {%- elif message.role == 'assistant' %}\n" + " {%- set reasoning_content = '' %}\n" + " {%- if message.reasoning_content is string %}\n" + " {%- set reasoning_content = message.reasoning_content %}\n" + " {%- else %}\n" + " {%- if '' in content %}\n" + " {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n" + " {%- set content = content.split('')[-1].lstrip('\\n') %}\n" + " {%- endif %}\n" + " {%- endif %}\n" + " {%- set reasoning_content = reasoning_content|trim %}\n" + " {%- if loop.index0 > ns.last_query_index %}\n" + " {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n\\n' + content }}\n" + " {%- else %}\n" + " {{- '<|im_start|>' + message.role + '\\n' + content }}\n" + " {%- endif %}\n" + " {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n" + " {%- for tool_call in message.tool_calls %}\n" + " {%- if tool_call.function is defined %}\n" + " {%- set tool_call = tool_call.function %}\n" + " {%- endif %}\n" + " {%- if loop.first %}\n" + " {%- if content|trim %}\n" + " {{- '\\n\\n\\n\\n' }}\n" + " {%- else %}\n" + " {{- '\\n\\n' }}\n" + " {%- endif %}\n" + " {%- else %}\n" + " {{- '\\n\\n\\n' }}\n" + " {%- endif %}\n" + " {%- if tool_call.arguments is defined %}\n" + " {%- for args_name, args_value in tool_call.arguments|items %}\n" + " {{- '\\n' }}\n" + " {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n" + " {{- args_value }}\n" + " {{- '\\n\\n' }}\n" + " {%- endfor %}\n" + " {%- endif %}\n" + " {{- '\\n' }}\n" + " {%- endfor %}\n" + " {%- endif %}\n" + " {{- '<|im_end|>\\n' }}\n" + " {%- elif message.role == 'tool' %}\n" + " {%- if loop.previtem and loop.previtem.role != 'tool' %}\n" + " {{- '<|im_start|>user' }}\n" + " {%- endif %}\n" + " {{- '\\n\\n' }}\n" + " {{- content }}\n" + " {{- '\\n' }}\n" + " {%- if not loop.last and loop.nextitem.role != 'tool' %}\n" + " {{- '<|im_end|>\\n' }}\n" + " {%- elif loop.last %}\n" + " {{- '<|im_end|>\\n' }}\n" + " {%- endif %}\n" + " {%- else %}\n" + " {{- raise_exception('Unexpected message role.') }}\n" + " {%- endif %}\n" + "{%- endfor %}\n" + "{%- if add_generation_prompt %}\n" + " {{- '<|im_start|>assistant\\n' }}\n" + " {%- if enable_thinking is defined and enable_thinking is false %}\n" + " {{- '\\n\\n\\n\\n' }}\n" + " {%- else %}\n" + " {{- '\\n' }}\n" + " {%- endif %}\n" + "{%- endif %}\n" + ) + + def __init__(self, enable_thinking: bool = True, **kwargs): + """ + Initializes the MiniCPM-V-4.6 Handler. + + Args: + enable_thinking (bool): Controls whether to open a `` block for reasoning. + Defaults to False as per the standard template logic. + """ + self.enable_thinking = enable_thinking + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Inject the thinking variable into the Jinja environment + self.extra_template_arguments["enable_thinking"] = self.enable_thinking + + # MiniCPM uses standard <|im_end|> ChatML stop formatting + kwargs['stop'] = [self.MINICPM_PAD_TOKEN, self.MINICPM_EOS_TOKEN] + + if self.verbose: + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") + + return super().__call__(**kwargs) + + +class Gemma3ChatHandler(MTMDChatHandler): + + GEMMA3_BOI_TOKEN = "" + GEMMA3_EOI_TOKEN = "" + GEMMA3_BOS_TOKEN = "" + GEMMA3_EOS_TOKEN = "" + + CHAT_FORMAT = ( + "{% if messages[0]['role'] == 'system' %}" + "{% set loop_messages = messages[1:] %}" + "{% if messages[0]['content'] is string %}" + "{% set first_user_prefix = messages[0]['content'] + '\n\n' %}" + "{% else %}" + "{% set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' %}" + "{% endif %}" + "{% else %}" + "{% set loop_messages = messages %}" + "{% set first_user_prefix = '' %}" + "{% endif %}" + + "{% for message in loop_messages %}" + "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}" + "{{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}" + "{% endif %}" + + "{% if message['role'] == 'assistant' %}" + "{% set role = 'model' %}" + "{% else %}" + "{% set role = message['role'] %}" + "{% endif %}" + + "{{ '' + role + '\n' + (first_user_prefix if loop.first else '') }}" + + "{% if message['content'] is string %}" + "{{ message['content'] | trim }}" + "{% elif message['content'] is iterable %}" + "{% for item in message['content'] %}" + "{% if item['type'] == 'image_url' and item['image_url'] is string %}" + "{{ '' + item['image_url'] + '' }}" + "{% elif item['type'] == 'image_url' and item['image_url'] is mapping %}" + "{{ '' + item['image_url']['url'] + '' }}" + "{% elif item['type'] == 'text' %}" + "{{ item['text'] | trim }}" + "{% endif %}" + "{% endfor %}" + "{% else %}" + "{{ raise_exception('Invalid content type') }}" + "{% endif %}" + + "\n" + "{% endfor %}" + + "{% if add_generation_prompt %}" + "model\n" + "{% endif %}" + ) + + +class Gemma4ChatHandler(MTMDChatHandler): + """ + Handler for Gemma 4 models. + + Note on `enable_thinking`: + The `enable_thinking` toggle is currently ONLY supported by Gemma4 31B and 26BA4B models. + It is NOT supported by Gemma4 E2B and E4B models. + + [Important Note for Audio Processing!] + It is recommended to use BF16 mmproj for Gemma4 E2B and E4B models. + Other quantizations are known to have degraded performance; + ref comment: https://github.com/ggml-org/llama.cpp/pull/21421#issuecomment-4230306463 + """ + + # The special token in Gemma 4 + GEMMA4_BOI_TOKEN = "<|image>" + GEMMA4_EOI_TOKEN = "" + GEMMA4_BOA_TOKEN = "<|audio>" + GEMMA4_EOA_TOKEN = "" + GEMMA4_BOS_TOKEN = "" + GEMMA4_EOS_TOKEN = "" + GEMMA4_SOT_TOKEN = "<|turn>" + GEMMA4_EOT_TOKEN = "" + GEMMA4_SOC_TOKEN = "<|channel>" + GEMMA4_EOC_TOKEN = "" + GEMMA4_STC_TOKEN = "<|tool_call>" + GEMMA4_ETC_TOKEN = "" + GEMMA4_STD_TOKEN = "<|tool>" + GEMMA4_ETD_TOKEN = "" + GEMMA4_STR_TOKEN = "<|tool_response>" + GEMMA4_ETR_TOKEN = "" + + CHAT_FORMAT = ( + "{%- macro format_parameters(properties, required, filter_keys=false) -%}\n" + " {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n" + " {%- set ns = namespace(found_first=false) -%}\n" + " {%- for key, value in properties | dictsort -%}\n" + " {%- set add_comma = false -%}\n" + " {%- if not filter_keys or key not in standard_keys -%}\n" + " {%- if ns.found_first %},{% endif -%}\n" + " {%- set ns.found_first = true -%}\n" + " {{ key }}:{\n" + " {%- if value['description'] -%}\n" + " description:<|\"|>{{ value['description'] }}<|\"|>\n" + " {%- set add_comma = true -%}\n" + " {%- endif -%}\n" + " {%- if value['type'] | upper == 'STRING' -%}\n" + " {%- if value['enum'] -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " enum:{{ format_argument(value['enum']) }}\n" + " {%- endif -%}\n" + " {%- elif value['type'] | upper == 'ARRAY' -%}\n" + " {%- if value['items'] is mapping and value['items'] -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " items:{\n" + " {%- set ns_items = namespace(found_first=false) -%}\n" + " {%- for item_key, item_value in value['items'] | dictsort -%}\n" + " {%- if item_value is not none -%}\n" + " {%- if ns_items.found_first %},{% endif -%}\n" + " {%- set ns_items.found_first = true -%}\n" + " {%- if item_key == 'properties' -%}\n" + " properties:{\n" + " {%- if item_value is mapping -%}\n" + " {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n" + " {%- endif -%}\n" + " }\n" + " {%- elif item_key == 'required' -%}\n" + " required:[\n" + " {%- for req_item in item_value -%}\n" + " <|\"|>{{- req_item -}}<|\"|>\n" + " {%- if not loop.last %},{% endif -%}\n" + " {%- endfor -%}\n" + " ]\n" + " {%- elif item_key == 'type' -%}\n" + " {%- if item_value is string -%}\n" + " type:{{ format_argument(item_value | upper) }}\n" + " {%- else -%}\n" + " type:{{ format_argument(item_value | map('upper') | list) }}\n" + " {%- endif -%}\n" + " {%- else -%}\n" + " {{ item_key }}:{{ format_argument(item_value) }}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " }\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- if value['nullable'] %}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " nullable:true\n" + " {%- endif -%}\n" + " {%- if value['type'] | upper == 'OBJECT' -%}\n" + " {%- if value['properties'] is defined and value['properties'] is mapping -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " properties:{\n" + " {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n" + " }\n" + " {%- elif value is mapping -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " properties:{\n" + " {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}\n" + " }\n" + " {%- endif -%}\n" + " {%- if value['required'] -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " required:[\n" + " {%- for item in value['required'] | default([]) -%}\n" + " <|\"|>{{- item -}}<|\"|>\n" + " {%- if not loop.last %},{% endif -%}\n" + " {%- endfor -%}\n" + " ]\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n" + " type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + "{%- endmacro -%}\n" + "{%- macro format_function_declaration(tool_data) -%}\n" + " declaration:{{- tool_data['function']['name'] -}}{description:<|\"|>{{- tool_data['function']['description'] -}}<|\"|>\n" + " {%- set params = tool_data['function']['parameters'] -%}\n" + " {%- if params -%}\n" + " ,parameters:{\n" + " {%- if params.get('properties') -%}\n" + " properties:{ {{- format_parameters(params['properties'], params['required']) -}} },\n" + " {%- endif -%}\n" + " {%- if params.get('required') -%}\n" + " required:[\n" + " {%- for item in params['required'] -%}\n" + " <|\"|>{{- item -}}<|\"|>\n" + " {{- ',' if not loop.last -}}\n" + " {%- endfor -%}\n" + " ],\n" + " {%- endif -%}\n" + " {%- if params.get('type') -%}\n" + " type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- if 'response' in tool_data['function'] -%}\n" + " {%- set response_declaration = tool_data['function']['response'] -%}\n" + " ,response:{\n" + " {%- if response_declaration['description'] -%}\n" + " description:<|\"|>{{- response_declaration['description'] -}}<|\"|>,\n" + " {%- endif -%}\n" + " {%- if response_declaration['type'] | upper == 'OBJECT' -%}\n" + " type:<|\"|>{{- response_declaration['type'] | upper -}}<|\"|>}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " }\n" + "{%- endmacro -%}\n" + "{%- macro format_argument(argument, escape_keys=True) -%}\n" + " {%- if argument is string -%}\n" + " {{- '<|\"|>' + argument + '<|\"|>' -}}\n" + " {%- elif argument is boolean -%}\n" + " {{- 'true' if argument else 'false' -}}\n" + " {%- elif argument is mapping -%}\n" + " {{- '{' -}}\n" + " {%- set ns = namespace(found_first=false) -%}\n" + " {%- for key, value in argument | dictsort -%}\n" + " {%- if ns.found_first %},{% endif -%}\n" + " {%- set ns.found_first = true -%}\n" + " {%- if escape_keys -%}\n" + " {{- '<|\"|>' + key + '<|\"|>' -}}\n" + " {%- else -%}\n" + " {{- key -}}\n" + " {%- endif -%}\n" + " :{{- format_argument(value, escape_keys=escape_keys) -}}\n" + " {%- endfor -%}\n" + " {{- '}' -}}\n" + " {%- elif argument is sequence -%}\n" + " {{- '[' -}}\n" + " {%- for item in argument -%}\n" + " {{- format_argument(item, escape_keys=escape_keys) -}}\n" + " {%- if not loop.last %},{% endif -%}\n" + " {%- endfor -%}\n" + " {{- ']' -}}\n" + " {%- else -%}\n" + " {{- argument -}}\n" + " {%- endif -%}\n" + "{%- endmacro -%}\n" + "{%- macro strip_thinking(text) -%}\n" + " {%- set ns = namespace(result='') -%}\n" + " {%- for part in text.split('') -%}\n" + " {%- if '<|channel>' in part -%}\n" + " {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n" + " {%- else -%}\n" + " {%- set ns.result = ns.result + part -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {{- ns.result | trim -}}\n" + "{%- endmacro -%}\n" + "\n" + "{%- macro format_tool_response_block(tool_name, response) -%}\n" + " {{- '<|tool_response>' -}}\n" + " {%- if response is mapping -%}\n" + " {{- 'response:' + tool_name + '{' -}}\n" + " {%- for key, value in response | dictsort -%}\n" + " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" + " {%- if not loop.last %},{% endif -%}\n" + " {%- endfor -%}\n" + " {{- '}' -}}\n" + " {%- else -%}\n" + " {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}\n" + " {%- endif -%}\n" + " {{- '' -}}\n" + "{%- endmacro -%}\n" + "\n" + "{%- set ns = namespace(prev_message_type=None) -%}\n" + "{%- set loop_messages = messages -%}\n" + "{{- bos_token -}}\n" + "{#- Handle System/Tool Definitions Block -#}\n" + "{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n" + " {{- '<|turn>system\\n' -}}\n" + " {#- Inject Thinking token at the very top of the FIRST system turn -#}\n" + " {%- if enable_thinking is defined and enable_thinking -%}\n" + " {{- '<|think|>\\n' -}}\n" + " {%- set ns.prev_message_type = 'think' -%}\n" + " {%- endif -%}\n" + " {%- if messages[0]['role'] in ['system', 'developer'] -%}\n" + " {%- if messages[0]['content'] is string -%}\n" + " {{- messages[0]['content'] | trim -}}\n" + " {%- elif messages[0]['content'] is sequence -%}\n" + " {%- for item in messages[0]['content'] -%}\n" + " {{- item['text'] | trim + ' '-}}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + " {%- set loop_messages = messages[1:] -%}\n" + " {%- endif -%}\n" + " {%- if tools -%}\n" + " {%- for tool in tools %}\n" + " {{- '<|tool>' -}}\n" + " {{- format_function_declaration(tool) | trim -}}\n" + " {{- '' -}}\n" + " {%- endfor %}\n" + " {%- set ns.prev_message_type = 'tool' -%}\n" + " {%- endif -%}\n" + " {{- '\\n' -}}\n" + "{%- endif %}\n" + "\n" + "{#- Pre-scan: find last user message index for reasoning guard -#}\n" + "{%- set ns_turn = namespace(last_user_idx=-1) -%}\n" + "{%- for i in range(loop_messages | length) -%}\n" + " {%- if loop_messages[i]['role'] == 'user' -%}\n" + " {%- set ns_turn.last_user_idx = i -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "\n" + "{#- Loop through messages -#}\n" + "{%- for message in loop_messages -%}\n" + " {%- if message['role'] != 'tool' -%}\n" + " {%- set ns.prev_message_type = None -%}\n" + " {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n" + " {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n" + " {%- set prev_nt = namespace(role=None, found=false) -%}\n" + " {%- if loop.index0 > 0 -%}\n" + " {%- for j in range(loop.index0 - 1, -1, -1) -%}\n" + " {%- if not prev_nt.found -%}\n" + " {%- if loop_messages[j]['role'] != 'tool' -%}\n" + " {%- set prev_nt.role = loop_messages[j]['role'] -%}\n" + " {%- set prev_nt.found = true -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + " {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n" + " {%- if not continue_same_model_turn -%}\n" + " {{- '<|turn>' + role + '\\n' }}\n" + " {%- endif -%}\n" + "\n" + " {#- Render reasoning/reasoning_content as thinking channel -#}\n" + " {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n" + " {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n" + " {{- '<|channel>thought\\n' + thinking_text + '\\n' -}}\n" + " {%- endif -%}\n" + "\n" + " {%- if message.get('tool_calls') -%}\n" + " {%- for tool_call in message['tool_calls'] -%}\n" + " {%- set function = tool_call['function'] -%}\n" + " {{- '<|tool_call>call:' + function['name'] + '{' -}}\n" + " {%- if function['arguments'] is mapping -%}\n" + " {%- set ns_args = namespace(found_first=false) -%}\n" + " {%- for key, value in function['arguments'] | dictsort -%}\n" + " {%- if ns_args.found_first %},{% endif -%}\n" + " {%- set ns_args.found_first = true -%}\n" + " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" + " {%- endfor -%}\n" + " {%- elif function['arguments'] is string -%}\n" + " {{- function['arguments'] -}}\n" + " {%- endif -%}\n" + " {{- '}' -}}\n" + " {%- endfor -%}\n" + " {%- set ns.prev_message_type = 'tool_call' -%}\n" + " {%- endif -%}\n" + "\n" + " {%- set ns_tr_out = namespace(flag=false) -%}\n" + " {%- if message.get('tool_responses') -%}\n" + " {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n" + " {%- for tool_response in message['tool_responses'] -%}\n" + " {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n" + " {%- set ns_tr_out.flag = true -%}\n" + " {%- set ns.prev_message_type = 'tool_response' -%}\n" + " {%- endfor -%}\n" + " {%- elif message.get('tool_calls') -%}\n" + " {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}\n" + " {%- set ns_tool_scan = namespace(stopped=false) -%}\n" + " {%- for k in range(loop.index0 + 1, loop_messages | length) -%}\n" + " {%- if ns_tool_scan.stopped -%}\n" + " {%- elif loop_messages[k]['role'] != 'tool' -%}\n" + " {%- set ns_tool_scan.stopped = true -%}\n" + " {%- else -%}\n" + " {%- set follow = loop_messages[k] -%}\n" + " {#- Resolve tool_call_id to function name -#}\n" + " {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n" + " {%- for tc in message['tool_calls'] -%}\n" + " {%- if tc.get('id') == follow.get('tool_call_id') -%}\n" + " {%- set ns_tname.name = tc['function']['name'] -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {#- Handle content as string or content-parts array -#}\n" + " {%- set tool_body = follow.get('content') -%}\n" + " {%- if tool_body is string -%}\n" + " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n" + " {%- elif tool_body is sequence and tool_body is not string -%}\n" + " {%- set ns_txt = namespace(s='') -%}\n" + " {%- for part in tool_body -%}\n" + " {%- if part.get('type') == 'text' -%}\n" + " {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n" + " {%- for part in tool_body -%}\n" + " {%- if part.get('type') == 'image_url' -%}\n" + " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- elif part.get('type') in ['audio_url', 'input_audio'] -%}\n" + " {%- if part.get('type') == 'audio_url' -%}\n" + " {%- set audio_val = part['audio_url'] if part['audio_url'] is string else part['audio_url']['url'] -%}\n" + " {{- '<|audio|>' + audio_val -}}\n" + " {%- elif part.get('type') == 'input_audio' -%}\n" + " {%- set audio_val = part['input_audio'] if part['input_audio'] is string else ('data:audio/' + part['input_audio']['format'] + ';base64,' + part['input_audio']['data']) -%}\n" + " {{- '<|audio|>' + audio_val -}}\n" + " {%- endif -%}\n" + # " {%- elif part.get('type') == 'video_url' -%}\n" + # " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" + # " {{- '<|video|>' + video_val -}}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- else -%}\n" + " {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n" + " {%- endif -%}\n" + " {%- set ns_tr_out.flag = true -%}\n" + " {%- set ns.prev_message_type = 'tool_response' -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + "\n" + " {%- set captured_content -%}\n" + " {%- if message['content'] is string -%}\n" + " {%- if role == 'model' -%}\n" + " {{- strip_thinking(message['content']) -}}\n" + " {%- else -%}\n" + " {{- message['content'] | trim -}}\n" + " {%- endif -%}\n" + " {%- elif message['content'] is sequence -%}\n" + " {%- for item in message['content'] -%}\n" + " {%- if item['type'] == 'text' -%}\n" + " {%- if role == 'model' -%}\n" + " {{- strip_thinking(item['text']) -}}\n" + " {%- else -%}\n" + " {{- item['text'] | trim -}}\n" + " {%- endif -%}\n" + " {%- elif item['type'] == 'image_url' -%}\n" + " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- set ns.prev_message_type = 'image' -%}\n" + " {%- elif item['type'] in ['audio_url', 'input_audio'] -%}\n" + " {%- if item['type'] == 'audio_url' -%}\n" + " {%- set audio_val = item['audio_url'] if item['audio_url'] is string else item['audio_url']['url'] -%}\n" + " {{- '<|audio|>' + audio_val -}}\n" + " {%- elif item['type'] == 'input_audio' -%}\n" + " {%- set audio_val = item['input_audio'] if item['input_audio'] is string else ('data:audio/' + item['input_audio']['format'] + ';base64,' + item['input_audio']['data']) -%}\n" + " {{- '<|audio|>' + audio_val -}}\n" + " {%- endif -%}\n" + " {%- set ns.prev_message_type = 'audio' -%}\n" + " {%- endif -%}\n" + # " {%- elif item['type'] == 'video_url' -%}\n" + # " {%- set video_val = item['video_url'] if item['video_url'] is string else item['video_url']['url'] -%}\n" + # " {{- '<|video|>' + video_val -}}\n" + # " {%- set ns.prev_message_type = 'video' -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + " {%- endset -%}\n" + "\n" + " {{- captured_content -}}\n" + " {%- set has_content = captured_content | trim | length > 0 -%}\n" + "\n" + " {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n" + " {{- '<|tool_response>' -}}\n" + " {%- elif not (ns_tr_out.flag and not has_content) -%}\n" + " {{- '\\n' -}}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "\n" + "{%- if add_generation_prompt -%}\n" + " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n" + " {{- '<|turn>model\\n' -}}\n" + " {%- if not enable_thinking | default(false) -%}\n" + " {{- '<|channel>thought\\n' -}}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + "{%- endif -%}\n" + ) + + def __init__(self, enable_thinking: bool = True, **kwargs): + """ + Initializes the Gemma 4 Handler. + + Args: + enable_thinking (bool): Controls whether the <|think|> tag is injected and + manages <|channel>thought behavior. + Note: ONLY supported on Gemma4 31B and 26BA4B models. + NOT supported on Gemma4 E2B and E4B models. + """ + self.enable_thinking = enable_thinking + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Inject the thinking variable into the Jinja environment + self.extra_template_arguments["enable_thinking"] = self.enable_thinking + + # Set the stop token based on Gemma 4's format () + # generation_config.json: "eos_token_id": [1, 106, 50] + kwargs['stop'] = [self.GEMMA4_EOS_TOKEN, self.GEMMA4_EOT_TOKEN, self.GEMMA4_STR_TOKEN] + + if self.verbose: + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") + + return super().__call__(**kwargs) + + +class GLM41VChatHandler(MTMDChatHandler): + # Note: Make sure the GGUF files of your converted model and mmproj are F16 or F32. + + GLM41V_EOS_TOKEN = "<|endoftext|>" + GLM41V_PAD_TOKEN = "<|endoftext|>" + GLM41V_IMAGE_START_TOKEN = "<|begin_of_image|>" + GLM41V_IMAGE_END_TOKEN = "<|end_of_image|>" + + CHAT_FORMAT = ( + "[gMASK]\n" + "{%- for msg in messages -%}" + "{%- if msg.role == 'system' -%}" + "<|system|>\n{{ msg.content }}{{ GLM41V_EOS_TOKEN }}" + "{%- elif msg.role == 'user' -%}" + "<|user|>\n" + "{%- if msg.content is string -%}" + "{{ msg.content }}" + "{%- else -%}" + "{%- for item in msg.content -%}" + "{%- if item.type == 'image_url' or 'image_url' in item -%}" + "<|begin_of_image|>" + "{%- if item.image_url is string -%}" + "{{- item.image_url -}}" + "{%- else -%}" + "{{- item.image_url.url -}}" + "{%- endif -%}" + "<|end_of_image|>" + "{%- elif item.type == 'text' -%}" + "{{ item.text }}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}{{ GLM41V_EOS_TOKEN }}" + "{%- elif msg.role == 'assistant' -%}" + "{%- if msg.metadata -%}" + "<|assistant|>{{ msg.metadata }}\n{{ msg.content }}{{ GLM41V_EOS_TOKEN }}" + "{%- else -%}" + "<|assistant|>\n{{ msg.content }}{{ GLM41V_EOS_TOKEN }}" + "{%- endif -%}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + "<|assistant|>\n" + "{%- endif -%}" + ) + + def __call__(self, **kwargs): + self.extra_template_arguments["GLM41V_EOS_TOKEN"] = self.GLM41V_EOS_TOKEN + # https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking/blob/main/generation_config.json + stop_tokens = [self.GLM41V_EOS_TOKEN, "<|user|>", "<|observation|>", ""] # Stop token patch + kwargs['stop'] = stop_tokens + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix} - Start processing") + + # Use parent implementation + return super().__call__(**kwargs) + + +class GLM46VChatHandler(MTMDChatHandler): + GLM46V_EOS_TOKEN = "<|endoftext|>" + GLM46V_PAD_TOKEN = "<|endoftext|>" + GLM46V_IMAGE_START_TOKEN = "<|begin_of_image|>" + GLM46V_IMAGE_END_TOKEN = "<|end_of_image|>" + + CHAT_FORMAT = ( + "[gMASK]" + "{%- if tools -%}" + "<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n" + "You are provided with function signatures within XML tags:\n\n" + "{%- for tool in tools -%}" + "{{ tool | tojson(ensure_ascii=False) }}\n" + "{%- endfor -%}" + "\n\nFor each function call, output the function name and arguments within the following XML format:\n" + "{function-name}\n{arg-key-1}\n{arg-value-1}\n...\n" + "{%- endif -%}" + + "{%- for m in messages -%}" + "{%- if m.role == 'system' -%}" + "<|system|>\n{{ m.content }}" + "{%- elif m.role == 'user' -%}" + "<|user|>\n" + "{%- if m.content is string -%}" + "{{ m.content }}" + "{%- else -%}" + "{%- for item in m.content -%}" + "{%- if item.type == 'image_url' or 'image_url' in item -%}" + "<|begin_of_image|>" + "{%- if item.image_url is string -%}" + "{{- item.image_url -}}" + "{%- else -%}" + "{{- item.image_url.url -}}" + "{%- endif -%}" + "<|end_of_image|>" + "{%- elif item.type == 'text' -%}" + "{{ item.text }}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + # If enable_thinking is disabled, insert `/nothink` according to the source code logic. + "{{ '/nothink' if not enable_thinking else '' }}" + "{%- elif m.role == 'assistant' -%}" + "<|assistant|>" + "{%- if enable_thinking -%}" + "{%- set reasoning = m.reasoning_content if m.reasoning_content is string else '' -%}" + "\n{{ reasoning.strip() }}" + "{%- else -%}" + "\n" + "{%- endif -%}" + "{{ '\n' + m.content.strip() if m.content.strip() else '' }}" + "{%- endif -%}" + "{{ GLM46V_EOS_TOKEN }}" + "{%- endfor -%}" + + "{%- if add_generation_prompt -%}" + "<|assistant|>\n" + "{{ '' if enable_thinking else '\n' }}" + "{%- endif -%}" + ) + + def __init__(self, enable_thinking: bool = True, **kwargs): + """ + GLM-4.6V Handler + Parameters: + - enable_thinking (bool): Whether to enable the model's think process. The default is True. + """ + self.enable_thinking = enable_thinking + super().__init__(**kwargs) + + def __call__(self, **kwargs): + self.extra_template_arguments["enable_thinking"] = self.enable_thinking + self.extra_template_arguments["GLM46V_EOS_TOKEN"] = self.GLM46V_EOS_TOKEN + + # https://huggingface.co/zai-org/GLM-4.6V-Flash/blob/main/generation_config.json + kwargs['stop'] = [self.GLM46V_EOS_TOKEN, "<|user|>", "<|observation|>", "<|code_middle|>"] # Stop token patch + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") + + return super().__call__(**kwargs) + + +class GraniteDoclingChatHandler(MTMDChatHandler): + """ + Handler for Granite-Docling models. + + Format(512x512): Content + + Note(JamePeng): The GGUF files for Model and MMPROJ should be BF16 version !!! + Since the model does not have special tokens for the start and end of an image, + it is recommended to process only one image at a time. + You can iterate through the images individually for recognition. + + """ + GRANITE_BOS_TOKEN = "<|start_of_role|>" + GRANITE_EOS_TOKEN = "<|end_of_text|>" + GRANITE_PAD_TOKEN = "<|end_of_text|>" + GRANITE_IMAGE_TOKEN = "" + + CHAT_FORMAT = ( + "{%- for message in messages -%}" + "{{- '<|start_of_role|>' + message['role'] + '<|end_of_role|>' -}}" + "{%- if message['content'] is string -%}" + "{{- message['content'] -}}" + "{%- else -%}" + "{%- for part in message['content'] -%}" + "{%- if part['type'] == 'text' -%}" + "{{- part['text'] -}}" + "{%- elif part['type'] == 'image_url' -%}" + "{%- if part.image_url is string -%}" + "{{- part.image_url -}}" + "{%- else -%}" + "{{- part.image_url.url -}}" + "{%- endif -%}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + "{{- '<|end_of_text|>\n' -}}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + "{{- '<|start_of_role|>assistant' -}}" + # Support the 'controls' parameter if present in generation arguments + "{%- if controls -%}{{- ' ' + controls | tojson() -}}{%- endif -%}" + "{{- '<|end_of_role|>' -}}" + "{%- endif -%}" + ) + + def __init__(self, controls: dict = None, **kwargs): + """ + Granite-Docling Handler + Args: + controls (dict, optional): Operational parameters passed to the assistant role. + + The 'controls' parameter is used to guide the model's behavior or output format. + Common examples for 'controls' include: + - Document Parsing: {"mode": "document_parsing", "format": "json"} + """ + self.controls = controls + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Inject controls into the template environment + self.extra_template_arguments["controls"] = self.controls + self.DEFAULT_SYSTEM_MESSAGE = None + kwargs['stop'] = [self.GRANITE_EOS_TOKEN] + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix} - Start processing") + + + return super().__call__(**kwargs) + + +class LFM2VLChatHandler(MTMDChatHandler): + LFM2VL_BOS_TOKEN = "<|startoftext|>" + LFM2VL_EOS_TOKEN = "<|im_end|>" + LFM2VL_IMAGE_START_TOKEN = "<|image_start|>" + LFM2VL_IMAGE_END_TOKEN = "<|image_end|>" + + CHAT_FORMAT = ( + "{%- for message in messages -%}" + "{{ '<|im_start|>' + message['role'] + '\n' }}" + "{%- if message['content'] is string -%}" + "{{ message['content'] }}" + "{%- else -%}" + "{%- for content in message['content'] -%}" + "{%- if 'image_url' in content -%}" + "{%- if content.image_url is string -%}" + "<|image_start|>{{ content.image_url }}<|image_end|>" + "{%- else -%}" + "<|image_start|>{{ content.image_url.url }}<|image_end|>" + "{%- endif -%}" + "{%- elif content['type'] == 'text' -%}" + "{{ content['text'] }}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + "{{ '<|im_end|>\n' }}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + "{{ '<|im_start|>assistant\n' }}" + "{%- endif -%}" + ) + + def __init__(self, image_min_tokens: int = -1, image_max_tokens: int = -1, **kwargs): + """ + LFM2-VL Handler + LiquidAI officially recommends configuring LFM2-VL with the following Vision parameters: min_image_tokens=64, max_image_tokens=256 + """ + self.image_min_tokens = image_min_tokens + self.image_max_tokens = image_max_tokens + super().__init__(image_min_tokens=self.image_min_tokens, image_max_tokens=self.image_max_tokens, **kwargs) + + def __call__(self, **kwargs): + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix} - Start processing") + + return super().__call__(**kwargs) + + +class LFM25VLChatHandler(MTMDChatHandler): + """ + Handler for LFM2.5-VL multimodal models. + + Note(JamePeng): The suggestion is to compress the input image to 512x512 pixels to achieve native resolution processing. + """ + # Aligned with LFM2.5-VL tokenizer_config + LFM25VL_BOS_TOKEN = "<|startoftext|>" + LFM25VL_EOS_TOKEN = "<|im_end|>" + LFM25VL_PAD_TOKEN = "<|pad|>" + + # Image specific tokens + LFM25VL_IMAGE_TOKEN = "" + LFM25VL_IMAGE_START_TOKEN = "<|image_start|>" + LFM25VL_IMAGE_END_TOKEN = "<|image_end|>" + LFM25VL_IMAGE_THUMBNAIL = "<|img_thumbnail|>" + + CHAT_FORMAT = ( + "{{- bos_token -}}\n" + "{%- set keep_past_thinking = keep_past_thinking | default(false) -%}\n" + "{%- set ns = namespace(system_prompt='', content='') -%}\n" + "{%- if messages[0]['role'] == 'system' -%}\n" + " {%- set ns.system_prompt = messages[0]['content'] -%}\n" + " {%- set messages = messages[1:] -%}\n" + "{%- endif -%}\n" + "{%- if tools -%}\n" + " {%- set ns.system_prompt = ns.system_prompt + ('\\n' if ns.system_prompt else '') + 'List of tools: [' -%}\n" + " {%- for tool in tools -%}\n" + " {%- if tool is not string -%}\n" + " {%- set tool = tool | tojson -%}\n" + " {%- endif -%}\n" + " {%- set ns.system_prompt = ns.system_prompt + tool -%}\n" + " {%- if not loop.last -%}\n" + " {%- set ns.system_prompt = ns.system_prompt + ', ' -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- set ns.system_prompt = ns.system_prompt + ']' -%}\n" + "{%- endif -%}\n" + "{%- if ns.system_prompt -%}\n" + " {{- '<|im_start|>system\\n' + ns.system_prompt + '<|im_end|>\\n' -}}\n" + "{%- endif -%}\n" + "{%- set ns.last_assistant_index = -1 -%}\n" + "{%- for message in messages -%}\n" + " {%- if message['role'] == 'assistant' -%}\n" + " {%- set ns.last_assistant_index = loop.index0 -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "{%- for message in messages -%}\n" + " {{- '<|im_start|>' + message['role'] + '\\n' -}}\n" + " {%- set content = message['content'] -%}\n" + " {%- if content is not string -%}\n" + " {%- set ns.content = '' -%}\n" + " {#- MTMD-style Multimodal Injection (Audio stripped for VL model) -#}\n" + " {%- for item in content -%}\n" + " {%- if item['type'] == 'image_url' -%}\n" + " {%- set img_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" + " {%- set ns.content = ns.content + img_val -%}\n" + " {%- elif item['type'] == 'text' -%}\n" + " {%- set ns.content = ns.content + item['text'] -%}\n" + " {%- else -%}\n" + " {%- set ns.content = ns.content + (item | tojson) -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- set content = ns.content -%}\n" + " {%- endif -%}\n" + " {%- if message['role'] == 'assistant' and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}\n" + " {%- if '' in content -%}\n" + " {%- set content = content.split('')[-1] | trim -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {{- content + '<|im_end|>\\n' -}}\n" + "{%- endfor -%}\n" + "{%- if add_generation_prompt -%}\n" + " {{- '<|im_start|>assistant\\n' -}}\n" + "{%- endif -%}\n" + ) + + def __init__(self, keep_past_thinking: bool = False, **kwargs): + self.keep_past_thinking = keep_past_thinking + super().__init__(**kwargs) + + + def __call__(self, **kwargs): + if self.image_min_tokens > 256: + if self.verbose: + print(f"{self.log_prefix}: For LFM2.5-VL, using values higher than 256 for `image_min_tokens` could cause errors. Please reset it to between 64 and 256.") + self.image_min_tokens = -1 + + self.extra_template_arguments["keep_past_thinking"] = self.keep_past_thinking + + kwargs['stop'] = [self.LFM25VL_EOS_TOKEN] + + if self.verbose: + print(f"{self.log_prefix}(keep_past_thinking={self.keep_past_thinking}) - Start processing") + return super().__call__(**kwargs) + + +class PaddleOCRChatHandler(MTMDChatHandler): + """ + Handler for PaddleOCR 1.5/1.6 multimodal models. + """ + + PADDLEOCR_CLS_TOKEN = "<|begin_of_sentence|>" + PADDLEOCR_BOS_TOKEN = "" + PADDLEOCR_EOS_TOKEN = "" + PADDLEOCR_SEP_TOKEN = "<|end_of_sentence|>" + PADDLEOCR_IMAGE_BOS_TOKEN = "<|IMAGE_START|>" + PADDLEOCR_IMAGE_EOS_TOKEN = "<|IMAGE_END|>" + + CHAT_FORMAT = ( + "{%- if not add_generation_prompt is defined -%}{%- set add_generation_prompt = true -%}{%- endif -%}" + "{%- if not cls_token is defined -%}{%- set cls_token = '" + PADDLEOCR_CLS_TOKEN + "' -%}{%- endif -%}" + "{%- if not eos_token is defined -%}{%- set eos_token = '" + PADDLEOCR_EOS_TOKEN + "' -%}{%- endif -%}" + + "{{- cls_token -}}" + "{%- for message in messages -%}" + "{%- if message['role'] == 'user' -%}" + "{{- 'User: ' -}}" + + # Robust parsing: Check if content is string or list + "{%- if message['content'] is string -%}" + "{{- message['content'] -}}" + "{%- else -%}" + # Pass 1: Render all images first + "{%- for content in message['content'] -%}" + "{%- if content['type'] == 'image_url' and 'image_url' in content -%}" + "{{- '<|IMAGE_START|>' -}}" + "{%- if content.image_url is string -%}" + "{{- content.image_url -}}" + "{%- else -%}" + "{{- content.image_url.url -}}" + "{%- endif -%}" + "{{- '<|IMAGE_END|>' -}}" + "{%- endif -%}" + "{%- endfor -%}" + + # Pass 2: Render all text second + "{%- for content in message['content'] -%}" + "{%- if content['type'] == 'text' -%}" + "{{- content['text'] -}}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + "{{- '\\n' -}}" + + "{%- elif message['role'] == 'assistant' -%}" + "{{- 'Assistant:\\n' -}}" + "{%- if message['content'] is string -%}" + "{{- message['content'] -}}" + "{%- else -%}" + "{%- for content in message['content'] -%}" + "{%- if content['type'] == 'text' -%}" + "{{- content['text'] -}}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + "{{- eos_token -}}" + + "{%- elif message['role'] == 'system' -%}" + "{%- if message['content'] is string -%}" + "{{- message['content'] + '\\n' -}}" + "{%- else -%}" + "{%- for content in message['content'] -%}" + "{%- if content['type'] == 'text' -%}" + "{{- content['text'] + '\\n' -}}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + "{%- endif -%}" + "{%- endfor -%}" + + "{%- if add_generation_prompt -%}" + "{{- 'Assistant:\\n' -}}" + "{%- endif -%}" + ) + + def __init__( + self, + image_min_tokens: int = -1, + image_max_tokens: int = -1, + **kwargs + ): + self.image_min_tokens = image_min_tokens + self.image_max_tokens = image_max_tokens + super().__init__( + image_min_tokens=self.image_min_tokens, + image_max_tokens=self.image_max_tokens, + **kwargs + ) + + def __call__(self, **kwargs): + # Set the specific stop token defined in the PaddleOCR template + kwargs['stop'] = [self.PADDLEOCR_EOS_TOKEN] + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix} - Start processing") + + return super().__call__(**kwargs) + + +class Qwen25VLChatHandler(MTMDChatHandler): + + QWEN25_VL_BOS_TOKEN = "<|endoftext|>" + QWEN25_VL_PAD_TOKEN = "<|endoftext|>" + QWEN25_VL_EOS_TOKEN = "<|im_end|>" + + CHAT_FORMAT = ( + "{% set image_count = namespace(value=0) %}" + "{% for message in messages %}" + "{% if loop.first and message['role'] != 'system' %}" + "<|im_start|>system\n" + "{{ self.DEFAULT_SYSTEM_MESSAGE }}<|im_end|>\n" + "{% endif %}" + "<|im_start|>{{ message['role'] }}\n" + "{% if message['content'] is string %}" + "{{ message['content'] }}<|im_end|>\n" + "{% else %}" + "{% for content in message['content'] %}" + "{% if content['type'] == 'image_url' %}" + "{% if content.image_url is string %}" + "{% set image_count.value = image_count.value + 1 %}" + "Picture {{ image_count.value }}: <|vision_start|> {{ content.image_url }} <|vision_end|>" + "{% else %}" + "{% set image_count.value = image_count.value + 1 %}" + "Picture {{ image_count.value }}: <|vision_start|> {{ content.image_url.url }} <|vision_end|>" + "{% endif %}" + "{% elif content['type'] == 'text' %}" + "{{ content['text'] }}" + "{% endif %}" + "{% endfor %}" + "<|im_end|>\n" + "{% endif %}" + "{% endfor %}" + "<|im_start|>assistant\n" + ) + + def __call__(self, **kwargs): + kwargs['stop'] = [self.QWEN25_VL_EOS_TOKEN, self.QWEN25_VL_PAD_TOKEN] + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix} - Start processing") + + # Use parent implementation + return super().__call__(**kwargs) + +class Qwen3ASRChatHandler(MTMDChatHandler): + """ + Handler for Qwen 3 ASR (Automatic Speech Recognition) models. + + Features: + - Highly specialized for Speech-to-Text tasks. + - Aggregates all system text into a single cohesive system block. + - Drops user text entirely, extracting ONLY audio data into a unified user turn. + - Wraps audio with <|audio_start|><|audio_pad|>[DATA]<|audio_end|>. + - Integrated MTMD-style URL and Base64 injection for input_audio and audio_url. + """ + + DEFAULT_SYSTEM_MESSAGE = """ + You are an advanced multilingual Speech-to-Text model. Accurately transcribe the audio into text in its original spoken language. + You should ignore background noise, filler words, and stutters where possible, and format the final output with correct grammar and capitalization. + """ + + QWEN3_ASR_BOS_TOKEN = "<|im_start|>" + QWEN3_ASR_PAD_TOKEN = "<|endoftext|>" + QWEN3_ASR_EOS_TOKEN = "<|im_end|>" + + + QWEN3_ASR_AUDIO_BOS_TOKEN = "<|audio_start|>" + QWEN3_ASR_AUDIO_PAD_TOKEN = "<|audio_pad|>" + QWEN3_ASR_AUDIO_EOS_TOKEN = "<|audio_end|>" + + CHAT_FORMAT = ( + "{%- set ns = namespace(system_text='') -%}\n" + "{%- for m in messages -%}\n" + " {%- if m.role == 'system' -%}\n" + " {%- if m.content is string -%}\n" + " {%- set ns.system_text = ns.system_text + m.content -%}\n" + " {%- else -%}\n" + " {%- for c in m.content -%}\n" + " {%- if c.type == 'text' and (c.text is defined) -%}\n" + " {%- set ns.system_text = ns.system_text + c.text -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "\n" + "{%- set ns2 = namespace(audio_tokens='') -%}\n" + "{%- for m in messages -%}\n" + " {%- if m.content is not string -%}\n" + " {%- for c in m.content -%}\n" + " {%- if c.type == 'audio' or ('audio' in c) or ('audio_url' in c) or c.type == 'input_audio' -%}\n" + " {#- MTMD Audio Injection -#}\n" + " {%- set audio_val = '' -%}\n" + " {%- if c.type == 'audio_url' or 'audio_url' in c -%}\n" + " {%- set audio_val = c.audio_url if c.audio_url is string else c.audio_url.url -%}\n" + " {%- elif c.type == 'input_audio' or 'input_audio' in c -%}\n" + " {%- set audio_val = c.input_audio if c.input_audio is string else ('data:audio/' + c.input_audio.format + ';base64,' + c.input_audio.data) -%}\n" + " {%- endif -%}\n" + " {%- set ns2.audio_tokens = ns2.audio_tokens + '<|audio_start|><|audio_pad|>' + audio_val + '<|audio_end|>' -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "\n" + "{{- '<|im_start|>system\\n' + (ns.system_text if ns.system_text is string else '') + '<|im_end|>\\n' -}}\n" + "{{- '<|im_start|>user\\n' + ns2.audio_tokens + '<|im_end|>\\n' -}}\n" + "{%- if add_generation_prompt -%}\n" + " {{- '<|im_start|>assistant\\n' -}}\n" + "{%- endif -%}\n" + ) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Qwen3 models universally use `<|endoftext|>` and `<|im_end|>` as the stop token + kwargs['stop'] = [self.QWEN3_ASR_AUDIO_PAD_TOKEN, self.QWEN3_ASR_AUDIO_EOS_TOKEN] + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix} - Start processing Qwen3-ASR (Audio Only)") + + return super().__call__(**kwargs) + +class Qwen3VLChatHandler(MTMDChatHandler): + + QWEN3_VL_BOS_TOKEN = "<|endoftext|>" + QWEN3_VL_PAD_TOKEN = "<|endoftext|>" + QWEN3_VL_EOS_TOKEN = "<|im_end|>" + + CHAT_FORMAT = ( + "{{- '<|im_start|>system\n' -}}" + "{%- if messages[0].content is string and messages[0].role == 'system' -%}" + "{{- messages[0].content -}}" + "{%- elif messages[0].role == 'system' -%}" + "{%- if 'text' in messages[0].content -%}" + "{{- messages[0].content.text -}}" + "{%- else -%}" + "{{- 'You are a helpful assistant.' -}}" + "{%- endif -%}" + "{%- endif -%}" + "{%- if tools -%}" + "{{- '\n\n' -}}" + "{{- '# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n' -}}" + "{%- for tool in tools -%}" + "{{- '\n' -}}" + "{{- tool | tojson -}}" + "{%- endfor -%}" + "{{- '\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n\n\nYou can also return a response for the user alongside a function call:\nRESPONSE FOR THE USER HERE\n\n{\"name\": , \"arguments\": }\n' -}}" + "{%- endif -%}" + "{{- '<|im_end|>\n' -}}" + "{%- set image_count = namespace(value=0) -%}" + #"{%- set video_count = namespace(value=0) -%}" + "{%- for message in messages -%}" + "{%- if message.role == 'tool' -%}" + "{{- '<|im_start|>user\n\n' -}}" + "{%- elif message.role != 'system' -%}" + "{{- '<|im_start|>' + message.role + '\n' -}}" + "{%- endif -%}" + "{%- if message.content is string and message.role != 'system' -%}" + "{{- message.content -}}" + "{%- elif message.role != 'system' -%}" + "{%- for content in message.content -%}" + "{%- if 'image_url' in content -%}" + "{%- set image_count.value = image_count.value + 1 -%}" + "{%- if add_vision_id -%}" + "{{- 'Picture ' -}}" + "{{- image_count.value | string -}}" + "{{- ': ' -}}" + "{%- endif -%}" + "{{- '<|vision_start|>' -}}" + "{%- if content.image_url is string -%}" + "{{- content.image_url -}}" + "{%- else -%}" + "{{- content.image_url.url -}}" + "{%- endif -%}" + "{{- '<|vision_end|>' -}}" + "{%- endif -%}" + # Video not supported yet + "{%- if 'text' in content -%}" + "{{- content.text -}}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- endif -%}" + "{%- if message.role == 'assistant' -%}" + "{%- if message.tool_calls -%}" + "{%- for tool_call in message.tool_calls -%}" + "{%- if (loop.first and message.content) or (not loop.first) -%}" + "{{- '\n' -}}" + "{%- endif -%}" + "{%- if tool_call.function -%}" + "{%- set tool_call = tool_call.function -%}" + "{%- endif -%}" + "{{- '\n{\"name\": \"' + tool_call.name + '\", \"arguments\": ' -}}" + "{%- if tool_call.arguments is string -%}" + "{{- tool_call.arguments -}}" + "{%- else -%}" + "{{- tool_call.arguments | tojson -}}" + "{%- endif -%}" + "{{- '}\n' -}}" + "{%- endfor -%}" + "{%- endif -%}" + "{%- elif message.role == 'tool' -%}" + "{{- '' -}}" + "{%- endif -%}" + "{%- if message.role != 'system' -%}" + "{{- '<|im_end|>\n' -}}" + "{%- endif -%}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + "{{- '<|im_start|>assistant\n' -}}" + "{%- if force_reasoning -%}" + "{{- '\n' -}}" + "{%- endif -%}" + "{%- endif -%}" + ) + + def __init__( + self, + force_reasoning: bool = False, + add_vision_id: bool = True, + **kwargs, + ): + """ + Parameters: + - force_reasoning (bool): + - True: Force the reasoning in the model by adding to the chat template. + - False (default): Don't force the reasoning. + - add_vision_id (bool): + - True (default): Count all the images. Recommended for multi-image. + - False: Doesn't count the images. Can save tokens with single-image. + """ + super().__init__(**kwargs) + self.force_reasoning = force_reasoning + self.extra_template_arguments["force_reasoning"] = force_reasoning + self.extra_template_arguments["add_vision_id"] = add_vision_id + + def __call__(self, **kwargs): + kwargs['stop'] = [self.QWEN3_VL_EOS_TOKEN, self.QWEN3_VL_PAD_TOKEN] + + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix}(force_reasoning={self.force_reasoning}) - Start processing") + + # Use parent implementation + return super().__call__(**kwargs) + +class Qwen35ChatHandler(MTMDChatHandler): + """ + Handler for Qwen3.5/Qwen3.6 models. + """ + CHAT_FORMAT = ( + "{%- set image_count = namespace(value=0) -%}" + "{%- set video_count = namespace(value=0) -%}" + "{%- macro render_content(content, do_vision_count, is_system_content=false) -%}" + " {%- if content is string -%}" + " {{- content -}}" + " {%- elif content is iterable and content is not mapping -%}" + " {%- for item in content -%}" + " {%- if 'image_url' in item or item.type == 'image_url' -%}" + " {%- if is_system_content -%}" + " {{- raise_exception('System message cannot contain images.') -}}" + " {%- endif -%}" + " {%- if do_vision_count -%}" + " {%- set image_count.value = image_count.value + 1 -%}" + " {%- endif -%}" + " {%- if add_vision_id -%}" + " {{- 'Picture ' -}}" + " {{- image_count.value | string -}}" + " {{- ': ' -}}" + " {%- endif -%}" + " {{- '<|vision_start|>' -}}" + " {%- if item.image_url is string -%}" + " {{- item.image_url -}}" + " {%- else -%}" + " {{- item.image_url.url -}}" + " {%- endif -%}" + " {{- '<|vision_end|>' -}}" + " {%- elif 'video' in item -%}" + " {{- raise_exception('llama.cpp does not currently support video.') -}}" # Video not supported, raise exception + " {%- if is_system_content -%}" + " {{- raise_exception('System message cannot contain videos.') -}}" + " {%- endif -%}" + " {%- if do_vision_count -%}" + " {%- set video_count.value = video_count.value + 1 -%}" + " {%- endif -%}" + " {%- if add_vision_id -%}" + " {{- 'Video ' ~ video_count.value ~ ': ' -}}" + " {%- endif -%}" + " {{- '<|vision_start|>' -}}" + " {{- item.video -}}" + " {{- '<|vision_end|>' -}}" + " {%- elif 'text' in item -%}" + " {{- item.text -}}" + " {%- else -%}" + " {{- raise_exception('Unexpected item type in content.') -}}" + " {%- endif -%}" + " {%- endfor -%}" + " {%- elif content is none or content is undefined -%}" + " {{- '' -}}" + " {%- else -%}" + " {{- raise_exception('Unexpected content type.') -}}" + " {%- endif -%}" + "{%- endmacro -%}" + "{%- if not messages -%}" + " {{- raise_exception('No messages provided.') -}}" + "{%- endif -%}" + "{%- if tools and tools is iterable and tools is not mapping -%}" + " {{- '<|im_start|>system\n' -}}" + " {{- '# Tools\n\nYou have access to the following functions:\n\n' -}}" + " {%- for tool in tools -%}" + " {{- '\n' -}}" + " {{- tool | tojson -}}" + " {%- endfor -%}" + " {{- '\n' -}}" + " {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' -}}" + " {%- if messages[0].role == 'system' -%}" + " {%- set content = render_content(messages[0].content, false, true) | trim -%}" + " {%- if content -%}" + " {{- '\n\n' + content -}}" + " {%- endif -%}" + " {%- endif -%}" + " {{- '<|im_end|>\n' -}}" + "{%- elif messages[0].role == 'system' -%}" + " {%- set content = render_content(messages[0].content, false, true) -%}" + " {{- '<|im_start|>system\n' + content + '<|im_end|>\n' -}}" + "{%- endif -%}" + "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages | length - 1) -%}" + "{%- for message in messages[::-1] -%}" + " {%- set index = messages | length - 1 - loop.index0 -%}" + " {%- if ns.multi_step_tool and message.role == 'user' -%}" + " {%- set content = render_content(message.content, false) | trim -%}" + " {%- if not (content.startswith('') and content.endswith('')) -%}" + " {%- set ns.multi_step_tool = false -%}" + " {%- set ns.last_query_index = index -%}" + " {%- endif -%}" + " {%- endif -%}" + "{%- endfor -%}" + "{%- if ns.multi_step_tool -%}" + " {{- raise_exception('No user query found in messages.') -}}" + "{%- endif -%}" + "{%- for message in messages -%}" + " {%- set content = render_content(message.content, true) | trim -%}" + " {%- if message.role == 'system' -%}" + " {%- if not loop.first -%}" + " {{- raise_exception('System message must be at the beginning.') -}}" + " {%- endif -%}" + " {%- elif message.role == 'user' -%}" + " {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' -}}" + " {%- elif message.role == 'assistant' -%}" + " {%- set reasoning_content = '' -%}" + " {%- if message.reasoning_content is string -%}" + " {%- set reasoning_content = message.reasoning_content -%}" + " {%- elif '' in content -%}" + " {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') -%}" + " {%- set content = content.split('')[-1].lstrip('\n') -%}" + " {%- endif -%}" + " {%- set reasoning_content = reasoning_content | trim -%}" + " {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) -%}" + " {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content -}}" + " {%- else -%}" + " {{- '<|im_start|>' + message.role + '\n' + content -}}" + " {%- endif -%}" + " {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping -%}" + " {%- for tool_call in message.tool_calls -%}" + " {%- if tool_call.function is defined -%}" + " {%- set tool_call = tool_call.function -%}" + " {%- endif -%}" + " {%- if loop.first -%}" + " {%- if content | trim -%}" + " {{- '\n\n\n\n' -}}" + " {%- else -%}" + " {{- '\n\n' -}}" + " {%- endif -%}" + " {%- else -%}" + " {{- '\n\n\n' -}}" + " {%- endif -%}" + " {%- if tool_call.arguments is defined -%}" + " {%- for (args_name, args_value) in tool_call.arguments | items -%}" + " {{- '\n' -}}" + " {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}" + " {{- args_value -}}" + " {{- '\n' -}}" + " {%- endfor -%}" + " {%- endif -%}" + " {{- '\n' -}}" + " {%- endfor -%}" + " {%- endif -%}" + " {{- '<|im_end|>\n' -}}" + " {%- elif message.role == 'tool' -%}" + " {%- if loop.previtem and loop.previtem.role != 'tool' -%}" + " {{- '<|im_start|>user' -}}" + " {%- endif -%}" + " {{- '\n\n' -}}" + " {{- content -}}" + " {{- '\n' -}}" + " {%- if not loop.last and loop.nextitem.role != 'tool' -%}" + " {{- '<|im_end|>\n' -}}" + " {%- elif loop.last -%}" + " {{- '<|im_end|>\n' -}}" + " {%- endif -%}" + " {%- else -%}" + " {{- raise_exception('Unexpected message role.') -}}" + " {%- endif -%}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + " {{- '<|im_start|>assistant\n' -}}" + " {%- if enable_thinking is defined and enable_thinking is false -%}" + " {{- '\n\n\n\n' -}}" + " {%- else -%}" + " {{- '\n' -}}" + " {%- endif -%}" + "{%- endif -%}" + ) + + def __init__( + self, + add_vision_id: bool = True, + enable_thinking: bool = True, + preserve_thinking: bool = False, + **kwargs, + ): + """ + Parameters: + - add_vision_id (bool): + - True (default): Count all the images. Recommended for multi-image. + - False: Doesn't count the images. Can save tokens with single-image. + - enable_thinking (bool): + - True (default): Enables reasoning for better results. + - False: Disables reasoning for faster results. + - preserve_thinking (bool): + - True: Keeps reasoning process for ALL historical conversational turns. + - False (default): Only keeps for the latest assistant reply to save tokens. + """ + super().__init__(**kwargs) + self.enable_thinking = enable_thinking + self.preserve_thinking = preserve_thinking + self.extra_template_arguments["add_vision_id"] = add_vision_id + self.extra_template_arguments["enable_thinking"] = enable_thinking + self.extra_template_arguments["preserve_thinking"] = preserve_thinking + + def __call__(self, **kwargs): + llama = kwargs['llama'] + + if hasattr(llama, 'input_ids'): + llama.input_ids.fill(0) + + if self.verbose: + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}, preserve_thinking={self.preserve_thinking}) - Start processing") + + # Use parent implementation + return super().__call__(**kwargs) + + +class Step3VLChatHandler(MTMDChatHandler): + """ + Handler for Step3-VL models. + """ + + STEP3VL_BOS_TOKEN = "<|im_start|>" + STEP3VL_EOS_TOKEN = "<|im_end|>" + STEP3VL_PAD_TOKEN = "<|endoftext|>" + STEP3VL_IMAGE_TOKEN = "" + + CHAT_FORMAT = ( + "{%- macro render_content(content) -%}\n" + " {%- if content is none -%}{{- '' -}}\n" + " {%- elif content is string -%}{{- content -}}\n" + " {%- elif content is mapping -%}{{- content['value'] if 'value' in content else content['text'] -}}\n" + " {%- elif content is iterable -%}\n" + " {%- for item in content -%}\n" + " {%- if item.type == 'text' -%}\n" + " {{- item['value'] if 'value' in item else item['text'] -}}\n" + " {%- elif item.type in ['image', 'image_url'] -%}\n" + " {%- set url_val = '' -%}\n" + " {%- if item.image_url -%}\n" + " {%- set url_val = item.image_url if item.image_url is string else item.image_url.url -%}\n" + " {%- endif -%}\n" + " {{- '' + url_val -}}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + " {%- endif -%}\n" + "{%- endmacro -%}\n" + "\n" + "{%- if tools -%}\n" + " {{- '<|im_start|>system\\n' -}}\n" + " {%- if messages[0].role == 'system' -%}\n" + " {{- render_content(messages[0].content) + '\\n\\n' -}}\n" + " {%- endif -%}\n" + " {{- '# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n' -}}\n" + " {%- for tool in tools -%}\n" + " {{- '\\n' -}}\n" + " {{- tool | tojson -}}\n" + " {%- endfor -%}\n" + " {{- '\\n\\n\\nAlways adhere to this exact format for tool use:\\n\\n\\n{\"name\": , \"arguments\": }\\n\\n{additional_tool_calls}\\n\\nNote:\\n- For each function call, return a json object with function name and arguments within XML tags.\\n- `` must be an exact match to one of the available tools.\\n- `` must be valid JSON that strictly follows the tool\\'s parameters schema.<|im_end|>\\n' -}}\n" + "{%- else -%}\n" + " {%- if messages[0].role == 'system' -%}\n" + " {{- '<|im_start|>system\\n' + render_content(messages[0].content) + '<|im_end|>\\n' -}}\n" + " {%- endif -%}\n" + "{%- endif -%}\n" + "\n" + "{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) -%}\n" + "{%- for message in messages[::-1] -%}\n" + " {%- set index = (messages|length - 1) - loop.index0 -%}\n" + " {%- if ns.multi_step_tool and message.role == 'user' and render_content(message.content) is string and not(render_content(message.content).startswith('') and render_content(message.content).endswith('')) -%}\n" + " {%- set ns.multi_step_tool = false -%}\n" + " {%- set ns.last_query_index = index -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "\n" + "{%- for message in messages -%}\n" + " {%- set content = render_content(message.content) -%}\n" + " {%- if (message.role == 'user') or (message.role == 'system' and not loop.first) -%}\n" + " {%- set role_name = 'observation' if (message.role == 'system' and not loop.first and message.name == 'observation') else message.role -%}\n" + " {{- '<|im_start|>' + role_name + '\\n' + content + '<|im_end|>' + '\\n' -}}\n" + " {%- elif message.role == 'assistant' -%}\n" + " {%- if message.reasoning_content is string -%}\n" + " {%- set reasoning_content = render_content(message.reasoning_content) -%}\n" + " {%- else -%}\n" + " {%- if '' in content -%}\n" + " {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') -%}\n" + " {%- set content = content.split('')[-1].lstrip('\\n') -%}\n" + " {%- else -%}\n" + " {%- set reasoning_content = '' -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- if loop.index0 > ns.last_query_index -%}\n" + " {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n' + content -}}\n" + " {%- else -%}\n" + " {{- '<|im_start|>' + message.role + '\\n' + content -}}\n" + " {%- endif -%}\n" + " {%- if message.tool_calls -%}\n" + " {{- '\\n' -}}\n" + " {%- for tool_call in message.tool_calls -%}\n" + " {{- '\\n' -}}\n" + " {%- if tool_call.function -%}\n" + " {%- set tool_call = tool_call.function -%}\n" + " {%- endif -%}\n" + " {{- '\\n{\"name\": \"' -}}\n" + " {{- tool_call.name -}}\n" + " {{- '\", \"arguments\": ' -}}\n" + " {%- if tool_call.arguments is string -%}\n" + " {{- tool_call.arguments -}}\n" + " {%- else -%}\n" + " {{- tool_call.arguments | tojson -}}\n" + " {%- endif -%}\n" + " {{- '}\\n' -}}\n" + " {%- endfor -%}\n" + " {{- '\\n' -}}\n" + " {%- endif -%}\n" + " {{- '<|im_end|>\\n' -}}\n" + " {%- elif message.role == 'tool' -%}\n" + " {%- if loop.first or (messages[loop.index0 - 1].role != 'tool') -%}\n" + " {{- '<|im_start|>tool_response' -}}\n" + " {%- endif -%}\n" + " {{- '\\n\\n' -}}\n" + " {{- content -}}\n" + " {{- '\\n' -}}\n" + " {%- if loop.last or (messages[loop.index0 + 1].role != 'tool') -%}\n" + " {{- '<|im_end|>\\n' -}}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + "{%- endfor -%}\n" + "{%- if add_generation_prompt -%}\n" + " {{- '<|im_start|>assistant\\n\\n\\n\\n' if (enable_thinking is defined and not enable_thinking) else '<|im_start|>assistant\\n' -}}\n" + "{%- endif -%}\n" + ) + + def __init__(self, enable_thinking: bool = True, **kwargs): + """ + Initializes the Step3-VL Handler. + + Args: + enable_thinking (bool): If False, injects an empty block to bypass reasoning. + """ + self.enable_thinking = enable_thinking + super().__init__(**kwargs) + + def __call__(self, **kwargs): + # Pass thinking toggle into Jinja + self.extra_template_arguments["enable_thinking"] = self.enable_thinking + + # Step3 uses standard <|im_end|> ChatML stop formatting + kwargs['stop'] = [self.STEP3VL_PAD_TOKEN, self.STEP3VL_EOS_TOKEN] + + if self.verbose: + print(f"{self.log_prefix}(enable_thinking={self.enable_thinking}) - Start processing") + + return super().__call__(**kwargs) From d84b0c21fa4a131df5c17ddd1b2447929dc1973f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 14 Jun 2026 22:08:32 +0800 Subject: [PATCH 192/304] fix(model): handle missing chat templates - Update LlamaModel.model_chat_template() to return Optional[str] and accept name=None for the default model chat template. - llama_model_chat_template() may return nullptr when no chat template is available. Handle that case explicitly instead of decoding a null pointer, and return None so callers can apply their own fallback logic. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 434921e6bd..91befb2247 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -152,12 +152,17 @@ def model_size(self) -> int: """ return llama_cpp.llama_model_size(self.model) - def model_chat_template(self, name: bytes) -> str: + def model_chat_template(self, name: Optional[bytes] = None) -> Optional[str]: """ - Get the default chat template. Returns nullptr if not available - If name is NULL, returns the default chat template + Get a chat template from the model. + + If name is None, returns the default chat template. + Returns None if no chat template is available. """ - return llama_cpp.llama_model_chat_template(self.model, name).decode("utf-8") + template = llama_cpp.llama_model_chat_template(self.model, name) + if template is None: + return None + return template.decode("utf-8") def n_params(self) -> int: """ From c9745316d748cec408b07c2f3a43fd97fa921e73 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 14 Jun 2026 23:43:33 +0800 Subject: [PATCH 193/304] feat(mtmd): enhance generic chat template support - Enhance GenericMTMDChatHandler to better support model-provided chat templates. - Allow the generic handler to accept an optional named chat template, load it from the model at call time via llama_model_chat_template(), fall back to the model's default chat template, and finally use the built-in MTMD CHAT_FORMAT when no model template is available. - Also expand the generic media placeholder list for common multimodal templates and document the handler as a template-driven MTMD implementation. This prepares the generic path for a later render-driven placeholder replacement pass. Signed-off-by: JamePeng --- llama_cpp/llama.py | 2 + llama_cpp/llama_multimodal.py | 118 +++++++++++++++++++++++++++++++--- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index dbc60eaf76..b6a2c8d5a7 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -174,6 +174,7 @@ def __init__( log_filters: Optional[Sequence[str]] = None, log_filters_case_sensitive: bool = True, # Extra Params + chat_template_name: Optional[str] = None, chat_handler_kwargs: Dict[str, Any] = {}, **kwargs, # type: ignore ): @@ -721,6 +722,7 @@ def __init__( chat_format = self.metadata.get("tokenizer.chat_template", None), mmproj_path = mmproj_path, verbose = self.verbose, + chat_template_name=chat_template_name, **chat_handler_kwargs ) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index a055869543..a0f7e594e4 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -91,6 +91,8 @@ class MTMDChatHandler: "{% endif %}" ) + KNOWN_MEDIA_TAGS: List[str] = [] + def __init__( self, mmproj_path: Optional[str] = None, @@ -1189,41 +1191,137 @@ def from_pretrained( **kwargs, ) -# Experiments are not recommended for this purpose at this time. +# Generic template-driven MTMD handler. class GenericMTMDChatHandler(MTMDChatHandler): + """ + Generic MTMD chat handler backed by the model-provided chat template. + + This handler is intentionally template-driven. It renders the model's + tokenizer.chat_template first, then normalizes rendered media URLs or + placeholder tokens into MTMD media markers before tokenization. + + It is designed for model templates that emit media placeholders such as + <|image_pad|>, <|image|>, , [IMG], or Kimi-style <|media_pad|>. + Model-specific handlers may still be preferable when a model requires + special stop tokens, generation flags, or non-standard template arguments. + """ + KNOWN_MEDIA_TAGS = [ + # Pad placeholders inside model-specific wrappers. "<|image_pad|>", "<|audio_pad|>", "<|video_pad|>", + + # Direct placeholders inside Gemma/Llama/GLM-style wrappers. "<|image|>", "<|audio|>", "<|video|>", - "[IMG]" + + # LLaVA / LFM / Mistral-style placeholders. + "", + "
+### Sanity Checking +`python.exe -c "from llama_cpp import Llama; print('llama-cpp import OK')"` +
CLI / requirements.txt From 0116e76da361026898d743b33a4bf18e71f9ce07 Mon Sep 17 00:00:00 2001 From: patrikpatrik Date: Wed, 24 Jun 2026 10:35:34 -0700 Subject: [PATCH 208/304] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4361c857a8..8b3daa9d81 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python ```
-### Sanity Checking +**Sanity Checking** `python.exe -c "from llama_cpp import Llama; print('llama-cpp import OK')"`
From cf44079c997f6f262d150982528ecd81ab0f19ac Mon Sep 17 00:00:00 2001 From: patrikpatrik Date: Wed, 24 Jun 2026 10:36:55 -0700 Subject: [PATCH 209/304] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8b3daa9d81..58f13d4fcf 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python ```
-**Sanity Checking** +**Sanity Checking** `python.exe -c "from llama_cpp import Llama; print('llama-cpp import OK')"`
From c71baefff09a511d9df54b544555ca83f54148bc Mon Sep 17 00:00:00 2001 From: patrikpatrik Date: Wed, 24 Jun 2026 10:37:59 -0700 Subject: [PATCH 210/304] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 58f13d4fcf..acf57289cb 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python ```
-**Sanity Checking** +**Sanity Checking** `python.exe -c "from llama_cpp import Llama; print('llama-cpp import OK')"`
From e5782b1c423b3aa80362165dc67abc2b9f9bf912 Mon Sep 17 00:00:00 2001 From: patrikpatrik Date: Wed, 24 Jun 2026 10:40:44 -0700 Subject: [PATCH 211/304] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index acf57289cb..8968d4ecb5 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,8 @@ pip install "llama-cpp-python @ git+https://github.com/JamePeng/llama-cpp-python
**Sanity Checking** -`python.exe -c "from llama_cpp import Llama; print('llama-cpp import OK')"` +Use this line to check if installation was successful before moving further. +```python.exe -c "from llama_cpp import Llama; print('llama-cpp import OK')"```
CLI / requirements.txt From b74cabb133cce7a996129eb019c490d8f5a07922 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 27 Jun 2026 01:53:19 +0800 Subject: [PATCH 212/304] Update Submodule vendor/llama.cpp 1191758..3fc4e10 - Append `n_layer_nextn` call method Signed-off-by: JamePeng --- llama_cpp/_internals.py | 3 +++ llama_cpp/llama.py | 4 ++++ llama_cpp/llama_cpp.py | 5 +++++ vendor/llama.cpp | 2 +- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 91befb2247..ef9238ad30 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -123,6 +123,9 @@ def n_embd_out(self) -> int: def n_layer(self) -> int: return llama_cpp.llama_model_n_layer(self.model) + def n_layer_nextn(self) -> int: + return llama_cpp.llama_model_n_layer_nextn(self.model) + def n_head(self) -> int: return llama_cpp.llama_model_n_head(self.model) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b6a2c8d5a7..1d8bb16887 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -3470,6 +3470,10 @@ def n_layer(self) -> int: """Return the n_layer value.""" return self._model.n_layer() + def n_layer_nextn(self) -> int: + """Return the n_layer_nextn value.""" + return self._model.n_layer_nextn() + def n_head(self) -> int: """Return the head size.""" return self._model.n_head() diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 1e81d80f65..ee1b5244f0 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1746,6 +1746,11 @@ def llama_model_n_layer(model: llama_model_p, /) -> int: ... +# LLAMA_API int32_t llama_model_n_layer_nextn(const struct llama_model * model); +@ctypes_function("llama_model_n_layer_nextn", [llama_model_p_ctypes], ctypes.c_int32) +def llama_model_n_layer_nextn(model: llama_model_p, /) -> int: + ... + # LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); @ctypes_function("llama_model_n_head", [llama_model_p_ctypes], ctypes.c_int32) def llama_model_n_head(model: llama_model_p, /) -> int: diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1191758c5d..3fc4e10527 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1191758c5d77e575b4531e057708a562f6f6f0b2 +Subproject commit 3fc4e105279105106b08a133a4e3e483116e621f From e6c16afd6baeb9ad67624ba13d224a26e1d441f9 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 29 Jun 2026 00:07:31 +0800 Subject: [PATCH 213/304] Update Submodule vendor/llama.cpp 3fc4e10..c818263 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 3fc4e10527..c818263f2a 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 3fc4e105279105106b08a133a4e3e483116e621f +Subproject commit c818263f2a5ddab028dea5f169ea2b2266421125 From bb03d2ada63a9b69f2844f5459ada29af17d189b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 1 Jul 2026 01:55:51 +0800 Subject: [PATCH 214/304] Update Submodule vendor/llama.cpp c818263..4f31eed Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c818263f2a..4f31eedb0c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c818263f2a5ddab028dea5f169ea2b2266421125 +Subproject commit 4f31eedb0ccf546b7e8d6bb243b170f12522f54d From 46708d11a26528dcd64a96df1c59fce7ec41e080 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 1 Jul 2026 01:57:59 +0800 Subject: [PATCH 215/304] refactor(mtmd): extract `mtmd_tokenize` into `_mtmd_tokenize` standalone helper - Introduce `_mtmd_tokenize()` to encapsulate llama.cpp mtmd_tokenize binding - Decouple hybrid tokenization logic from `_process_mtmd_prompt` - Improve separation of concerns between prompt construction and C++ binding - Preserve strict media marker validation to ensure token/bitmap alignment Signed-off-by: JamePeng --- llama_cpp/llama_multimodal.py | 109 ++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 26 deletions(-) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index f1b320b772..60e29d7dd5 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -515,6 +515,75 @@ def _is_audio_chunk(self, chunk_type: int) -> bool: == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO ) + def _mtmd_tokenize( + self, + llama: "llama_core.Llama", + text: str, + bitmaps: Optional[list] = None, + chunks: Optional[Any] = None, + ) -> Any: + """ + Perform MTMD hybrid tokenization. + + This function isolates the llama.cpp mtmd_tokenize call + so that prompt construction logic is decoupled from runtime execution. + + It guarantees: + - stable interface for future async/batch decoding + - isolated error handling for tokenizer failures + - clean separation between prompt building and C++ binding + """ + + if chunks is None: + chunks = self._mtmd_cpp.mtmd_input_chunks_init() + if chunks is None: + raise ValueError( + f"{self.log_prefix}(_mtmd_tokenize): failed to init mtmd_input_chunks" + ) + + # Validate strict alignment between rendered media markers and provided bitmaps + # to ensure MTMD tokenization consistency and prevent decoding mismatch errors. + if bitmaps is not None: + marker_count = text.count(self.media_marker) + if marker_count != len(bitmaps): + raise ValueError( + f"{self.log_prefix}(_mtmd_tokenize): marker mismatch " + f"(marker_count={marker_count}, bitmap_count={len(bitmaps)})" + ) + + input_text = self._mtmd_cpp.mtmd_input_text() + input_text.text = ctypes.c_char_p(text.encode("utf-8")) + input_text.add_special = (llama.n_tokens == 0) + input_text.parse_special = True + + bitmap_array = None + n_bitmaps = 0 + + if bitmaps: + n_bitmaps = len(bitmaps) + bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * n_bitmaps)(*bitmaps) + else: + bitmap_array = None + n_bitmaps = 0 + + result = self._mtmd_cpp.mtmd_tokenize( + self.mtmd_ctx, + chunks, + ctypes.byref(input_text), + bitmap_array, + n_bitmaps, + ) + + if result != 0: + raise ValueError( + f"{self.log_prefix}(_mtmd_tokenize): tokenize failed\n" + f"- result={result}\n" + f"- text_len={len(text)}\n" + f"- n_bitmaps={n_bitmaps}\n" + ) + + return chunks + def _process_mtmd_prompt( self, llama: llama_core.Llama, @@ -572,8 +641,12 @@ def _process_mtmd_prompt( text = text.replace(item["url"], media_marker) if self.verbose: - print(f"{self.log_prefix}(_process_mtmd_prompt): Rendered prompt length: {len(text)} chars, Media count: {len(media_items)}.", file=sys.stderr) - print(f"{self.log_prefix}(_process_mtmd_prompt): Rendered prompt: {text}", file=sys.stderr) + print( + f"{self.log_prefix}(_process_mtmd_prompt): " + f"Rendered prompt length: {len(text)} chars, Media count: {len(media_items)}.\n" + f"Rendered prompt: {text}", + file=sys.stderr, + ) # 3. Pre-allocate bitmap array to guarantee chronological order during concurrent decoding bitmaps = [None] * len(media_items) @@ -614,29 +687,13 @@ def _create_bitmap_func(idx: int, item: dict): # If there are no images, set the bitmaps to empty. bitmaps = [] - # 4. Initialize mtmd_input_chunks - input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode('utf-8') - input_text.add_special = (llama.n_tokens == 0) - input_text.parse_special = True - - chunks = self._mtmd_cpp.mtmd_input_chunks_init() - if chunks is None: - raise ValueError(f"{self.log_prefix}(mtmd_input_chunks_init): Failed to initialize mtmd_input_chunks.") - - # 5. Hybrid Tokenization (Text + Media binding) - if len(bitmaps) > 0: - bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) - result = self._mtmd_cpp.mtmd_tokenize( - self.mtmd_ctx, chunks, ctypes.byref(input_text), bitmap_array, len(bitmaps) - ) - else: - result = self._mtmd_cpp.mtmd_tokenize( - self.mtmd_ctx, chunks, ctypes.byref(input_text), None, 0 - ) - - if result != 0: - raise ValueError(f"{self.log_prefix}(mtmd_tokenize): Unable to tokenize prompt, res = {result}.") + # 4. Hybrid Tokenization (Text + Media) + chunks = self._mtmd_tokenize( + llama=llama, + text=text, + bitmaps=bitmaps, + chunks=None, + ) # Video helper contexts only need to stay alive until mtmd_tokenize() completes. if video_cleanup: @@ -644,7 +701,7 @@ def _create_bitmap_func(idx: int, item: dict): self._mtmd_cpp.mtmd_helper_video_free(video_ctx) video_cleanup.clear() - # 6. Virtual Token Ledger Construction + # 5. Virtual Token Ledger Construction full_prompt_ids = [] chunk_token_spans = [] current_idx = 0 From 6f21d4df8b711f8db2c87e10c5387c21fc9b29ed Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 1 Jul 2026 02:08:25 +0800 Subject: [PATCH 216/304] docs: update mtmd chat handler import paths in README - Update import statements for multi-modal chat handlers from llama_cpp.llama_chat_format to llama_cpp.llama_multimodal in the documentation examples. Signed-off-by: JamePeng --- README.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8968d4ecb5..20d14e42e2 100644 --- a/README.md +++ b/README.md @@ -1053,11 +1053,18 @@ Below are the supported multi-modal models and their respective chat handlers (P | [qwen3.6](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF) | `Qwen35ChatHandler` | `qwen3.6` | | [step3-vl](https://huggingface.co/JamePeng2023/Step3-VL-10B-GGUF) | `Step3VLChatHandler` | `step3-vl` | -Then you'll need to use a custom chat handler to load the clip model and process the chat messages and images. +Then you'll need to use a custom chat handler to load the mmproj model and process the chat messages and images. + +Note: Starting from 0.3.41-preview, in order to extend MTMD capabilities, multimodal-related logic has been separated from `llama_chat_format` into `llama_multimodal`. + New implementations are recommended to use the updated multimodal interfaces in `llama_multimodal` + For backward compatibility, the legacy `llama_chat_format` path is still retained and continues to support existing integrations, but may be deprecated in future versions. + Additionally, the parameter `clip_model_path` has been renamed to `mmproj_path` to better reflect its purpose and align with the underlying multimodal projection model naming convention. + The old parameter name `clip_model_path` is kept as a compatibility alias in some interfaces, but new code should use `mmproj_path` exclusively. ```python from llama_cpp import Llama -from llama_cpp.llama_chat_format import Llava15ChatHandler +# from llama_cpp.llama_chat_format import Llava15ChatHandler +from llama_cpp.llama_multimodal import Llava15ChatHandler model_path="path/to/llava/ggml-model-f16.gguf" mmproj_path="path/to/llava/mmproj-model-f16.gguf" @@ -1086,7 +1093,8 @@ You can also pull the model from the Hugging Face Hub using the `from_pretrained ```python from llama_cpp import Llama -from llama_cpp.llama_chat_format import MoondreamChatHandler +# from llama_cpp.llama_chat_format import MoondreamChatHandler +from llama_cpp.llama_multimodal import MoondreamChatHandler chat_handler = MoondreamChatHandler.from_pretrained( repo_id="vikhyatk/moondream2", @@ -1128,7 +1136,8 @@ print(response["choices"][0]["text"]) ```python # Import necessary libraries from llama_cpp import Llama -from llama_cpp.llama_chat_format import Qwen3VLChatHandler +# from llama_cpp.llama_chat_format import Qwen3VLChatHandler +from llama_cpp.llama_multimodal import Qwen3VLChatHandler import base64 import os @@ -1285,7 +1294,8 @@ The `Qwen3ASRChatHandler` is specifically designed for the Qwen3 Automatic Speec ```python from llama_cpp import Llama -from llama_cpp.llama_chat_format import Qwen3ASRChatHandler +# from llama_cpp.llama_chat_format import Qwen3ASRChatHandler +from llama_cpp.llama_multimodal import Qwen3ASRChatHandler import base64 import os @@ -1390,7 +1400,8 @@ Below is a complete, production-ready example demonstrating how to dynamically r ```python from llama_cpp import Llama -from llama_cpp.llama_chat_format import Gemma4ChatHandler +# from llama_cpp.llama_chat_format import Gemma4ChatHandler +from llama_cpp.llama_multimodal import Gemma4ChatHandler import base64 import os From 8b38e72c655c15e064862a48e91c3fe0e32b28f1 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 3 Jul 2026 01:05:56 +0800 Subject: [PATCH 217/304] Update Submodule vendor/llama.cpp 4f31eed..fdb1db8 Signed-off-by: JamePeng --- llama_cpp/_internals.py | 6 ++++++ llama_cpp/llama_cpp.py | 29 +++++++++++++++++++++++++++++ vendor/llama.cpp | 2 +- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index ef9238ad30..b05a4207da 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -149,6 +149,12 @@ def model_desc(self) -> str: llama_cpp.llama_model_desc(self.model, buf, 256) return buf.value.decode("utf-8") + def model_ftype(self) -> int: + """ + Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 + """ + return llama_cpp.llama_model_ftype(self.model) + def model_size(self) -> int: """ Returns the total size of all the tensors in the model in bytes diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index ee1b5244f0..a4086fd704 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -408,6 +408,20 @@ class llama_ftype(enum.IntEnum): LLAMA_FTYPE_MOSTLY_Q1_0 = 40 LLAMA_FTYPE_GUESSED = 1024 +# // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" +# LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); +@ctypes_function( + "llama_ftype_name", + [ctypes.c_int], + ctypes.c_char_p, +) +def llama_ftype_name( + ftype: llama_ftype, / +) -> bytes: + """ + Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" + """ + # enum llama_rope_scaling_type { # LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, # LLAMA_ROPE_SCALING_TYPE_NONE = 0, @@ -1922,6 +1936,21 @@ def llama_model_desc( ... +# // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 +# LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); +@ctypes_function( + "llama_model_ftype", + [llama_model_p_ctypes], + ctypes.c_int, +) +def llama_model_ftype( + model: llama_model_p, + /, +) -> int: + """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0""" + ... + + # // Returns the total size of all the tensors in the model in bytes # LLAMA_API uint64_t llama_model_size(const struct llama_model * model); @ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4f31eedb0c..fdb1db877c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4f31eedb0ccf546b7e8d6bb243b170f12522f54d +Subproject commit fdb1db877c526ec90f668eca1b858da5dba85560 From 79a86357e9893399a2c5622e725f0579cb010fd3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 5 Jul 2026 00:39:23 +0800 Subject: [PATCH 218/304] fix(vocab): update vocab_type to use self.vocab and add None checks Signed-off-by: JamePeng --- llama_cpp/_internals.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b05a4207da..0ae347c004 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -100,9 +100,13 @@ def __del__(self): self.close() def vocab_type(self) -> int: - return llama_cpp.llama_vocab_type(self.model) + if self.vocab is None: + raise RuntimeError("LlamaModel.vocab_type: vocab is None") + return llama_cpp.llama_vocab_type(self.vocab) def n_vocab(self) -> int: + if self.vocab is None: + raise RuntimeError("LlamaModel.n_vocab: vocab is None") return llama_cpp.llama_vocab_n_tokens(self.vocab) def n_ctx_train(self) -> int: From 9f87f6186a69583f2ad9a901db9fc2abfe3b183b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 5 Jul 2026 00:45:14 +0800 Subject: [PATCH 219/304] Update Submodule vendor/llama.cpp fdb1db8..6658925 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index fdb1db877c..665892536d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit fdb1db877c526ec90f668eca1b858da5dba85560 +Subproject commit 665892536dfb1b7532161e3182304bd35c33e768 From 8a0d085407f1eeddd9740e0d907c3e1191bcd79c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 7 Jul 2026 02:56:29 +0800 Subject: [PATCH 220/304] Update Submodule vendor/llama.cpp 6658925..3899b39 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 665892536d..3899b39ce2 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 665892536dfb1b7532161e3182304bd35c33e768 +Subproject commit 3899b39ce2acc2e019f149b7107f24b6ca297390 From d3f0ac931ced646aade26198a6fd3a8be2dd5368 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Jul 2026 02:01:57 +0800 Subject: [PATCH 221/304] refactor(mtmd): extract prompt rendering and media marker normalization Add extra_template_arguments to MTMD chat handlers and pass them through to the Jinja chat template render call. This allows generic model templates to receive render-time options such as enable_thinking, add_vision_id, or model-specific template jinja variables. Extract MTMD prompt rendering into dedicated helpers: * _render_mtmd_prompt() for pure chat template rendering * _replace_media_placeholders() for normalizing rendered media tags and URLs into the MTMD runtime marker * _render_and_replace_media() for the combined render-and-normalize stage This removes inline render/replace logic from _process_mtmd_prompt(), keeps media marker validation after normalization, and improves separation between prompt construction and MTMD tokenization. Signed-off-by: JamePeng --- llama_cpp/llama.py | 1 - llama_cpp/llama_multimodal.py | 128 +++++++++++++++++++++++++++++----- 2 files changed, 110 insertions(+), 19 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 1d8bb16887..75af691ebf 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -721,7 +721,6 @@ def __init__( self.chat_handler = llama_multimodal.GenericMTMDChatHandler( chat_format = self.metadata.get("tokenizer.chat_template", None), mmproj_path = mmproj_path, - verbose = self.verbose, chat_template_name=chat_template_name, **chat_handler_kwargs ) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index 60e29d7dd5..5ef1a2feb9 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -102,6 +102,7 @@ def __init__( image_max_tokens: int = -1, chat_template_override: Optional[str] = None, batch_max_tokens: int = 1024, + extra_template_arguments: Optional[Dict[str, Any]] = None, **kwargs ): @@ -148,7 +149,13 @@ def __init__( import llama_cpp.mtmd_cpp as mtmd_cpp self._mtmd_cpp = mtmd_cpp self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None - self.extra_template_arguments: dict[str, Any] = {} + + if extra_template_arguments is not None and not isinstance(extra_template_arguments, dict): + raise TypeError( + f"{self.log_prefix}(__init__): `extra_template_arguments` must be a dict." + ) + + self.extra_template_arguments: dict[str, Any] = dict(extra_template_arguments or {}) self.is_support_vision = False self.is_support_audio = False @@ -515,6 +522,104 @@ def _is_audio_chunk(self, chunk_type: int) -> bool: == self._mtmd_cpp.mtmd_input_chunk_type.MTMD_INPUT_CHUNK_TYPE_AUDIO ) + def _render_mtmd_prompt( + self, + messages: List[llama_types.ChatCompletionRequestMessage], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + add_generation_prompt: bool = True, + ) -> str: + """ + Render the chat template into plain prompt text. + + This stage only renders the Jinja template. It does not normalize media + placeholders or replace media URLs with the MTMD runtime marker. + """ + return self.chat_template.render( + messages=messages, + add_generation_prompt=add_generation_prompt, + eos_token=self.mtmd_eos_token, + bos_token=self.mtmd_bos_token, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + **getattr(self, "extra_template_arguments", {}), + ) + + def _replace_media_placeholders( + self, + text: str, + media_items: List[Dict[str, str]], + ) -> str: + """ + Normalize rendered media placeholders and media URLs into the MTMD runtime marker. + + llama.cpp MTMD tokenization recognizes the canonical media marker, usually + `<__media__>`. Model chat templates may render media as model-specific tags + such as ``, `<|image|>`, `[IMG]`, `<|image_pad|>`, or as the original + URL/data URI. This stage converts those rendered forms into the canonical + MTMD marker and validates that the final marker count matches the number of + media payloads. + """ + media_marker = self.media_marker + + # 1. Replace known template-specific media tags first. + # + # This handles templates that render placeholders such as: + # , <|image|>, [IMG], <|image_pad|>, <|media_pad|>, etc. + for tag in self._chat_format_parser_tags: + if tag in text: + text = text.replace(tag, media_marker) + + # 2. Replace rendered media URLs/data URIs. + # + # This handles templates that directly render the original image/audio/video + # URL or data URI instead of a symbolic placeholder. + for item in media_items: + url = item.get("url", "") + if url and url in text: + text = text.replace(url, media_marker, 1) + + # 3. Validate only after all normalization is complete. + marker_count = text.count(media_marker) + if marker_count != len(media_items): + raise ValueError( + f"{self.log_prefix}(_replace_media_placeholders): media marker mismatch " + f"(marker_count={marker_count}, media_count={len(media_items)})" + ) + + return text + + def _render_and_replace_media( + self, + messages: List[llama_types.ChatCompletionRequestMessage], + media_items: List[Dict[str, str]], + functions: Optional[List[llama_types.ChatCompletionFunction]] = None, + function_call: Optional[llama_types.ChatCompletionRequestFunctionCall] = None, + tools: Optional[List[llama_types.ChatCompletionTool]] = None, + tool_choice: Optional[llama_types.ChatCompletionToolChoiceOption] = None, + add_generation_prompt: bool = True, + ) -> str: + """ + Render chat messages and normalize rendered media placeholders into MTMD markers. + """ + text = self._render_mtmd_prompt( + messages=messages, + functions=functions, + function_call=function_call, + tools=tools, + tool_choice=tool_choice, + add_generation_prompt=add_generation_prompt, + ) + + return self._replace_media_placeholders( + text=text, + media_items=media_items, + ) + def _mtmd_tokenize( self, llama: "llama_core.Llama", @@ -615,31 +720,18 @@ def _process_mtmd_prompt( messages = [{"role": "system", "content": self.DEFAULT_SYSTEM_MESSAGE}] + messages media_items = self._get_media_items(messages) - media_marker = self.media_marker - # 2. Render the chat template and replace actual URLs with C++ media markers - text = self.chat_template.render( + # 2. Render chat template and normalize media placeholders to MTMD markers. + text = self._render_and_replace_media( messages=messages, - add_generation_prompt=add_generation_prompt, - eos_token=self.mtmd_eos_token, - bos_token=self.mtmd_bos_token, + media_items=media_items, functions=functions, function_call=function_call, tools=tools, tool_choice=tool_choice, - **getattr(self, 'extra_template_arguments', {}) + add_generation_prompt=add_generation_prompt, ) - for tag in self._chat_format_parser_tags: - if tag not in text: - continue - - text = text.replace(tag, media_marker) - - # Replace image_url by media_marker in text - for item in media_items: - text = text.replace(item["url"], media_marker) - if self.verbose: print( f"{self.log_prefix}(_process_mtmd_prompt): " From 8cf0be28c1b8f82d57542287570cedae4ffb77cd Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Jul 2026 02:41:49 +0800 Subject: [PATCH 222/304] docs(README): add GenericMTMDChatHandler usage guide Replace the legacy Llava multimodal loading example with a GenericMTMDChatHandler usage guide for template-driven multimodal GGUF models. Document loading mmproj through Llama, chat template resolution order, extra_template_arguments for model-specific Jinja variables, and when to prefer a dedicated multimodal chat handler. Also clarify the mmproj_path naming, llama_multimodal migration, and note that the generic handler is intended as a flexible fallback for models without dedicated handlers and may require additional testing for model-specific prompting behavior. Signed-off-by: JamePeng --- README.md | 150 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 108 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 20d14e42e2..f11f26a582 100644 --- a/README.md +++ b/README.md @@ -1053,76 +1053,142 @@ Below are the supported multi-modal models and their respective chat handlers (P | [qwen3.6](https://huggingface.co/unsloth/Qwen3.6-35B-A3B-GGUF) | `Qwen35ChatHandler` | `qwen3.6` | | [step3-vl](https://huggingface.co/JamePeng2023/Step3-VL-10B-GGUF) | `Step3VLChatHandler` | `step3-vl` | -Then you'll need to use a custom chat handler to load the mmproj model and process the chat messages and images. +Then you'll need to load the multimodal projection model (`mmproj`) together with the main language model. -Note: Starting from 0.3.41-preview, in order to extend MTMD capabilities, multimodal-related logic has been separated from `llama_chat_format` into `llama_multimodal`. - New implementations are recommended to use the updated multimodal interfaces in `llama_multimodal` - For backward compatibility, the legacy `llama_chat_format` path is still retained and continues to support existing integrations, but may be deprecated in future versions. - Additionally, the parameter `clip_model_path` has been renamed to `mmproj_path` to better reflect its purpose and align with the underlying multimodal projection model naming convention. - The old parameter name `clip_model_path` is kept as a compatibility alias in some interfaces, but new code should use `mmproj_path` exclusively. +Starting from `0.3.41-preview`, new multimodal implementations are recommended to use the updated interfaces in `llama_multimodal`. For backward compatibility, the legacy `llama_chat_format` path is still retained, but may be deprecated in future versions. + +The parameter `clip_model_path` has been renamed to `mmproj_path` to better reflect its purpose and align with llama.cpp's multimodal projection model naming convention. New code should use `mmproj_path` exclusively. + +### Generic MTMD Chat Handler + +For multimodal GGUF models that already include a valid `tokenizer.chat_template`, you can use the generic MTMD handler through `mmproj_path`. + +This is especially useful for newer multimodal models that have not yet received a dedicated Python chat handler. The generic handler renders the model-provided Jinja chat template, then normalizes rendered media placeholders or media URLs into the canonical llama.cpp MTMD media marker, usually `<__media__>`, before calling `mtmd_tokenize`. + +> **Note:** `GenericMTMDChatHandler` is intended as a flexible fallback for template-driven multimodal models. Because different model families may use different media ordering rules, reasoning switches, stop tokens, or special template variables, some models may still require a dedicated chat handler. Please test carefully and report issues if you encounter incorrect prompts, missing media markers, or mismatched media counts. ```python from llama_cpp import Llama -# from llama_cpp.llama_chat_format import Llava15ChatHandler -from llama_cpp.llama_multimodal import Llava15ChatHandler -model_path="path/to/llava/ggml-model-f16.gguf" -mmproj_path="path/to/llava/mmproj-model-f16.gguf" +# Model and multimodal projection paths +MODEL_PATH = r"path/to/model.gguf" +MMPROJ_PATH = r"path/to/mmproj.gguf" llm = Llama( - model_path=model_path, - chat_handler=Llava15ChatHandler(clip_model_path=mmproj_path), - n_ctx=2048, + model_path=MODEL_PATH, + mmproj_path=MMPROJ_PATH, + n_gpu_layers=-1, + n_ctx=10240, + verbose=True, + verbosity=2, + chat_handler_kwargs={ + "verbose": True, + }, ) -llm.create_chat_completion( - messages = [ - {"role": "system", "content": "You are an assistant who perfectly describes images."}, +response = llm.create_chat_completion( + messages=[ { "role": "user", "content": [ - {"type" : "text", "text": "What's in this image?"}, - {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" } } - ] + { + "type": "image_url", + "image_url": { + "url": "path/to/image.jpg", + }, + }, + { + "type": "text", + "text": "Describe this image in detail.", + }, + ], } ] ) + +print(response["choices"][0]["message"]["content"]) +```` + +#### Chat Template Resolution Order + +`GenericMTMDChatHandler` resolves the chat template in the following order: + +1. Use the explicit `chat_format` passed through `chat_handler_kwargs`, if provided. +2. Use the named model chat template if `chat_template_name` is provided. +3. Fall back to the default `tokenizer.chat_template` stored in the GGUF model metadata. +4. Fall back to the built-in MTMD chat template if no model template is available. + +Example using a named chat template: + +```python +llm = Llama( + model_path=r"path/to/model.gguf", + mmproj_path=r"path/to/mmproj.gguf", + # chat_template_name="default", + n_gpu_layers=-1, + n_ctx=4096, + chat_handler_kwargs={ + "verbose": False, + }, +) ``` -You can also pull the model from the Hugging Face Hub using the `from_pretrained` method. +#### Passing Extra Template Arguments + +Some model chat templates expose optional Jinja variables such as `enable_thinking`, `add_vision_id`, or model-specific media token switches. Further details can be obtained by analyzing the chat templates provided in `chat_template.jinja` or `tokenizer_config.json` for each model. + +You can pass those values through `chat_handler_kwargs["extra_template_arguments"]`: ```python from llama_cpp import Llama -# from llama_cpp.llama_chat_format import MoondreamChatHandler -from llama_cpp.llama_multimodal import MoondreamChatHandler -chat_handler = MoondreamChatHandler.from_pretrained( - repo_id="vikhyatk/moondream2", - filename="*mmproj*", -) +# Model and multimodal projection paths +MODEL_PATH = r"path/to/model.gguf" +MMPROJ_PATH = r"path/to/mmproj.gguf" -llm = Llama.from_pretrained( - repo_id="vikhyatk/moondream2", - filename="*text-model*", - chat_handler=chat_handler, - n_ctx=2048, # n_ctx should be increased to accommodate the image embedding +llm = Llama( + model_path=MODEL_PATH, + mmproj_path=MMPROJ_PATH, + n_gpu_layers=-1, + n_ctx=10240, + verbose=False, + verbosity=1, + chat_handler_kwargs={ + "extra_template_arguments": { + "enable_thinking": True, + }, + "verbose": False, + }, ) +... +``` -response = llm.create_chat_completion( - messages = [ - { - "role": "user", - "content": [ - {"type" : "text", "text": "What's in this image?"}, - {"type": "image_url", "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" } } +The values inside `extra_template_arguments` are passed directly into the Jinja template render call. - ] - } - ] +For models that already have a dedicated handler, you can still instantiate that handler directly: + +```python +from llama_cpp import Llama +from llama_cpp.llama_multimodal import PaddleOCRChatHandler + +MODEL_PATH = r"path/to/model.gguf" +MMPROJ_PATH = r"path/to/mmproj.gguf" + +llm = Llama( + model_path=MODEL_PATH, + chat_handler=PaddleOCRChatHandler( + mmproj_path=MMPROJ_PATH, + ), + n_gpu_layers=-1, # Use all available GPU layers + n_ctx = 0, # Context window size + n_batch=2048, ) -print(response["choices"][0]["text"]) +... ``` +Use `GenericMTMDChatHandler` when the model-provided `tokenizer.chat_template` already works correctly. Prefer a dedicated handler when the model requires custom prompt construction, special reasoning behavior, custom stop tokens, OCR/ASR-specific handling, or non-standard media ordering. + + **Note**: Multi-modal models also support tool calling and JSON mode. From b9b58594023ab673c2dda6723f8909d85d65a2e5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Jul 2026 02:49:18 +0800 Subject: [PATCH 223/304] docs(README): Update Generic MTMD Chat Handler directory index. Signed-off-by: JamePeng --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f11f26a582..ab8c39d779 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ This package provides: - [How to use the ReasoningBudgetSampler](https://github.com/JamePeng/llama-cpp-python#reasoning-budget-first-reasoning-block) - [Multi-modal Models Support](https://github.com/JamePeng/llama-cpp-python#multi-modal-models) - Support Models Lists + - [Introducing Generic MTMD Chat Handler](https://github.com/JamePeng/llama-cpp-python#generic-mtmd-chat-handler) - [Loading a Local Image With Qwen3VL(Thinking/Instruct)](https://github.com/JamePeng/llama-cpp-python#loading-a-local-image-with-qwen3vlthinkinginstruct) - [Speech Recognition With Qwen3-ASR (Speech-to-Text)](https://github.com/JamePeng/llama-cpp-python#speech-recognition-with-qwen3-asr-speech-to-text) - [Comprehensive Omni MultiModal Example: Gemma-4 (Vision + Audio + Text)](https://github.com/JamePeng/llama-cpp-python#comprehensive-omni-multimodal-example-gemma-4-vision--audio--text) From 169d5e1a43fb6ff4e5b6f5d0f26f1ec8acbd97b8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Jul 2026 03:27:38 +0800 Subject: [PATCH 224/304] Bump version to 0.3.41 - This release is mainly focused on improving the multimodal architecture, making MTMD chat handling cleaner, more flexible, and easier to extend for future image, audio, and video workflows. I also continued syncing with the latest llama.cpp APIs and improved several developer-facing diagnostics around model templates, shared library loading, and Windows OpenMP runtime discovery. Signed-off-by: JamePeng --- CHANGELOG.md | 133 ++++++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1865195db3..83c3beb128 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,139 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.41] Template-Driven MTMD, Broader Multimodal Inputs, and Smarter N-Gram Drafting + +- refactor(mtmd): extract prompt rendering and media marker normalization + - Add extra_template_arguments to MTMD chat handlers and pass them through to the Jinja chat template render call. This allows generic model templates to receive render-time options such as enable_thinking, add_vision_id, or model-specific template jinja variables. + - Extract MTMD prompt rendering into dedicated helpers: + * _render_mtmd_prompt() for pure chat template rendering + * _replace_media_placeholders() for normalizing rendered media tags and URLs into the MTMD runtime marker + * _render_and_replace_media() for the combined render-and-normalize stage + - This removes inline render/replace logic from _process_mtmd_prompt(), keeps media marker validation after normalization, and improves separation between prompt construction and MTMD tokenization. + +- docs(README): add GenericMTMDChatHandler usage guide + - Replace the legacy Llava multimodal loading example with a GenericMTMDChatHandler + usage guide for template-driven multimodal GGUF models. + - Document loading mmproj through Llama, chat template resolution order, + extra_template_arguments for model-specific Jinja variables, and when to prefer + a dedicated multimodal chat handler. + - Also clarify the mmproj_path naming, llama_multimodal migration, and note that + the generic handler is intended as a flexible fallback for models without + dedicated handlers and may require additional testing for model-specific + prompting behavior. + - Update Generic MTMD Chat Handler directory index. + +- refactor(mtmd): extract mtmd_tokenize into _mtmd_tokenize standalone helper + - Introduce `_mtmd_tokenize()` to encapsulate llama.cpp mtmd_tokenize binding + - Decouple hybrid tokenization logic from `_process_mtmd_prompt` + - Improve separation of concerns between prompt construction and C++ binding + - Preserve strict media marker validation to ensure token/bitmap alignment + +- feat(speculative): Improve ngram-map draft selection and accept feedback + - Store accepted draft lengths per key/value and truncate future drafts accordingly + - Make key-only mode draft on any key match without applying min_hits + - Select k4v continuations by frequency instead of latest occurrence + - Skip ambiguous k4v drafts when the top continuation is not dominant + - Track fixed-size k4v continuations to keep frequency statistics comparable + +- feat(mtmd): broaden multimodal media extraction + - Broaden MTMD media extraction to support common multimodal content shapes used + by model chat templates. + - In addition to OpenAI-style image_url/audio_url/video_url chunks, accept + image/audio/video typed chunks and direct media keys such as {"image": "..."}, + {"audio": "..."}, or {"video": "..."}. This keeps the extracted media list + aligned with templates that emit placeholders for image, audio, or video content + without requiring URL-specific chunk names. + - Add a shared helper for extracting URLs, local paths, existing data URIs, or + inline base64 payloads from media content items. Preserve capability checks, + strict input_audio format validation, and explicit errors for missing or + ambiguous media payloads. + +- feat(mtmd): enhance generic chat template support + - Enhance GenericMTMDChatHandler to better support model-provided chat templates. + - Allow the generic handler to accept an optional named chat template, load it + from the model at call time via llama_model_chat_template(), fall back to the + model's default chat template, and finally use the built-in MTMD CHAT_FORMAT + when no model template is available. + - Also expand the generic media placeholder list for common multimodal templates + and document the handler as a template-driven MTMD implementation. This prepares + the generic path for a later render-driven placeholder replacement pass. + +- fix(model): handle missing chat templates + - Update `LlamaModel.model_chat_template()` to return Optional[str] and accept + name=None for the default model chat template. + - `llama_model_chat_template()` may return nullptr when no chat template is + available. Handle that case explicitly instead of decoding a null pointer, and + return None so callers can apply their own fallback logic. + +- fix(vocab): update `LlamaModel.vocab_type` to use self.vocab and add None checks + +- refactor(mtmd): move multimodal handlers to separate module `llama_multimodal` + - Move `MTMDChatHandler`, `GenericMTMDChatHandler``, and model-specific multimodal + chat handlers out of `llama_chat_format.py` into `llama_multimodal.py`. + - `llama_chat_format.py` has grown too large and difficult to maintain, especially + as MTMD support expands beyond image-only use cases. Splitting multimodal + handling into its own module makes the chat formatting layer smaller and keeps + media loading, MTMD tokenization, multimodal KV-cache bookkeeping, and handler + implementations in a dedicated place. + - This also prepares the codebase for broader multimodal support and future video + frame / image batch evaluation, where the media-processing path will need to + evolve independently from text-only chat formatting. + - Keep backward-compatible re-exports from `llama_chat_format.py` so existing + imports continue to work. + - Also keep `clip_model_path` as a deprecated initialization alias for + `mmproj_path` in the base MTMD handler. + - docs: update mtmd chat handler import paths in README + - Update import statements for multi-modal chat handlers from llama_cpp.llama_chat_format to llama_cpp.llama_multimodal in the documentation examples. + +- feat: Implemented generic multimodal chat handler prototype (by **@alcoftTAO**) + +- docs(README): Added command prompt scenario for README.md (by **@patrikpatrik**) + - Updated command prompt scenario under Configuration -> Environment Variables + - Sanity checking after successful installation of wheel + +- feat(MTMDChatHandler): add chunk type helpers + - Add small helper methods `_is_text_chunk`/`_is_image_chunk`/`_is_audio_chunk` for checking + MTMD text, image, and audio chunk type enum values. + - This keeps MTMD prompt processing easier to read and avoids repeating direct + enum comparisons when building token spans for text and media chunks. + +- feat(mtmd): add video input support to `MTMDChatHandler` + - Add video_url handling to the MTMD chat template and media extraction + pipeline. Detect whether the loaded libmtmd build supports video helpers + and reject video inputs early when MTMD_VIDEO is unavailable. + - Update media loading and bitmap creation for the new helper wrapper API. + mtmd_helper_bitmap_init_from_buf now returns a bitmap wrapper containing + both the decoded bitmap and an optional video helper context, so keep the + video context alive until mtmd_tokenize completes and release it afterward. + - Also consolidate duplicated audio/video byte loading into a shared + _load_bytes helper, reuse it for image loading, and add richer default HTTP + headers for remote media requests. + +- build(CMakelists): Improve Windows LLVM OpenMP runtime `libomp140.x86_64.dll` discovery + - Also improve diagnostics by reporting the selected runtime source and path, + warning when an explicit override points to a missing file, and keeping a clear + runtime warning when no OpenMP DLL can be found. + - prefer VS 2022 VC143 OpenMP redist and keep System32 as final fallback怂 + +- feat(_ctypes_extensions): improve error diagnostics for shared library loading + When `load_shared_library` fails, the resulting `RuntimeError` now + includes a listing of the contents of the searched directories. This + provides immediate context to help developers diagnose missing, misplaced, + or incorrectly named library files. + + - Added `_format_library_dir_contents` to safely format directory listings. + - Appended the directory listing to the failure message. + - Confined this diagnostic work strictly to the failure path to avoid any + performance overhead during successful imports. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/3899b39ce2acc2e019f149b7107f24b6ca297390](https://github.com/ggml-org/llama.cpp/commit/3899b39ce2acc2e019f149b7107f24b6ca297390) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260707 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/12861b918f67b62f78f28c5cabb7223f766e1097...b9b58594023ab673c2dda6723f8909d85d65a2e5 + + ## [0.3.40-Milestone] Reasoning Budget Control, Gemma 4 12B Support, Enhanced Jinja2ChatFormatter, NGram k/k4v Speculative Decoding, Faster Native Sampling and Multimodal Improvements - feat(internals): Add `ReasoningBudgetSampler` support diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 1650e6af69..3c3aa6690c 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.40" +__version__ = "0.3.41" From b4c74bf9b3c95a2d63dc676e78f83183f4c6f0b2 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 8 Jul 2026 03:39:31 +0800 Subject: [PATCH 225/304] fix(types): make assistant message name optional - Mark the assistant message `name` field as `NotRequired[Optional[str]]` to match the optional nature of assistant message metadata and avoid requiring callers to provide `name` in typed chat completion requests. Signed-off-by: JamePeng --- llama_cpp/llama_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_types.py b/llama_cpp/llama_types.py index 37b041ee87..56451ea251 100644 --- a/llama_cpp/llama_types.py +++ b/llama_cpp/llama_types.py @@ -427,7 +427,7 @@ class ChatCompletionRequestAssistantMessageFunctionCall(TypedDict): class ChatCompletionRequestAssistantMessage(TypedDict): """Messages sent by the model in response to user messages.""" role: Literal["assistant"] - name: Optional[str] + name: NotRequired[Optional[str]] content: NotRequired[Optional[str]] refusal: NotRequired[Optional[str]] tool_calls: NotRequired[ChatCompletionMessageToolCalls] From dbb4e21efcaadb412264a314606bb9dbed60bafe Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 9 Jul 2026 01:48:30 +0800 Subject: [PATCH 226/304] Update Submodule vendor/llama.cpp 3899b39..a646006 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 3899b39ce2..a646006f09 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 3899b39ce2acc2e019f149b7107f24b6ca297390 +Subproject commit a646006f09d2f76f2d62d6c0d5e8e8490d570720 From 6c7b16fcccd6884d9163188bba25ecb40ab2d617 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 9 Jul 2026 03:45:34 +0800 Subject: [PATCH 227/304] fix: validate eval tokens before native decode Add token-id validation at the Llama.eval() boundary before context shifting, batch construction, or llama_decode execution. This prevents invalid token types, negative token ids, and out-of-vocabulary ids from reaching the native decode path, where they may otherwise cause hard crashes instead of Python exceptions. Wrap llama_decode with defensive exception handling in LlamaContext.decode() so native exceptions are surfaced with clearer diagnostic context. Also include a small token preview in Llama.eval() fatal decode errors to make backend failures easier to debug without changing the existing recoverable KV slot handling behavior. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 9 +++++++- llama_cpp/llama.py | 40 ++++++++++++++++++++++++++++++++++- llama_cpp/llama_multimodal.py | 7 +++++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 0ae347c004..431077822d 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -746,7 +746,14 @@ def decode(self, batch: 'LlamaBatch') -> int: (e.g., negative error codes or invalid batch structures). """ self._assert_ctx() - return_code = llama_cpp.llama_decode(self.ctx, batch.batch) + try: + return_code = llama_cpp.llama_decode(self.ctx, batch.batch) + except Exception as e: + raise RuntimeError( + "llama_decode raised a native exception before returning a status code. " + "This may indicate an invalid batch, invalid token id, corrupted context, " + "backend memory issue, or native access violation." + ) from e if return_code == 0: return 0 diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 75af691ebf..2ee8abaf5b 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1082,6 +1082,37 @@ def abort(self) -> None: print(f"Llama.abort: Abort signal received. Terminating generation...", file=sys.stderr) self._abort_event.set() + def _validate_eval_tokens( + self, + tokens: Sequence[int], + ) -> None: + """Validate token ids before passing them to llama_decode. + + This mirrors llama.cpp server-side token validation and prevents invalid + token ids from reaching the native decode path, where they may cause hard + crashes instead of Python exceptions. + """ + if not tokens: + return + + for i, tok in enumerate(tokens): + if not isinstance(tok, int): + raise ValueError( + f"Llama.eval: invalid token type at index {i}: " + f"{type(tok).__name__}" + ) + + if tok < 0: + raise ValueError( + f"Llama.eval: invalid negative token id at index {i}: {tok}" + ) + + if tok >= self._n_vocab: + raise ValueError( + f"Llama.eval: token out of vocab at index {i}: " + f"{tok} >= n_vocab({self._n_vocab})" + ) + def eval( self, tokens: Sequence[int], @@ -1106,6 +1137,11 @@ def eval( if n_eval == 0: return + # Validate token ids before any context shifting, batch construction, or + # native llama_decode call. Invalid ids may otherwise reach the C/C++ backend + # and cause hard crashes instead of Python exceptions. + self._validate_eval_tokens(tokens) + # Context Shift: Prevent OOM by discarding older tokens when context limit is reached. if self.n_tokens + n_eval > self._n_ctx: # 0. Check if the memory supports shifting @@ -1265,9 +1301,11 @@ def eval( current_batch_size //= 2 except Exception as e: + min_pos = min(current_batch_size, 16) + preview = chunk[:min_pos] # Catch fatal backend failures (e.g., Code -2, -3) raise RuntimeError(f"Llama.eval(decode): Fatal Decode Error at Pos {self.n_tokens}, " - f"Batch size {current_batch_size}: {str(e)}") from e + f"Batch size {current_batch_size}, chunk[:{min_pos}]={preview}: {str(e)}") from e if not success: raise RuntimeError("Llama.eval(decode): Failed completely even with batch size 1.") diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index 5ef1a2feb9..5a41e58366 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -1010,7 +1010,12 @@ def __call__( if tokens_to_eval: if self.verbose: - print(f"{self.log_prefix}(__call__): Evaluating TEXT chunk ({len(tokens_to_eval)} tokens) at pos {llama.n_tokens}...", file=sys.stderr) + print( + f"{self.log_prefix}(__call__): Evaluating TEXT chunk " + f"({len(tokens_to_eval)} tokens) at pos {llama.n_tokens}...", + file=sys.stderr, + ) + # Text evaluation delegates shift and chunking to native llama.eval llama.eval(tokens_to_eval) n_past = llama.n_tokens From ae50508bcf256ad149592477035fa34145b3ea76 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Jul 2026 02:42:16 +0800 Subject: [PATCH 228/304] Update Submodule vendor/llama.cpp a646006..049326a Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a646006f09..049326a000 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a646006f09d2f76f2d62d6c0d5e8e8490d570720 +Subproject commit 049326a00025d00b08cc188ed716b681e984a3f8 From 0fdbc3d364c300e0ce985ebc96ed5905fe4a3cc1 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Jul 2026 04:09:53 +0800 Subject: [PATCH 229/304] fix(_internals): harden LlamaBatch token writes Clarify llama_batch token vs embedding allocation semantics and keep future embedding/mixed-batch support open. Add token-buffer checks before add_token/add_sequence, validate add_sequence input lengths and seq_ids, and improve error messages for invalid batch configuration. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 104 +++++++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 18 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 431077822d..5058b7e321 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -1008,9 +1008,11 @@ def __init__( ): # logical validity of parameters if n_tokens <= 0: - raise ValueError(f"n_tokens must be positive, got {n_tokens}") + raise ValueError(f"LlamaBatch[__init__]: n_tokens must be positive, got {n_tokens}") + if embd < 0: + raise ValueError(f"LlamaBatch[__init__]: embd must be non-negative, got {embd}") if n_seq_max <= 0: - raise ValueError(f"n_seq_max must be positive, got {n_seq_max}") + raise ValueError(f"LlamaBatch[__init__]: n_seq_max must be positive, got {n_seq_max}") self.n_tokens_capacity = n_tokens self.embd = embd @@ -1018,11 +1020,24 @@ def __init__( self.verbose = verbose self._exit_stack = ExitStack() - batch = llama_cpp.llama_batch_init(self.n_tokens_capacity, self.embd, self.n_seq_max) + # llama_batch_init allocates either batch.token or batch.embd: + # + # embd == 0 -> token batch + # embd > 0 -> embedding batch + # + # Some llama.cpp paths, such as EAGLE3/MTP, manually create mixed + # token+embd batches after initialization. This wrapper keeps that + # possibility open, but add_token/add_sequence only support token input. + batch = llama_cpp.llama_batch_init( + self.n_tokens_capacity, + self.embd, + self.n_seq_max, + ) if batch is None: raise MemoryError( - f"Failed to allocate memory for llama_batch via llama_batch_init({n_tokens},{embd},{n_seq_max})" + f"Failed to allocate memory for llama_batch via " + f"llama_batch_init({n_tokens},{embd},{n_seq_max})" ) self.batch = batch @@ -1066,17 +1081,43 @@ def space_left(self) -> int: return self.n_tokens_capacity - self.batch.n_tokens else: raise RuntimeError( - f"LlamaBatch Critical Error: n_tokens ({self.batch.n_tokens}) exceeds capacity ({self.n_tokens_capacity}). " - "This implies a buffer overflow or corrupted internal state." + f"LlamaBatch Critical Error: n_tokens ({self.batch.n_tokens}) exceeds capacity " + f"({self.n_tokens_capacity}). This implies a buffer overflow or " + "corrupted internal state." ) def reset(self): """ - Resets the batch counter to 0. Does not free memory, just resets the index. - Call this before starting a new decoding step. + Reset the logical batch counter. + + This does not free or clear the underlying C buffers. llama_decode only + reads entries in [0, batch.n_tokens), so resetting n_tokens is enough and + matches llama.cpp's reusable batch pattern. """ - if self.batch is not None: - self.batch.n_tokens = 0 + if self.batch is None: + return + self.batch.n_tokens = 0 + + def _require_open(self, where: str) -> None: + if self.batch is None: + raise RuntimeError(f"LlamaBatch.{where}: batch has been closed.") + + def _require_token_buffer(self, where: str) -> None: + """ + Require that batch.token is available. + + llama_batch_init allocates batch.token only when embd == 0. Some advanced + llama.cpp paths manually create mixed token+embd batches, but this Python + token API should only write token ids when batch.token is non-null. + """ + self._require_open(where) + + if not bool(self.batch.token): + raise RuntimeError( + f"LlamaBatch.{where} requires a token buffer, but batch.token is NULL. " + "This batch was likely initialized as an embedding batch. Use a " + "separate embedding or mixed-batch path instead." + ) def add_token(self, token: int, pos: int, seq_ids: Sequence[int], logits: bool): """ @@ -1091,6 +1132,8 @@ def add_token(self, token: int, pos: int, seq_ids: Sequence[int], logits: bool): A single token can be part of multiple sequences simultaneously. logits: A boolean flag indicating whether the backend should compute logits for this token. """ + self._require_token_buffer("add_token") + idx = self.batch.n_tokens if idx >= self.n_tokens_capacity: raise IndexError(f"LlamaBatch overflow[add_token]: Cannot add token. Capacity {self.n_tokens_capacity} reached.") @@ -1114,23 +1157,37 @@ def add_sequence( self, token_array: Sequence[int], pos_array: Sequence[int], - seq_ids: Sequence[Sequence[int]], + seq_ids: Sequence[int], logits_array: Sequence[bool] ): """ - Adds a sequence of tokens to the batch in a vectorized manner. - Strictly maps the provided arrays to the underlying C++ batch structure without subjective overriding. + Adds a sequence of tokens to the batch. Args: - token_array: A sequence of token IDs to be evaluated. - pos_array: A sequence of logical positions corresponding to each token. - seq_id_array: A sequence of lists, where each list contains the sequence IDs for the respective token. - (e.g., [[0], [0], [0]] for 3 tokens belonging to sequence 0). - logits_array: A sequence of boolean flags indicating whether to compute logits for each token. + token_array: Token ids to evaluate. + pos_array: Logical positions for each token. + seq_ids: Sequence ids shared by every token in this call, usually [0]. + A token can belong to multiple sequences, for example [0, 1], + matching llama.cpp's per-token seq_id list. + logits_array: Whether to request logits/output for each token. """ + self._require_token_buffer("add_sequence") + n_tokens = len(token_array) current_count = self.batch.n_tokens + if len(pos_array) != n_tokens: + raise ValueError( + f"LlamaBatch.add_sequence: pos_array length mismatch: " + f"{len(pos_array)} != {n_tokens}." + ) + + if len(logits_array) != n_tokens: + raise ValueError( + f"LlamaBatch.add_sequence: logits_array length mismatch: " + f"{len(logits_array)} != {n_tokens}." + ) + if current_count + n_tokens > self.n_tokens_capacity: raise IndexError( f"LlamaBatch overflow[add_sequence]: Cannot add {n_tokens} tokens. " @@ -1138,12 +1195,23 @@ def add_sequence( ) n_seq_id = len(seq_ids) + if n_seq_id <= 0: + raise ValueError("LlamaBatch Error[add_sequence]: seq_ids must not be empty.") + if n_seq_id > self.n_seq_max: raise ValueError(f"LlamaBatch Error[add_sequence]: Token belongs to {n_seq_id} sequences, " f"but n_seq_max was initialized to {self.n_seq_max}.") + for seq_id in seq_ids: + if seq_id < 0 or seq_id >= self.n_seq_max: + raise ValueError( + f"LlamaBatch Error[add_sequence]: invalid seq_id {seq_id}; " + f"expected 0 <= seq_id < {self.n_seq_max}" + ) + for i in range(n_tokens): j = current_count + i + self.batch.token[j] = token_array[i] self.batch.pos[j] = pos_array[i] From a5ceecb723a58450ab5d9ee9fbfcb492a7f31c4b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Jul 2026 05:15:23 +0800 Subject: [PATCH 230/304] feat: add embedding rows to LlamaBatch - Add shared seq_id validation for token and embedding batch writes. - Introduce embedding-buffer checks plus add_embedding and add_embeddings helpers for embd-only llama_batch inputs, enabling decoder paths that consume external embedding rows while keeping token writes restricted to token buffers. - This prepares LlamaBatch for embedding-only decode paths, such as speculative decoding feature injection or external encoder/projector outputs. - It does not implement mixed token+embedding batches yet; those still need a separate ownership-safe design for the token buffer. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 183 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 165 insertions(+), 18 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 5058b7e321..c8f0496739 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -1119,6 +1119,33 @@ def _require_token_buffer(self, where: str) -> None: "separate embedding or mixed-batch path instead." ) + def _validate_seq_ids(self, seq_ids: Sequence[int], where: str) -> int: + n_seq_id = len(seq_ids) + + if n_seq_id <= 0: + raise ValueError(f"LlamaBatch.{where}: seq_ids must not be empty.") + + if n_seq_id > self.n_seq_max: + raise ValueError( + f"LlamaBatch.{where}: token belongs to {n_seq_id} sequences, " + f"but n_seq_max was initialized to {self.n_seq_max}." + ) + + for seq_id in seq_ids: + if not isinstance(seq_id, int): + raise ValueError( + f"LlamaBatch.{where}: seq_id must be int, got " + f"{type(seq_id).__name__}." + ) + + if seq_id < 0 or seq_id >= self.n_seq_max: + raise ValueError( + f"LlamaBatch.{where}: invalid seq_id {seq_id}; " + f"expected 0 <= seq_id < {self.n_seq_max}." + ) + + return n_seq_id + def add_token(self, token: int, pos: int, seq_ids: Sequence[int], logits: bool): """ Adds a single token to the batch. @@ -1141,10 +1168,8 @@ def add_token(self, token: int, pos: int, seq_ids: Sequence[int], logits: bool): self.batch.token[idx] = token self.batch.pos[idx] = pos - n_seq_id = len(seq_ids) - if n_seq_id > self.n_seq_max: - raise ValueError(f"LlamaBatch Error[add_token]: Token belongs to {n_seq_id} sequences, " - f"but n_seq_max was initialized to {self.n_seq_max}.") + n_seq_id = self._validate_seq_ids(seq_ids, "add_token") + self.batch.n_seq_id[idx] = n_seq_id for i, seq_id in enumerate(seq_ids): @@ -1194,20 +1219,7 @@ def add_sequence( f"Space left: {self.n_tokens_capacity - current_count}" ) - n_seq_id = len(seq_ids) - if n_seq_id <= 0: - raise ValueError("LlamaBatch Error[add_sequence]: seq_ids must not be empty.") - - if n_seq_id > self.n_seq_max: - raise ValueError(f"LlamaBatch Error[add_sequence]: Token belongs to {n_seq_id} sequences, " - f"but n_seq_max was initialized to {self.n_seq_max}.") - - for seq_id in seq_ids: - if seq_id < 0 or seq_id >= self.n_seq_max: - raise ValueError( - f"LlamaBatch Error[add_sequence]: invalid seq_id {seq_id}; " - f"expected 0 <= seq_id < {self.n_seq_max}" - ) + n_seq_id = self._validate_seq_ids(seq_ids, "add_sequence") for i in range(n_tokens): j = current_count + i @@ -1223,6 +1235,141 @@ def add_sequence( self.batch.n_tokens += n_tokens + def _require_embedding_buffer(self, where: str) -> None: + self._require_open(where) + + if self.embd <= 0: + raise RuntimeError( + f"LlamaBatch.{where} requires an embedding batch, but embd={self.embd}." + ) + + if not bool(self.batch.embd): + raise RuntimeError( + f"LlamaBatch.{where} requires batch.embd, but batch.embd is NULL." + ) + + def add_embedding( + self, + embedding: Sequence[float], + pos: int, + seq_ids: Sequence[int], + logits: bool = False, + ) -> None: + """ + Add one embedding row to an embedding batch. + + This is for embd-only llama_batch input: + batch.token == NULL + batch.embd != NULL + + Args: + embedding: One embedding vector of length self.embd. + pos: Logical sequence position. + seq_ids: Sequence ids this embedding belongs to, usually [0]. + logits: Whether to request output for this row. + """ + self._require_embedding_buffer("add_embedding") + + if len(embedding) != self.embd: + raise ValueError( + f"LlamaBatch.add_embedding: embedding length mismatch: " + f"{len(embedding)} != embd({self.embd})." + ) + + idx = self.batch.n_tokens + if idx >= self.n_tokens_capacity: + raise IndexError( + f"LlamaBatch overflow[add_embedding]: capacity " + f"{self.n_tokens_capacity} reached." + ) + + n_seq_id = self._validate_seq_ids(seq_ids, "add_embedding") + + base = idx * self.embd + for d, value in enumerate(embedding): + self.batch.embd[base + d] = float(value) + + self.batch.pos[idx] = pos + self.batch.n_seq_id[idx] = n_seq_id + + for i, seq_id in enumerate(seq_ids): + self.batch.seq_id[idx][i] = seq_id + + self.batch.logits[idx] = logits + self.batch.n_tokens += 1 + + def add_embeddings( + self, + embeddings: Sequence[float], + *, + pos_array: Sequence[int], + seq_ids: Sequence[int], + logits_array: Optional[Sequence[bool]] = None, + ) -> None: + """ + Add multiple embedding rows to an embedding batch. + + embeddings layout: + row-major [n_tokens, self.embd] + + The number of rows is inferred from pos_array. This method supports + embedding-only llama_batch inputs: + + batch.token == NULL + batch.embd != NULL + + It only supports one logical position per embedding row. M-RoPE media + embedding batches should continue to use MTMD helper APIs. + """ + self._require_embedding_buffer("add_embeddings") + + n_tokens = len(pos_array) + if n_tokens <= 0: + raise ValueError("LlamaBatch.add_embeddings: pos_array must not be empty.") + + if logits_array is None: + logits_array = [False] * n_tokens + elif len(logits_array) != n_tokens: + raise ValueError( + f"LlamaBatch.add_embeddings: logits_array length mismatch: " + f"{len(logits_array)} != {n_tokens}." + ) + + expected = n_tokens * self.embd + if len(embeddings) != expected: + raise ValueError( + f"LlamaBatch.add_embeddings: embeddings length mismatch: " + f"{len(embeddings)} != n_tokens({n_tokens}) * embd({self.embd}) = {expected}." + ) + + current_count = self.batch.n_tokens + if current_count + n_tokens > self.n_tokens_capacity: + raise IndexError( + f"LlamaBatch overflow[add_embeddings]: cannot add {n_tokens} rows. " + f"Space left: {self.n_tokens_capacity - current_count}." + ) + + n_seq_id = self._validate_seq_ids(seq_ids, "add_embeddings") + + for i in range(n_tokens): + j = current_count + i + + src_base = i * self.embd + dst_base = j * self.embd + + for d in range(self.embd): + self.batch.embd[dst_base + d] = float(embeddings[src_base + d]) + + self.batch.pos[j] = int(pos_array[i]) + self.batch.n_seq_id[j] = n_seq_id + + for k, seq_id in enumerate(seq_ids): + self.batch.seq_id[j][k] = int(seq_id) + + self.batch.logits[j] = int(logits_array[i]) + + self.batch.n_tokens += n_tokens + # Embedding functions def normalize_embedding(embedding): From e3bd06bfde0018116e429bf6fbe6917ed0aa410e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Jul 2026 21:23:28 +0800 Subject: [PATCH 231/304] Update Submodule vendor/llama.cpp 049326a..a935fbf Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 049326a000..a935fbffe1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 049326a00025d00b08cc188ed716b681e984a3f8 +Subproject commit a935fbffe1a3d31509c325c116454ab5d56b2eb8 From 11364d1cb0ae10e3a951375c4ca29e99d6213206 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Jul 2026 22:14:38 +0800 Subject: [PATCH 232/304] feat(LlamaBatch): add mixed token embedding batch support - Add optional mixed=True initialization for LlamaBatch so token+embedding rows can be represented in a single llama_batch. Mixed batches keep the native embd buffer from llama_batch_init and attach a Python-owned token buffer, which is cleared before llama_batch_free() to avoid invalid ownership. - Route token-only and embedding-only write APIs away from mixed batches, add mixed-batch validation, and introduce add_token_embedding for EAGLE3/MTP-style decoder inputs containing both token ids and embedding vectors. - This prepares LlamaBatch for speculative decoding paths that require mixed token+hidden-state inputs, especially EAGLE3 and MTP. It keeps ordinary token-only and embedding-only APIs separated while providing a dedicated add_token_embedding path for mixed decoder rows. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 117 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index c8f0496739..4eb2df7db5 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -1004,6 +1004,7 @@ def __init__( n_tokens: int, embd: int, n_seq_max: int, + mixed: bool = False, verbose: bool = True ): # logical validity of parameters @@ -1013,11 +1014,16 @@ def __init__( raise ValueError(f"LlamaBatch[__init__]: embd must be non-negative, got {embd}") if n_seq_max <= 0: raise ValueError(f"LlamaBatch[__init__]: n_seq_max must be positive, got {n_seq_max}") + if mixed and embd <= 0: + raise ValueError("LlamaBatch[__init__]: mixed batch requires embd > 0.") self.n_tokens_capacity = n_tokens self.embd = embd self.n_seq_max = n_seq_max + self.mixed = mixed self.verbose = verbose + self._token_buf = None + self._owns_token = False self._exit_stack = ExitStack() # llama_batch_init allocates either batch.token or batch.embd: @@ -1040,17 +1046,42 @@ def __init__( f"llama_batch_init({n_tokens},{embd},{n_seq_max})" ) + if mixed: + if bool(batch.token): + raise RuntimeError( + "LlamaBatch[__init__]: expected batch.token to be NULL for " + "mixed embedding batch initialized with embd > 0." + ) + if not bool(batch.embd): + raise RuntimeError( + "LlamaBatch[__init__]: expected batch.embd to be non-NULL " + "for mixed batch." + ) + + self._token_buf = ( + llama_cpp.llama_token * self.n_tokens_capacity + )() + batch.token = self._token_buf + self._owns_token = True + self.batch = batch def close(self): """Manually free LlamaBatch resources.""" if getattr(self, "batch", None) is not None: try: + if getattr(self, "_owns_token", False): + # batch.token points to a Python-owned ctypes buffer in mixed mode. + # llama_batch_free() would call free(batch.token), so clear it first. + self.batch.token = None llama_cpp.llama_batch_free(self.batch) except Exception: pass self.batch = None + self._token_buf = None + self._owns_token = False + if getattr(self, "_exit_stack", None) is not None and hasattr(self._exit_stack, "close"): self._exit_stack.close() self._exit_stack = None @@ -1112,6 +1143,12 @@ def _require_token_buffer(self, where: str) -> None: """ self._require_open(where) + if self.mixed: + raise RuntimeError( + f"LlamaBatch.{where} is for token-only batches. " + "Use add_token_embedding for mixed batches." + ) + if not bool(self.batch.token): raise RuntimeError( f"LlamaBatch.{where} requires a token buffer, but batch.token is NULL. " @@ -1238,6 +1275,12 @@ def add_sequence( def _require_embedding_buffer(self, where: str) -> None: self._require_open(where) + if self.mixed: + raise RuntimeError( + f"LlamaBatch.{where} is for embedding-only batches. " + "Use add_token_embedding for mixed batches." + ) + if self.embd <= 0: raise RuntimeError( f"LlamaBatch.{where} requires an embedding batch, but embd={self.embd}." @@ -1370,6 +1413,80 @@ def add_embeddings( self.batch.n_tokens += n_tokens + def _require_mixed_buffer(self, where: str) -> None: + self._require_open(where) + + if not self.mixed: + raise RuntimeError( + f"LlamaBatch.{where} requires mixed=True batch." + ) + + if self.embd <= 0: + raise RuntimeError( + f"LlamaBatch.{where} requires mixed token+embedding batch, " + f"but embd={self.embd}." + ) + + if not bool(self.batch.token): + raise RuntimeError( + f"LlamaBatch.{where} requires batch.token, but batch.token is NULL." + ) + + if not bool(self.batch.embd): + raise RuntimeError( + f"LlamaBatch.{where} requires batch.embd, but batch.embd is NULL." + ) + + def add_token_embedding( + self, + token: int, + embedding: Sequence[float], + pos: int, + seq_ids: Sequence[int], + logits: bool, + ) -> None: + """ + Add one mixed token+embedding row. + + This is for EAGLE3/MTP-style decoder inputs where each batch row contains: + token id + embedding vector + position + seq ids + logits flag + """ + self._require_mixed_buffer("add_token_embedding") + + if len(embedding) != self.embd: + raise ValueError( + f"LlamaBatch.add_token_embedding: embedding length mismatch: " + f"{len(embedding)} != embd({self.embd})." + ) + + idx = self.batch.n_tokens + if idx >= self.n_tokens_capacity: + raise IndexError( + f"LlamaBatch overflow[add_token_embedding]: capacity " + f"{self.n_tokens_capacity} reached." + ) + + n_seq_id = self._validate_seq_ids(seq_ids, "add_token_embedding") + + self.batch.token[idx] = token + + base = idx * self.embd + for d, value in enumerate(embedding): + self.batch.embd[base + d] = float(value) + + self.batch.pos[idx] = pos + self.batch.n_seq_id[idx] = n_seq_id + + for i, seq_id in enumerate(seq_ids): + self.batch.seq_id[idx][i] = seq_id + + self.batch.logits[idx] = logits + self.batch.n_tokens += 1 + # Embedding functions def normalize_embedding(embedding): From 7f59a8611d76b4e202750355df7d01f3270a15da Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 10 Jul 2026 23:42:27 +0800 Subject: [PATCH 233/304] fix(mtmd): validate MTMD inputs before tokenization - Add Python-side MTMD input validation before calling the native mtmd_tokenize path. Normalize missing bitmap lists to empty lists for pure text prompts, check that rendered media markers match decoded bitmap inputs, reject missing bitmap entries, and validate that the media marker is available. - Improve media placeholder mismatch errors with marker counts and marker details, and surface mtmd_tokenize failures with richer diagnostic context including media counts and backend support flags. Signed-off-by: JamePeng --- llama_cpp/llama_multimodal.py | 130 ++++++++++++++++++++++++++++------ 1 file changed, 108 insertions(+), 22 deletions(-) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index 5a41e58366..a8802baf69 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -565,6 +565,10 @@ def _replace_media_placeholders( media payloads. """ media_marker = self.media_marker + if not media_marker: + raise ValueError( + f"{self.log_prefix}(_replace_media_placeholders): media marker must not be empty." + ) # 1. Replace known template-specific media tags first. # @@ -583,12 +587,19 @@ def _replace_media_placeholders( if url and url in text: text = text.replace(url, media_marker, 1) - # 3. Validate only after all normalization is complete. + # 3. Validate after all normalization is complete. marker_count = text.count(media_marker) - if marker_count != len(media_items): + media_count = len(media_items) + + if marker_count != media_count: raise ValueError( - f"{self.log_prefix}(_replace_media_placeholders): media marker mismatch " - f"(marker_count={marker_count}, media_count={len(media_items)})" + f"{self.log_prefix}(_replace_media_placeholders): media marker mismatch\n" + f"- marker_count={marker_count}\n" + f"- media_count={media_count}\n" + f"- media_marker={media_marker!r}\n" + "Each media item must render to exactly one MTMD media marker. " + "Check whether the chat template rendered both a media tag and the " + "original URL/data URI, or failed to render a media placeholder." ) return text @@ -620,11 +631,68 @@ def _render_and_replace_media( media_items=media_items, ) + def _validate_mtmd_inputs( + self, + *, + text: str, + bitmaps: Optional[List[Any]] = None, + ) -> None: + """ + Validate Python-side MTMD tokenizer inputs before calling mtmd_tokenize. + + This mirrors the most important checks in llama.cpp mtmd_tokenizer: + - mtmd context must be initialized + - rendered text must be a string + - media marker must not be empty + - media marker count must match bitmap count + - bitmap entries must not be None + + Pure text input is valid: + bitmaps is None or [] + marker_count == 0 + """ + if self.mtmd_ctx is None: + raise ValueError( + f"{self.log_prefix}(_validate_mtmd_inputs): mtmd context not initialized." + ) + + if not isinstance(text, str): + raise TypeError( + f"{self.log_prefix}(_validate_mtmd_inputs): text must be str, " + f"got {type(text).__name__}." + ) + + if not self.media_marker: + raise ValueError( + f"{self.log_prefix}(_validate_mtmd_inputs): media marker must not be empty." + ) + + if bitmaps is None: + bitmaps = [] + + marker_count = text.count(self.media_marker) + bitmap_count = len(bitmaps) + + if marker_count != bitmap_count: + raise ValueError( + f"{self.log_prefix}(_validate_mtmd_inputs): media marker mismatch\n" + f"- marker_count={marker_count}\n" + f"- bitmap_count={bitmap_count}\n" + f"- media_marker={self.media_marker!r}\n" + "The rendered prompt must contain exactly one media marker per decoded media input." + ) + + for i, bitmap in enumerate(bitmaps): + if bitmap is None: + raise ValueError( + f"{self.log_prefix}(_validate_mtmd_inputs): bitmap[{i}] is None." + ) + def _mtmd_tokenize( self, llama: "llama_core.Llama", text: str, - bitmaps: Optional[list] = None, + bitmaps: Optional[List[Any]] = None, chunks: Optional[Any] = None, ) -> Any: """ @@ -637,7 +705,19 @@ def _mtmd_tokenize( - stable interface for future async/batch decoding - isolated error handling for tokenizer failures - clean separation between prompt building and C++ binding + - strict Python-side marker/bitmap validation before native tokenization + + Pure text input is valid: + bitmaps is None or [] + marker_count == 0 """ + if bitmaps is None: + bitmaps = [] + + self._validate_mtmd_inputs( + text=text, + bitmaps=bitmaps, + ) if chunks is None: chunks = self._mtmd_cpp.mtmd_input_chunks_init() @@ -646,30 +726,19 @@ def _mtmd_tokenize( f"{self.log_prefix}(_mtmd_tokenize): failed to init mtmd_input_chunks" ) - # Validate strict alignment between rendered media markers and provided bitmaps - # to ensure MTMD tokenization consistency and prevent decoding mismatch errors. - if bitmaps is not None: - marker_count = text.count(self.media_marker) - if marker_count != len(bitmaps): - raise ValueError( - f"{self.log_prefix}(_mtmd_tokenize): marker mismatch " - f"(marker_count={marker_count}, bitmap_count={len(bitmaps)})" - ) - input_text = self._mtmd_cpp.mtmd_input_text() input_text.text = ctypes.c_char_p(text.encode("utf-8")) input_text.add_special = (llama.n_tokens == 0) input_text.parse_special = True - bitmap_array = None - n_bitmaps = 0 + n_bitmaps = len(bitmaps) - if bitmaps: - n_bitmaps = len(bitmaps) - bitmap_array = (self._mtmd_cpp.mtmd_bitmap_p_ctypes * n_bitmaps)(*bitmaps) + if n_bitmaps > 0: + bitmap_array = ( + self._mtmd_cpp.mtmd_bitmap_p_ctypes * n_bitmaps + )(*bitmaps) else: bitmap_array = None - n_bitmaps = 0 result = self._mtmd_cpp.mtmd_tokenize( self.mtmd_ctx, @@ -680,11 +749,19 @@ def _mtmd_tokenize( ) if result != 0: + marker_count = text.count(self.media_marker) raise ValueError( - f"{self.log_prefix}(_mtmd_tokenize): tokenize failed\n" + f"{self.log_prefix}(_mtmd_tokenize): mtmd_tokenize failed\n" f"- result={result}\n" f"- text_len={len(text)}\n" + f"- marker_count={marker_count}\n" f"- n_bitmaps={n_bitmaps}\n" + f"- supports_vision={self.is_support_vision}\n" + f"- supports_audio={self.is_support_audio}\n" + f"- supports_video={self.is_support_video}\n" + "Possible causes: marker/bitmap mismatch, invalid image/audio data, " + "unsupported vision/audio projector, failed media preprocessing, " + "or text tokenization failure." ) return chunks @@ -859,6 +936,15 @@ def _create_bitmap_func(idx: int, item: dict): else: raise TypeError(f"{self.log_prefix}(mtmd_input_chunk_get_type): Invalid chunk type, chunk_type = {chunk_type}.") + if media_items_cur != media_items_count: + raise RuntimeError( + f"{self.log_prefix}(_process_mtmd_prompt): not all media inputs were consumed by MTMD chunks\n" + f"- consumed={media_items_cur}\n" + f"- media_items={media_items_count}\n" + "This usually means the rendered prompt did not produce enough media chunks, " + "or the chat template/media marker normalization is incorrect." + ) + return full_prompt_ids, chunk_token_spans, chunks, bitmap_cleanup except Exception as e: From 36bdecf341852ee7975ceefd1ea7350cba61070f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 11 Jul 2026 21:11:18 +0800 Subject: [PATCH 234/304] fix(ggml): load ggml-base before ggml library - Load ggml-base shared library before ggml to ensure the base runtime dependency is initialized prior to loading the main ggml library. - This improves dynamic library loading reliability on platforms where ggml depends on ggml-base during initialization. Signed-off-by: JamePeng --- llama_cpp/_ggml.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index c4ae7c94bf..6ab16a9715 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -27,14 +27,14 @@ libggml_base_path / "bin", ] -libggml = load_shared_library("ggml", libggml_base_paths) - -ggml_function = ctypes_function_for_shared_library(libggml) - libggml_base = load_shared_library("ggml-base", libggml_base_paths) ggml_base_function = ctypes_function_for_shared_library(libggml_base) +libggml = load_shared_library("ggml", libggml_base_paths) + +ggml_function = ctypes_function_for_shared_library(libggml) + # // ====== ggml.h ====== GGML_FILE_MAGIC = 0x67676d6c # b"ggml" From 1992f6a43e297ac8602ccd51ec2c6c9662576062 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 11 Jul 2026 21:14:03 +0800 Subject: [PATCH 235/304] build: Update CMakeLists.txt - append ggml-et backend Signed-off-by: JamePeng --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b2cfeeb8c..2286fe5eed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -336,6 +336,7 @@ if (LLAMA_BUILD) set(GGML_BACKEND_TARGETS ggml-cann ggml-cuda + ggml-et ggml-hexagon ggml-hip ggml-metal From 6c954f42d1622e4f211abd3265f1d4fe2172f528 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 11 Jul 2026 21:21:24 +0800 Subject: [PATCH 236/304] Update Submodule vendor/llama.cpp a935fbf..c92e806 Signed-off-by: JamePeng --- llama_cpp/_ggml.py | 102 +++++++++++++++++++++++---------------------- vendor/llama.cpp | 2 +- 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 6ab16a9715..0cb20d40e6 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -292,6 +292,7 @@ class GGMLFType(enum.IntEnum): # GGML_OP_IM2COL, # GGML_OP_IM2COL_BACK, # GGML_OP_IM2COL_3D, +# GGML_OP_COL2IM_1D, # GGML_OP_CONV_2D, # GGML_OP_CONV_3D, # GGML_OP_CONV_2D_DW, @@ -324,6 +325,7 @@ class GGMLFType(enum.IntEnum): # GGML_OP_RWKV_WKV7, # GGML_OP_SOLVE_TRI, # GGML_OP_GATED_DELTA_NET, +# GGML_OP_LIGHTNING_INDEXER, # GGML_OP_UNARY, @@ -401,55 +403,57 @@ class GGML_OP(enum.IntEnum): GGML_OP_IM2COL = 52 GGML_OP_IM2COL_BACK = 53 GGML_OP_IM2COL_3D = 54 - GGML_OP_CONV_2D = 55 - GGML_OP_CONV_3D = 56 - GGML_OP_CONV_2D_DW = 57 - GGML_OP_CONV_TRANSPOSE_2D = 58 - GGML_OP_POOL_1D = 59 - GGML_OP_POOL_2D = 60 - GGML_OP_POOL_2D_BACK = 61 - GGML_OP_UPSCALE = 62 - GGML_OP_PAD = 63 - GGML_OP_PAD_REFLECT_1D = 64 - GGML_OP_ROLL = 65 - GGML_OP_ARANGE = 66 - GGML_OP_TIMESTEP_EMBEDDING = 67 - GGML_OP_ARGSORT = 68 - GGML_OP_TOP_K = 69 - GGML_OP_LEAKY_RELU = 70 - GGML_OP_TRI = 71 - GGML_OP_FILL = 72 - - GGML_OP_FLASH_ATTN_EXT = 73 - GGML_OP_FLASH_ATTN_BACK = 74 - GGML_OP_SSM_CONV = 75 - GGML_OP_SSM_SCAN = 76 - GGML_OP_WIN_PART = 77 - GGML_OP_WIN_UNPART = 78 - GGML_OP_GET_REL_POS = 79 - GGML_OP_ADD_REL_POS = 80 - GGML_OP_RWKV_WKV6 = 81 - GGML_OP_GATED_LINEAR_ATTN = 82 - GGML_OP_RWKV_WKV7 = 83 - GGML_OP_SOLVE_TRI = 84 - GGML_OP_GATED_DELTA_NET = 85 - - GGML_OP_UNARY = 86 - - GGML_OP_MAP_CUSTOM1 = 87 - GGML_OP_MAP_CUSTOM2 = 88 - GGML_OP_MAP_CUSTOM3 = 89 - - GGML_OP_CUSTOM = 90 - - GGML_OP_CROSS_ENTROPY_LOSS = 91 - GGML_OP_CROSS_ENTROPY_LOSS_BACK = 92 - GGML_OP_OPT_STEP_ADAMW = 93 - GGML_OP_OPT_STEP_SGD = 94 - - GGML_OP_GLU = 95 - - GGML_OP_COUNT = 96 + GGML_OP_COL2IM_1D = 55 + GGML_OP_CONV_2D = 56 + GGML_OP_CONV_3D = 57 + GGML_OP_CONV_2D_DW = 58 + GGML_OP_CONV_TRANSPOSE_2D = 59 + GGML_OP_POOL_1D = 60 + GGML_OP_POOL_2D = 61 + GGML_OP_POOL_2D_BACK = 62 + GGML_OP_UPSCALE = 63 + GGML_OP_PAD = 64 + GGML_OP_PAD_REFLECT_1D = 65 + GGML_OP_ROLL = 66 + GGML_OP_ARANGE = 67 + GGML_OP_TIMESTEP_EMBEDDING = 68 + GGML_OP_ARGSORT = 69 + GGML_OP_TOP_K = 70 + GGML_OP_LEAKY_RELU = 71 + GGML_OP_TRI = 72 + GGML_OP_FILL = 73 + + GGML_OP_FLASH_ATTN_EXT = 74 + GGML_OP_FLASH_ATTN_BACK = 75 + GGML_OP_SSM_CONV = 76 + GGML_OP_SSM_SCAN = 77 + GGML_OP_WIN_PART = 78 + GGML_OP_WIN_UNPART = 79 + GGML_OP_GET_REL_POS = 80 + GGML_OP_ADD_REL_POS = 81 + GGML_OP_RWKV_WKV6 = 82 + GGML_OP_GATED_LINEAR_ATTN = 83 + GGML_OP_RWKV_WKV7 = 84 + GGML_OP_SOLVE_TRI = 85 + GGML_OP_GATED_DELTA_NET = 86 + GGML_OP_LIGHTNING_INDEXER = 87 + + GGML_OP_UNARY = 88 + + GGML_OP_MAP_CUSTOM1 = 89 + GGML_OP_MAP_CUSTOM2 = 90 + GGML_OP_MAP_CUSTOM3 = 91 + + GGML_OP_CUSTOM = 92 + + GGML_OP_CROSS_ENTROPY_LOSS = 93 + GGML_OP_CROSS_ENTROPY_LOSS_BACK = 94 + GGML_OP_OPT_STEP_ADAMW = 95 + GGML_OP_OPT_STEP_SGD = 96 + + GGML_OP_GLU = 97 + + GGML_OP_COUNT = 98 # enum ggml_unary_op { # GGML_UNARY_OP_ABS, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a935fbffe1..c92e806d1c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a935fbffe1a3d31509c325c116454ab5d56b2eb8 +Subproject commit c92e806d1c81091c9035edce99c35374da1b465e From 516ec3fbb91d02f710d36a8cab7e45a63884260d Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 12 Jul 2026 00:10:37 +0800 Subject: [PATCH 237/304] fix(loader): improve Windows DLL search path handling and diagnostics - Remove duplicated Windows DLL directory registration logic - Add optional CUDA, HIP, and Vulkan runtime DLL search paths - Keep bundled library paths with correct priority order for loading - Add comments explaining DLL search path ordering behavior - Add load source diagnostics for system and bundled libraries - Improve visibility when debugging shared library loading issues Signed-off-by: JamePeng --- llama_cpp/_ctypes_extensions.py | 41 +++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index 1a9f8eb8c5..1b4c05ff26 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -90,17 +90,8 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list # Add the library directory to the DLL search path on Windows (if needed) if sys.platform == "win32": - for base_path in base_paths: - p = pathlib.Path(base_path) - if p.exists() and p.is_dir(): - os.add_dll_directory(str(p)) - os.environ["PATH"] = str(p) + os.pathsep + os.environ["PATH"] - if sys.platform == "win32" and sys.version_info >= (3, 9): - for base_path in base_paths: - p = pathlib.Path(base_path) - if p.exists() and p.is_dir(): - os.add_dll_directory(str(p)) + # Add CUDA runtime DLL directories if CUDA is available. if "CUDA_PATH" in os.environ: cuda_path = os.environ["CUDA_PATH"] sub_dirs_to_add = [ @@ -114,14 +105,36 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list if os.path.exists(full_path): os.add_dll_directory(full_path) + # Add HIP runtime DLL directories when HIP backend is available. if "HIP_PATH" in os.environ: os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) + # Add Vulkan SDK DLL directories when Vulkan backend is enabled. if "VULKAN_SDK" in os.environ: os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Bin")) os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Lib")) + # Add package-provided library directories. + # + # The paths are added in reverse order intentionally. + # This ensures that the first entry in base_paths gets prepended + # to PATH last, making it the highest priority search location. + # + # Example: + # base_paths = [ + # package/lib, + # package/bin, + # ] + # + # After reversed iteration: + # PATH = package/lib;package/bin;... + for base_path in reversed(base_paths): + p = pathlib.Path(base_path) + if p.exists() and p.is_dir(): + os.add_dll_directory(str(p)) + os.environ["PATH"] = str(p) + os.pathsep + os.environ["PATH"] + cdll_args["winmode"] = ctypes.RTLD_GLOBAL errors = [] @@ -130,7 +143,9 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list lib_path = find_library(lib_base_name) if lib_path: try: - return ctypes.CDLL(lib_path, **cdll_args) + lib = ctypes.CDLL(lib_path, **cdll_args) + print(f"[llama-cpp-python].find_library: loaded library from {lib_path}") + return lib except Exception as e: errors.append(f"{lib_path}: {e}") @@ -141,7 +156,9 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list if lib_path.exists(): try: - return ctypes.CDLL(str(lib_path), **cdll_args) + lib = ctypes.CDLL(str(lib_path), **cdll_args) + print(f"[llama-cpp-python].provided_path: loaded library from {lib_path}") + return lib except Exception as e: errors.append(f"{lib_path}: {e}") From 3da4c603612c3344031b32ffbeb1da1c84bb205a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 12 Jul 2026 08:34:39 +0800 Subject: [PATCH 238/304] Update Submodule vendor/llama.cpp c92e806..e3546c7 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index c92e806d1c..e3546c7948 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit c92e806d1c81091c9035edce99c35374da1b465e +Subproject commit e3546c7948e3af463d0b401e6421d5a4c2faf565 From e522cecb93907c67ffe2e339b7009c93d3fb0f59 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 12 Jul 2026 09:03:57 +0800 Subject: [PATCH 239/304] Bump version to 0.3.42 - This release mainly addresses several issues discovered during real-world backend deployment, especially on Windows environments with dynamic backend loading, as well as improving the robustness of MTMD, batching, and native API interactions. Signed-off-by: JamePeng --- CHANGELOG.md | 95 +++++++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83c3beb128..9654d354ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,101 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.42] More Reliable Dynamic Backend Loading, Safer MTMD Processing, and Advanced Batch Support + +- fix(loader): improve Windows DLL search path handling and diagnostics + - Remove duplicated Windows DLL directory registration logic + - Add optional CUDA, HIP, and Vulkan runtime DLL search paths + - Keep bundled library paths with correct priority order for loading, need `/lib` > `/bin` + - Add comments explaining DLL search path ordering behavior + - Add load source diagnostics for system and bundled libraries + - Improve visibility when debugging shared library loading issues + + **Note**: + * For most single-DLL backends, the bin directory can still work as a fallback search path. However, some cases may fail due to missing dependencies such as `libomp140.x86_64.dll`. + * For `multi-DLL backends`, such as the `SYCL backend`, which depends on multiple DLLs (`dnnl.dll`, `tbb12.dll`, `mk_*.dll`, etc.), loading ggml-sycl.dll may fail when its dependent DLLs cannot be found, potentially resulting in an `access violation` crash. + * This update ensures that the DLL search path prioritizes /lib instead of /bin during the initial lookup stage, improving backend loading reliability. + * Special thanks to **@allanmeng** for reporting and testing the SYCL backend issue. + +- fix(ggml): load ggml-base before ggml library + - Load ggml-base shared library before ggml to ensure the base + runtime dependency is initialized prior to loading the main ggml + library. + + - This improves dynamic library loading reliability on platforms + where ggml depends on ggml-base during initialization. + +- fix(mtmd): validate MTMD inputs before tokenization + - Add Python-side MTMD input validation before calling the native mtmd_tokenize + path. Normalize missing bitmap lists to empty lists for pure text prompts, check + that rendered media markers match decoded bitmap inputs, reject missing bitmap + entries, and validate that the media marker is available. + + - Improve media placeholder mismatch errors with marker counts and marker details, + and surface mtmd_tokenize failures with richer diagnostic context including media + counts and backend support flags. + +- feat(LlamaBatch): add mixed token embedding batch support + - Add optional mixed=True initialization for LlamaBatch so token+embedding rows can + be represented in a single llama_batch. Mixed batches keep the native embd buffer + from llama_batch_init and attach a Python-owned token buffer, which is cleared + before llama_batch_free() to avoid invalid ownership. + + - Route token-only and embedding-only write APIs away from mixed batches, add + mixed-batch validation, and introduce add_token_embedding for EAGLE3/MTP-style + decoder inputs containing both token ids and embedding vectors. + + - This prepares LlamaBatch for speculative decoding paths that require mixed + token+hidden-state inputs, especially EAGLE3 and MTP. It keeps ordinary + token-only and embedding-only APIs separated while providing a dedicated + add_token_embedding path for mixed decoder rows. + +- feat(LlamaBatch): add embedding rows to LlamaBatch + - Add shared seq_id validation for token and embedding batch writes. + + - Introduce embedding-buffer checks plus add_embedding and add_embeddings helpers + for embd-only llama_batch inputs, enabling decoder paths that consume external + embedding rows while keeping token writes restricted to token buffers. + + - This prepares LlamaBatch for embedding-only decode paths, such as speculative + decoding feature injection or external encoder/projector outputs. + + - It does not implement mixed token+embedding batches yet; those still need a + separate ownership-safe design for the token buffer. + +- fix(LlamaBatch): harden LlamaBatch token writes + - Clarify llama_batch token vs embedding allocation semantics and keep future + embedding/mixed-batch support open. + + - Add token-buffer checks before add_token/add_sequence, validate add_sequence + input lengths and seq_ids, and improve error messages for invalid batch + configuration. + +- fix(eval): validate eval tokens before native decode + - Add token-id validation at the Llama.eval() boundary before context shifting, + batch construction, or llama_decode execution. This prevents invalid token + types, negative token ids, and out-of-vocabulary ids from reaching the native + decode path, where they may otherwise cause hard crashes instead of Python + exceptions. + + - Wrap llama_decode with defensive exception handling in LlamaContext.decode() so + native exceptions are surfaced with clearer diagnostic context. + + - Also include a small token preview in Llama.eval() fatal decode errors to make + backend failures easier to debug without changing the existing recoverable KV + slot handling behavior. + +- fix(types): make assistant message name optional + - Mark the assistant message `name` field as `NotRequired[Optional[str]]` + to match the optional nature of assistant message metadata and avoid + requiring callers to provide `name` in typed chat completion requests. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/e3546c7948e3af463d0b401e6421d5a4c2faf565](https://github.com/ggml-org/llama.cpp/commit/e3546c7948e3af463d0b401e6421d5a4c2faf565) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260711 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/169d5e1a43fb6ff4e5b6f5d0f26f1ec8acbd97b8...3da4c603612c3344031b32ffbeb1da1c84bb205a + ## [0.3.41] Template-Driven MTMD, Broader Multimodal Inputs, and Smarter N-Gram Drafting - refactor(mtmd): extract prompt rendering and media marker normalization diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 3c3aa6690c..c6676c294b 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.41" +__version__ = "0.3.42" From f077c566ec8b4211a00c7687d95eac742e5885b0 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 12 Jul 2026 09:40:44 +0800 Subject: [PATCH 240/304] ci(metal): get package version from importlib metadata Avoid importing llama_cpp when detecting the package version. This prevents initialization side effects and keeps CI version extraction reliable. Signed-off-by: JamePeng --- .github/workflows/build-wheels-metal.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml index 2b00d1abaa..caca8907f2 100644 --- a/.github/workflows/build-wheels-metal.yaml +++ b/.github/workflows/build-wheels-metal.yaml @@ -37,7 +37,7 @@ jobs: id: get_version shell: bash run: | - VERSION=$(python -c "import llama_cpp; print(llama_cpp.__version__)") + VERSION=$(python -c "import importlib.metadata; print(importlib.metadata.version('llama-cpp-python'))") echo "Detected version: $VERSION" echo "version=$VERSION" >> $GITHUB_OUTPUT From e417924131d0a26ade0668f64f7293a7d1c7280c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 13 Jul 2026 02:07:13 +0800 Subject: [PATCH 241/304] patch(Llama): Increase chunk preview limit to 128 in Llama.eval exception - Raises the maximum tokens captured for the error message preview from 16 to 128, improving visibility into the offending chunk during fatal backend crashes. Signed-off-by: JamePeng --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2ee8abaf5b..f1f71be8e3 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1301,7 +1301,7 @@ def eval( current_batch_size //= 2 except Exception as e: - min_pos = min(current_batch_size, 16) + min_pos = min(current_batch_size, 128) preview = chunk[:min_pos] # Catch fatal backend failures (e.g., Code -2, -3) raise RuntimeError(f"Llama.eval(decode): Fatal Decode Error at Pos {self.n_tokens}, " From 1762647fe473a8cfc25447b53b32316f2ca3c391 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 14 Jul 2026 22:50:33 +0800 Subject: [PATCH 242/304] Update Submodule vendor/llama.cpp e3546c7..7f575c3 Signed-off-by: JamePeng --- llama_cpp/llama_multimodal.py | 4 +++- llama_cpp/mtmd_cpp.py | 2 ++ vendor/llama.cpp | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index a8802baf69..166b69143f 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -727,7 +727,9 @@ def _mtmd_tokenize( ) input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = ctypes.c_char_p(text.encode("utf-8")) + encoded_text = text.encode("utf-8") + input_text.text = ctypes.c_char_p(encoded_text) + input_text.text_len = len(encoded_text) input_text.add_special = (llama.n_tokens == 0) input_text.parse_special = True diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 27a1a56d8d..fcfaa86ee7 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -178,12 +178,14 @@ class mtmd_pos_type(enum.IntEnum): # struct mtmd_input_text { # const char * text; +# size_t text_len; # bool add_special; # bool parse_special; # }; class mtmd_input_text(Structure): _fields_ = [ ("text", c_char_p), + ("text_len", c_size_t), ("add_special", c_bool), ("parse_special", c_bool), ] diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e3546c7948..7f575c39d6 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3546c7948e3af463d0b401e6421d5a4c2faf565 +Subproject commit 7f575c39d6a29a40c0ef22278eca6bd4a573c8a6 From 9d6e598623cad6a33684d04b23ced2ef0b86febe Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 15 Jul 2026 06:06:21 +0800 Subject: [PATCH 243/304] refactor(mtmd): cache Generic MTMD chat template resolution - Refactor MTMD chat template handling to resolve and analyze the chat template only once per handler instance instead of on every request. - Add template initialization state, cache parsed media placeholder tags, and support explicit chat template overrides through a dedicated field. Improve lifecycle cleanup by resetting cached template state and MTMD resources during handler close. - This keeps runtime processing focused on message rendering and media tokenization while avoiding repeated chat template resolution overhead. Signed-off-by: JamePeng --- llama_cpp/llama_multimodal.py | 49 ++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index 166b69143f..9e5b8af471 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -155,19 +155,23 @@ def __init__( f"{self.log_prefix}(__init__): `extra_template_arguments` must be a dict." ) + self.chat_format_override = chat_template_override self.extra_template_arguments: dict[str, Any] = dict(extra_template_arguments or {}) self.is_support_vision = False self.is_support_audio = False self.is_support_video = False + self.chat_template = None + self._chat_format_parser_tags = [] + self._template_initialized = False + # Pre-compile Jinja template - if (not hasattr(self, "chat_format") or self.chat_format is None) and chat_template_override is None: + if (not hasattr(self, "chat_format") or self.chat_format is None) and self.chat_format_override is not None: + self.chat_format = self.chat_format_override + elif self.chat_format is None and self.chat_format_override is None: self.chat_format = self.CHAT_FORMAT - elif chat_template_override is not None: - self.chat_format = chat_template_override - self._chat_format_parser_tags = [] self._change_chat_template(self.chat_format) self._exit_stack = ExitStack() @@ -250,11 +254,15 @@ def close(self) -> None: if getattr(self, "mtmd_ctx", None) is not None: try: self._mtmd_cpp.mtmd_free(self.mtmd_ctx) + self.mtmd_ctx = None except Exception: pass - self.mtmd_ctx = None - self.mctx_params = None - self.chat_template = None + self.mctx_params = None + self.chat_format = None + self.chat_template = None + self.chat_template_override = None + self._template_initialized = False + self._chat_format_parser_tags = [] if getattr(self, "_exit_stack", None) is not None and hasattr(self._exit_stack, "close"): self._exit_stack.close() @@ -1661,8 +1669,18 @@ def _resolve_chat_format(self, llama: llama_core.Llama) -> str: self.chat_format = chat_format return chat_format - def __call__(self, **kwargs): - llama = kwargs["llama"] + def _ensure_chat_template( + self, + llama: llama_core.Llama, + ) -> None: + """ + Resolve and analyze chat template once. + + Chat template metadata is static for a model instance, + so it should not be recomputed for every request. + """ + if self._template_initialized: + return self._resolve_chat_format(llama) @@ -1675,7 +1693,18 @@ def __call__(self, **kwargs): "a model that provides tokenizer.chat_template metadata." ) - self._chat_format_parser_tags = [tag for tag in self.KNOWN_MEDIA_TAGS if tag in self.chat_format] + self._chat_format_parser_tags = [ + tag + for tag in self.KNOWN_MEDIA_TAGS + if tag in self.chat_format + ] + + self._template_initialized = True + + def __call__(self, **kwargs): + llama = kwargs["llama"] + + self._ensure_chat_template(llama) if self.verbose: print(f"{self.log_prefix} - Start processing", file=sys.stderr) From 5334cb9025219811f10b8f0ac6b319e9c95fc0e3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 16 Jul 2026 01:42:58 +0800 Subject: [PATCH 244/304] Update Submodule vendor/llama.cpp 7f575c3..aff6eb6 Signed-off-by: JamePeng --- llama_cpp/_ggml.py | 6 +++++- llama_cpp/llama_cpp.py | 9 +++++++-- vendor/llama.cpp | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 0cb20d40e6..42ff664944 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -122,6 +122,7 @@ class GGMLStatus(enum.IntEnum): # GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) # GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) # GGML_TYPE_Q1_0 = 41, +# GGML_TYPE_Q2_0 = 42, # GGML_TYPE_COUNT = 42, # }; class GGMLType(enum.IntEnum): @@ -159,7 +160,8 @@ class GGMLType(enum.IntEnum): GGML_TYPE_MXFP4 = 39 GGML_TYPE_NVFP4 = 40 GGML_TYPE_Q1_0 = 41 - GGML_TYPE_COUNT = 42 + GGML_TYPE_Q2_0 = 42 + GGML_TYPE_COUNT = 43 # // precision @@ -201,6 +203,7 @@ class GGMLPrec(enum.IntEnum): # GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors # GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors # GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors +# GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors # }; class GGMLFType(enum.IntEnum): GGML_FTYPE_UNKNOWN = -1 @@ -230,6 +233,7 @@ class GGMLFType(enum.IntEnum): GGML_FTYPE_MOSTLY_MXFP4 = 25 GGML_FTYPE_MOSTLY_NVFP4 = 26 GGML_FTYPE_MOSTLY_Q1_0 = 27 + GGML_FTYPE_MOSTLY_Q2_0 = 28 # // available tensor operations: diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a4086fd704..a6bbb26d38 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -364,6 +364,7 @@ class llama_token_type(enum.IntEnum): # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -372,6 +373,9 @@ class llama_ftype(enum.IntEnum): LLAMA_FTYPE_MOSTLY_F16 = 1 LLAMA_FTYPE_MOSTLY_Q4_0 = 2 LLAMA_FTYPE_MOSTLY_Q4_1 = 3 + # LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4 + # LLAMA_FTYPE_MOSTLY_Q4_2 = 5 + # LLAMA_FTYPE_MOSTLY_Q4_3 = 6 LLAMA_FTYPE_MOSTLY_Q8_0 = 7 LLAMA_FTYPE_MOSTLY_Q5_0 = 8 LLAMA_FTYPE_MOSTLY_Q5_1 = 9 @@ -406,6 +410,7 @@ class llama_ftype(enum.IntEnum): LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_MOSTLY_NVFP4 = 39 LLAMA_FTYPE_MOSTLY_Q1_0 = 40 + LLAMA_FTYPE_MOSTLY_Q2_0 = 41 LLAMA_FTYPE_GUESSED = 1024 # // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" @@ -428,7 +433,7 @@ def llama_ftype_name( # LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, # LLAMA_ROPE_SCALING_TYPE_YARN = 2, # LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, -# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN, +# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE, # }; class llama_rope_scaling_type(enum.IntEnum): LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 @@ -436,7 +441,7 @@ class llama_rope_scaling_type(enum.IntEnum): LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 LLAMA_ROPE_SCALING_TYPE_YARN = 2 LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3 - LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN + LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE # enum llama_pooling_type { # LLAMA_POOLING_TYPE_UNSPECIFIED = -1, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7f575c39d6..aff6eb6e75 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7f575c39d6a29a40c0ef22278eca6bd4a573c8a6 +Subproject commit aff6eb6e7503538fec1532dec2f584bc7a4a4e4d From 859a99d74f70b9ab0ad2795225dda0e0f528dea6 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 16 Jul 2026 02:08:14 +0800 Subject: [PATCH 245/304] refactor(embedding): rename llama_cpp import alias to llama_cpp_lib Rename the 'llama_cpp.llama_cpp' import alias to 'llama_cpp_lib' to avoid potential namespace conflicts with the local '.llama_cpp' imports. Update all affected call sites in llama_embedding.py. Signed-off-by: JamePeng --- llama_cpp/llama_embedding.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama_embedding.py b/llama_cpp/llama_embedding.py index 0c1df339ce..d8f6e6bf21 100644 --- a/llama_cpp/llama_embedding.py +++ b/llama_cpp/llama_embedding.py @@ -1,6 +1,6 @@ import numpy as np from typing import Union, List, Optional, Dict, Any, Tuple -import llama_cpp.llama_cpp as llama_cpp +import llama_cpp.llama_cpp as llama_cpp_lib from .llama_types import Embedding from .llama import Llama # Pooling types from .llama_cpp @@ -141,7 +141,7 @@ def embed( # Determine the output dimension if is_rank: - out_dim = llama_cpp.llama_model_n_cls_out(self._model.model) + out_dim = llama_cpp_lib.llama_model_n_cls_out(self._model.model) else: out_dim = self.n_embd() @@ -166,9 +166,9 @@ def embed( # Reset Context and Batch if self.verbose: - llama_cpp.llama_perf_context_reset(ctx) + llama_cpp_lib.llama_perf_context_reset(ctx) self._batch.reset() - llama_cpp.llama_memory_clear(llama_cpp.llama_get_memory(ctx), True) + llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(ctx), True) # Initialize State Variables results: List[Any] = [] @@ -190,7 +190,7 @@ def _decode_batch(): doc_tokens_embd = [] for _ in range(seq_len): # Get the vector of the i-th token - ptr = llama_cpp.llama_get_embeddings_ith(ctx, curr_token_idx) + ptr = llama_cpp_lib.llama_get_embeddings_ith(ctx, curr_token_idx) if ptr is None: # Fallback: append zero vector or skip (here we zero-pad to keep shape) doc_tokens_embd.append([0.0] * out_dim) @@ -207,7 +207,7 @@ def _decode_batch(): else: for i in range(len(batch_seq_lens)): # Obtain the vector of the i-th sequence. - ptr = llama_cpp.llama_get_embeddings_seq(ctx, i) + ptr = llama_cpp_lib.llama_get_embeddings_seq(ctx, i) data = ptr[:out_dim] if not is_rank: @@ -219,7 +219,7 @@ def _decode_batch(): results.append(data) self._batch.reset() - llama_cpp.llama_memory_clear(llama_cpp.llama_get_memory(ctx), True) + llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(ctx), True) batch_seq_lens = [] # Main Streaming Loop @@ -272,7 +272,7 @@ def _decode_batch(): _decode_batch() if self.verbose: - llama_cpp.llama_perf_context_print(ctx) + llama_cpp_lib.llama_perf_context_print(ctx) final_result = results[0] if is_single else results From 849f02a445a944c2defffd2ff65e2678af32ee63 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 16 Jul 2026 03:43:05 +0800 Subject: [PATCH 246/304] feat(ctypes): support ABI-compatible symbol aliases - Allow ctypes_function_for_shared_library to accept either a single symbol name or an ordered iterable of ABI-compatible aliases. - Resolve aliases in order and bind the first exported symbol found while preserving the selected symbol name for runtime diagnostics. Also improve error reporting for empty alias lists and missing symbols. Signed-off-by: JamePeng --- llama_cpp/_ctypes_extensions.py | 55 ++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index 1b4c05ff26..03d7ee5cbc 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -9,6 +9,7 @@ from typing import ( Any, Callable, + Iterable, List, Union, Optional, @@ -201,20 +202,60 @@ class CtypesRef(Generic[CtypesCData]): def ctypes_function_for_shared_library(lib: ctypes.CDLL): - """Decorator for defining ctypes functions with type hints""" + """Create a decorator used to bind typed Python declarations to C symbols. + + The returned decorator accepts either a single exported symbol name or an + iterable of ABI-compatible aliases. When aliases are provided, they are + checked in order and the first available symbol is selected. + """ def ctypes_function( - name: str, argtypes: List[Any], restype: Any, enabled: bool = True + name: Union[str, Iterable[str]], + argtypes: List[Any], + restype: Any, + enabled: bool = True, ): + """Bind a Python declaration to one of the requested C symbols. + + Args: + name: A symbol name or an ordered iterable of compatible aliases. + argtypes: The ctypes argument types assigned to the C function. + restype: The ctypes return type assigned to the C function. + enabled: Return the original Python declaration when disabled. + + Raises: + ValueError: If no symbol names are provided. + AttributeError: If none of the requested symbols exist in the + shared library. + """ + symbol_names = (name,) if isinstance(name, str) else tuple(name) + + if not symbol_names: + raise ValueError("At least one shared library symbol name is required") + def decorator(f: F) -> F: - if enabled: - func = getattr(lib, name) + if not enabled: + return f + + for symbol_name in symbol_names: + try: + func = getattr(lib, symbol_name) + except AttributeError: + continue + func.argtypes = argtypes func.restype = restype - functools.wraps(f)(func) + functools.update_wrapper(func, f) + + # Preserve the actual exported symbol selected at runtime for + # diagnostics, especially when ABI aliases are being used. + func.__ctypes_symbol_name__ = symbol_name return func - else: - return f + + raise AttributeError( + "None of the shared library symbols were found: " + + ", ".join(symbol_names) + ) return decorator From 142c58d790e6961c97b68c967111b2afd3bb21f3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 16 Jul 2026 18:57:18 +0800 Subject: [PATCH 247/304] Update Submodule vendor/llama.cpp aff6eb6..79bba02 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index aff6eb6e75..79bba02a67 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit aff6eb6e7503538fec1532dec2f584bc7a4a4e4d +Subproject commit 79bba02a6741de194912d370015866414faa83ad From a804dc2c1483e5859693cf77f8b245bbf6a9c2c3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 16 Jul 2026 19:11:40 +0800 Subject: [PATCH 248/304] fix(mtmd): preserve subclass chat format during MTMD initialization - Ensure MTMDChatHandler initialization remains compatible with specialized chat handlers that define their own chat_format before calling super().__init__(). - Initialize chat_format only when it is not already provided by the subclass, then apply chat_format_override or fallback to the built-in MTMD template. This prevents AttributeError during inherited handler initialization while keeping template override behavior unchanged. Signed-off-by: JamePeng --- llama_cpp/llama_multimodal.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index 9e5b8af471..f3b7382e8a 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -155,6 +155,10 @@ def __init__( f"{self.log_prefix}(__init__): `extra_template_arguments` must be a dict." ) + # Preserve subclass attributes + if not hasattr(self, "chat_format"): + self.chat_format = None + self.chat_format_override = chat_template_override self.extra_template_arguments: dict[str, Any] = dict(extra_template_arguments or {}) @@ -167,10 +171,11 @@ def __init__( self._template_initialized = False # Pre-compile Jinja template - if (not hasattr(self, "chat_format") or self.chat_format is None) and self.chat_format_override is not None: - self.chat_format = self.chat_format_override - elif self.chat_format is None and self.chat_format_override is None: - self.chat_format = self.CHAT_FORMAT + if self.chat_format is None: + if self.chat_format_override is not None: + self.chat_format = self.chat_format_override + else: + self.chat_format = self.CHAT_FORMAT self._change_chat_template(self.chat_format) From afafdd59251e7fbef0e80ea2c705c1f0798dffd3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 17 Jul 2026 03:35:30 +0800 Subject: [PATCH 249/304] feat(ctypes): handle missing optional symbols gracefully - Allow ctypes bindings to mark symbols as optional through the `required` flag. - Missing symbols caused by ABI naming differences, API changes, or experimental extensions will no longer break library loading. Optional APIs emit diagnostic warnings and provide runtime unavailable stubs instead. Signed-off-by: JamePeng --- llama_cpp/_ctypes_extensions.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index 03d7ee5cbc..3c1709e750 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -214,6 +214,7 @@ def ctypes_function( argtypes: List[Any], restype: Any, enabled: bool = True, + required: bool = True, ): """Bind a Python declaration to one of the requested C symbols. @@ -222,6 +223,7 @@ def ctypes_function( argtypes: The ctypes argument types assigned to the C function. restype: The ctypes return type assigned to the C function. enabled: Return the original Python declaration when disabled. + required: Raise if symbol is missing. If False, create a runtime unavailable stub. Raises: ValueError: If no symbol names are provided. @@ -252,11 +254,37 @@ def decorator(f: F) -> F: func.__ctypes_symbol_name__ = symbol_name return func - raise AttributeError( + message = ( "None of the shared library symbols were found: " + ", ".join(symbol_names) ) + if required: + raise AttributeError(message) + + # Optional extension API. + # Keep import working when the symbol is unavailable. + print( + "[llama-cpp-python].ctypes_function: WARNING! optional API unavailable\n" + f" symbols: {', '.join(symbol_names)}\n" + f" library: {getattr(lib, '_name', '')}" + ) + + def unavailable(*args, **kwargs): + raise RuntimeError( + "This llama.cpp extension API is unavailable.\n" + f"Required symbol(s): {', '.join(symbol_names)}\n" + f"Library: {getattr(lib, '_name', '')}" + ) + + functools.update_wrapper(unavailable, f) + + # Mark unavailable extension API. + unavailable.__ctypes_symbol_name__ = None + unavailable.__ctypes_optional__ = True + + return unavailable + return decorator return ctypes_function From 083c4b59e812c7a3fc63f32af4b7c2c331da8c9c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 17 Jul 2026 04:38:36 +0800 Subject: [PATCH 250/304] Update Submodule vendor/llama.cpp 79bba02..e8f19cc Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 79bba02a67..e8f19cc0ad 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 79bba02a6741de194912d370015866414faa83ad +Subproject commit e8f19cc0ad70a243c8012bf17b4be601abfc8ea2 From 6c163c261c5d33492787b0de5c477de5d589e856 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 17 Jul 2026 04:46:10 +0800 Subject: [PATCH 251/304] fix(ctypes): validate argument types before binding shared library functions - Add explicit validation for ctypes function argument declarations before assigning them to the loaded shared library function. - This provides clearer error messages when invalid Python types are passed to `argtypes`, instead of exposing the internal ctypes error about missing `from_param()` methods. Signed-off-by: JamePeng --- llama_cpp/_ctypes_extensions.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index 3c1709e750..e88595f361 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -244,6 +244,18 @@ def decorator(f: F) -> F: func = getattr(lib, symbol_name) except AttributeError: continue + # Validate ctypes argument declarations before assigning them. + # ctypes requires every argtype to provide from_param(). + for index, argtype in enumerate(argtypes): + if not hasattr(argtype, "from_param"): + raise TypeError( + "Invalid ctypes argument type:\n" + f" function: {f.__name__}\n" + f" symbol: {symbol_name}\n" + f" arg index: {index}\n" + f" arg type: {argtype!r}\n" + f" expected: a ctypes type with from_param()" + ) func.argtypes = argtypes func.restype = restype From 978ff3252c6c56c2871470e67146b79aa3609e2e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 17 Jul 2026 04:51:52 +0800 Subject: [PATCH 252/304] feat(llama_ext): support optional llama-ext.h API bindings - Add Python ctypes bindings for the experimental APIs exposed by llama-ext.h, including NextN/MTP embeddings, and model metadata extraction. - Extension symbols are loaded optionally to handle ABI changes, renamed symbols, and builds that do not export experimental APIs without breaking the main Python bindings. Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 293 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 293 insertions(+) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a6bbb26d38..2f402cd74b 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -10,6 +10,7 @@ ggml_backend_sched_eval_callback, ggml_log_callback, ggml_opt_get_optimizer_params, + ggml_cgraph ) from typing import ( @@ -5069,3 +5070,295 @@ def llama_opt_epoch( callback_eval: ctypes.c_void_p, / ): ... + +############################## +# // llama.cpp/src/llama-ext.h +############################## + +# // this is a staging header for new llama.cpp API +# // breaking changes and C++ are allowed. everything here should be considered WIP +# // try as much as possible to not include this header in the rest of the codebase + +ctypes_function_llama_ext = ctypes_function_for_shared_library(_lib) + +# // Reserve a new compute graph. It is valid until the next call to llama_graph_reserve. +# LLAMA_API struct ggml_cgraph * llama_graph_reserve( +# struct llama_context * ctx, +# uint32_t n_tokens, +# uint32_t n_seqs, +# uint32_t n_outputs); +@ctypes_function_llama_ext( + [ + "llama_graph_reserve", + "?llama_graph_reserve@@YAPEAUggml_cgraph@@PEAUllama_context@@III@Z", + "__Z19llama_graph_reserveP13llama_contextjjj", + ], + [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32], + ctypes.POINTER(ggml_cgraph), + required=False, +) +def llama_graph_reserve( + ctx: llama_context_p, + n_tokens: ctypes.c_uint32, + n_seqs: ctypes.c_uint32, + n_outputs: ctypes.c_uint32, +) -> ctypes.POINTER(ggml_cgraph): # type: ignore + """ + Reserve a new compute graph. It is valid until the next call to llama_graph_reserve. + """ + ... + +# // Get the default ggml_type for a given ftype. +# LLAMA_API ggml_type llama_ftype_get_default_type(llama_ftype ftype); +@ctypes_function_llama_ext( + [ + "llama_ftype_get_default_type", + "?llama_ftype_get_default_type@@YA?AW4ggml_type@@W4llama_ftype@@@Z", + "__Z28llama_ftype_get_default_type11llama_ftype", + ], + [ctypes.c_int], + int, + required=False, +) +def llama_ftype_get_default_type( + ftype: llama_ftype +) -> int: + """ + Get the default ggml_type for a given ftype. + """ + ... + +# LLAMA_API int32_t llama_model_n_expert (const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_n_expert", + "?llama_model_n_expert@@YAHPEBUllama_model@@@Z", + "__Z20llama_model_n_expertPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.c_int32, + required=False, +) +def llama_model_n_expert( + model: llama_model_p +) -> ctypes.c_int32: + ... + +# LLAMA_API int32_t llama_model_n_devices(const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_n_devices", + "?llama_model_n_devices@@YAHPEBUllama_model@@@Z", + "__Z21llama_model_n_devicesPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.c_int32, + required=False, +) +def llama_model_n_devices( + model: llama_model_p +) -> ctypes.c_int32: + ... + +# LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * model, int i); +@ctypes_function_llama_ext( + [ + "llama_model_get_device", + "?llama_model_get_device@@YAPEAUggml_backend_device@@PEBUllama_model@@H@Z", + "__Z22llama_model_get_devicePK11llama_modeli", + ], + [llama_model_p_ctypes, ctypes.c_int], + ctypes.c_void_p, + required=False, +) +def llama_model_get_device( + model: llama_model_p, + i: int, +) -> ctypes.c_void_p: + ... + +# // Set whether the context outputs nextn embeddings or not +# // If masked == true, output the embeddings only for the tokens with batch.logits != 0 +# // If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits +# LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value, bool masked); +@ctypes_function_llama_ext( + [ + "llama_set_embeddings_nextn", + "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", + "__Z26llama_set_embeddings_nextnP13llama_contextbb", + ], + [llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], + None, + required=False, +) +def llama_set_embeddings_nextn( + ctx: llama_context_p, + value: bool, + masked: bool, +): + """ + Set whether the context outputs nextn embeddings or not + If masked == true, output the embeddings only for the tokens with batch.logits != 0 + If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits + """ + ... + +# // Select which appended NextN block the DECODER_MTP graph runs (offset past +# // the trunk: il = n_layer() + offset). Used by the speculative NextN driver to +# // chain multiple trained NextN heads. Default 0 (first head). +# LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset); +@ctypes_function_llama_ext( + [ + "llama_set_nextn_layer_offset", + "?llama_set_nextn_layer_offset@@YAXPEAUllama_context@@H@Z", + "__Z28llama_set_nextn_layer_offsetP13llama_contexti", + ], + [llama_context_p_ctypes, ctypes.c_int32], + None, + required=False, +) +def llama_set_nextn_layer_offset( + ctx: llama_context_p, + offset: ctypes.c_int32, +): + """ + Select which appended NextN block the DECODER_MTP graph runs (offset past + the trunk: il = n_layer() + offset). Used by the speculative NextN driver to + chain multiple trained NextN heads. Default 0 (first head). + """ + ... + +# // mirrors: +# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); +# LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_nextn", + "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", + "__Z26llama_get_embeddings_nextnP13llama_context", + ], + [llama_context_p_ctypes], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_nextn( + ctx: llama_context_p, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# // LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); +# LLAMA_API float * llama_get_embeddings_nextn_ith(struct llama_context * ctx, int32_t i); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_nextn_ith", + "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", + "__Z30llama_get_embeddings_nextn_ithP13llama_contexti", + ], + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_nextn_ith( + ctx: llama_context_p, + i: ctypes.c_int32, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# // Set whether the context outputs the input embeddings of a specific layer +# LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid, bool value); +@ctypes_function_llama_ext( + [ + "llama_set_embeddings_layer_inp", + "?llama_set_embeddings_layer_inp@@YAXPEAUllama_context@@I_N@Z", + "__Z30llama_set_embeddings_layer_inpP13llama_contextjb", + ], + [llama_context_p_ctypes, ctypes.c_int32, ctypes.c_bool], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_set_embeddings_layer_inp( + ctx: llama_context_p, + lid: ctypes.c_int32, + value: bool, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + """ + Set whether the context outputs the input embeddings of a specific layer + """ + ... + +# // mirrors: +# // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); +# LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid); +@ctypes_function_llama_ext( + [ + "llama_get_embeddings_layer_inp", + "?llama_get_embeddings_layer_inp@@YAPEAMPEAUllama_context@@I@Z", + "__Z30llama_get_embeddings_layer_inpP13llama_contextj", + ], + [llama_context_p_ctypes, ctypes.c_int32], + ctypes.POINTER(ctypes.c_float), + required=False, +) +def llama_get_embeddings_layer_inp( + ctx: llama_context_p, + lid: ctypes.c_int32, +) -> ctypes.POINTER(ctypes.c_float): # type: ignore + ... + +# LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); +@ctypes_function_llama_ext( + [ + "llama_get_ctx_other", + "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", + "__Z19llama_get_ctx_otherP13llama_context", + ], + [llama_context_p_ctypes], + llama_context_p_ctypes, + required=False, +) +def llama_get_ctx_other( + ctx: llama_context_p, +) -> llama_context_p: + ... + +# // model/context data extraction + +# // returns pointer to the target-model layer indices +# LLAMA_API const int32_t * llama_model_target_layer_ids (const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_target_layer_ids", + "?llama_model_target_layer_ids@@YAPEBHPEBUllama_model@@@Z", + "__Z28llama_model_target_layer_idsPK11llama_model", + ], + [llama_model_p_ctypes], + ctypes.POINTER(ctypes.c_int32), + required=False, +) +def llama_model_target_layer_ids( + model: llama_model_p +) -> ctypes.POINTER(ctypes.c_int32): # type: ignore + """ + returns pointer to the target-model layer indices + """ + ... + +# // returns the number of extracted layers from target model +# LLAMA_API uint32_t llama_model_target_layer_ids_n(const struct llama_model * model); +@ctypes_function_llama_ext( + [ + "llama_model_target_layer_ids_n", + "?llama_model_target_layer_ids_n@@YAIPEBUllama_model@@@Z", + "__Z30llama_model_target_layer_ids_nPK11llama_model" + ], + [llama_model_p_ctypes], + ctypes.POINTER(ctypes.c_uint32), + required=False, +) +def llama_model_target_layer_ids_n( + model: llama_model_p +) -> ctypes.POINTER(ctypes.c_uint32): # type: ignore + """ + returns the number of extracted layers from target model + """ + ... From 3df6144d2fd04e3e657b4fe354bb8bdd382dc1e7 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 18 Jul 2026 00:09:43 +0800 Subject: [PATCH 253/304] Update Submodule vendor/llama.cpp e8f19cc..86d86ed Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e8f19cc0ad..86d86ed439 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e8f19cc0ad70a243c8012bf17b4be601abfc8ea2 +Subproject commit 86d86ed4396b4130922f7b9af26e3d9fc11a591b From a64128351a1d04c6dd644e3908070f7ea2002f20 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 18 Jul 2026 01:35:17 +0800 Subject: [PATCH 254/304] =?UTF-8?q?patch(Gemma4ChatHandler):=20Synchronize?= =?UTF-8?q?=20huggingface=20gemma4=20latest=20chat=20template=20-=20fix:?= =?UTF-8?q?=20chat=20template=20=E2=80=94=20null=20handling,=20reasoning?= =?UTF-8?q?=20preservation,=20turn-tag=20balance,=20input=20validation=20-?= =?UTF-8?q?=20https://huggingface.co/google/gemma-4-31B-it/commit/68abe480?= =?UTF-8?q?10cbe15293462fa11e901a60639a44e5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: JamePeng --- llama_cpp/llama_multimodal.py | 142 ++++++++++++++++++++++------------ 1 file changed, 91 insertions(+), 51 deletions(-) diff --git a/llama_cpp/llama_multimodal.py b/llama_cpp/llama_multimodal.py index f3b7382e8a..cc159924fb 100644 --- a/llama_cpp/llama_multimodal.py +++ b/llama_cpp/llama_multimodal.py @@ -2582,7 +2582,9 @@ class Gemma4ChatHandler(MTMDChatHandler): " }\n" "{%- endmacro -%}\n" "{%- macro format_argument(argument, escape_keys=True) -%}\n" - " {%- if argument is string -%}\n" + " {%- if argument is none -%}\n" + " {{- 'null' -}}\n" + " {%- elif argument is string -%}\n" " {{- '<|\"|>' + argument + '<|\"|>' -}}\n" " {%- elif argument is boolean -%}\n" " {{- 'true' if argument else 'false' -}}\n" @@ -2638,18 +2640,21 @@ class Gemma4ChatHandler(MTMDChatHandler): " {{- '' -}}\n" "{%- endmacro -%}\n" "\n" + "{#- ===== SETUP ===== -#}" "{%- set ns = namespace(prev_message_type=None) -%}\n" "{%- set loop_messages = messages -%}\n" + "{%- set enable_thinking = enable_thinking | default(false) -%}\n" + "{%- set preserve_thinking = preserve_thinking | default(false) -%}\n" "{{- bos_token -}}\n" "{#- Handle System/Tool Definitions Block -#}\n" - "{%- if (enable_thinking is defined and enable_thinking) or tools or messages[0]['role'] in ['system', 'developer'] -%}\n" + "{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%}\n" " {{- '<|turn>system\\n' -}}\n" " {#- Inject Thinking token at the very top of the FIRST system turn -#}\n" - " {%- if enable_thinking is defined and enable_thinking -%}\n" + " {%- if enable_thinking -%}\n" " {{- '<|think|>\\n' -}}\n" " {%- set ns.prev_message_type = 'think' -%}\n" " {%- endif -%}\n" - " {%- if messages[0]['role'] in ['system', 'developer'] -%}\n" + " {%- if messages and messages[0]['role'] in ['system', 'developer'] -%}\n" " {%- if messages[0]['content'] is string -%}\n" " {{- messages[0]['content'] | trim -}}\n" " {%- elif messages[0]['content'] is sequence -%}\n" @@ -2683,31 +2688,21 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- if message['role'] != 'tool' -%}\n" " {%- set ns.prev_message_type = None -%}\n" " {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n" - " {#- Detect continuation: suppress duplicate <|turn>model when previous non-tool message was also assistant -#}\n" - " {%- set prev_nt = namespace(role=None, found=false) -%}\n" - " {%- if loop.index0 > 0 -%}\n" - " {%- for j in range(loop.index0 - 1, -1, -1) -%}\n" - " {%- if not prev_nt.found -%}\n" - " {%- if loop_messages[j]['role'] != 'tool' -%}\n" - " {%- set prev_nt.role = loop_messages[j]['role'] -%}\n" - " {%- set prev_nt.found = true -%}\n" - " {%- endif -%}\n" - " {%- endif -%}\n" - " {%- endfor -%}\n" - " {%- endif -%}\n" - " {%- set continue_same_model_turn = (role == 'model' and prev_nt.role == 'assistant') -%}\n" + "{#- Detect continuation using tracked state ā€” O(1) instead of O(n) backward scan -#}\n" + "{%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%}\n" " {%- if not continue_same_model_turn -%}\n" " {{- '<|turn>' + role + '\\n' }}\n" " {%- endif -%}\n" "\n" " {#- Render reasoning/reasoning_content as thinking channel -#}\n" " {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n" - " {%- if thinking_text and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}\n" - " {{- '<|channel>thought\\n' + thinking_text + '\\n' -}}\n" + " {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%}\n" + " {%- if thinking_text and thinking_gate -%}\n" + " {{- '<|channel>thought\n' + thinking_text + '\n' -}}\n" " {%- endif -%}\n" "\n" " {%- if message.get('tool_calls') -%}\n" - " {%- for tool_call in message['tool_calls'] -%}\n" + " {%- for tool_call in message.get('tool_calls') -%}\n" " {%- set function = tool_call['function'] -%}\n" " {{- '<|tool_call>call:' + function['name'] + '{' -}}\n" " {%- if function['arguments'] is mapping -%}\n" @@ -2717,8 +2712,13 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set ns_args.found_first = true -%}\n" " {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n" " {%- endfor -%}\n" - " {%- elif function['arguments'] is string -%}\n" - " {{- function['arguments'] -}}\n" + " {%- elif function['arguments'] is none -%}\n" + " {%- else -%}\n" + " {{- raise_exception(\n" + " \"chat_template: tool_calls[].function.arguments must be a \"\n" + " \"JSON object (mapping), not a string. Deserialize arguments \"\n" + " \"before passing to the template.\"\n" + " ) -}}\n" " {%- endif -%}\n" " {{- '}' -}}\n" " {%- endfor -%}\n" @@ -2728,8 +2728,8 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set ns_tr_out = namespace(flag=false) -%}\n" " {%- if message.get('tool_responses') -%}\n" " {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n" - " {%- for tool_response in message['tool_responses'] -%}\n" - " {{- format_tool_response_block(tool_response['name'] | default('unknown'), tool_response['response']) -}}\n" + " {%- for tool_response in message.get('tool_responses') -%}\n" + " {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}}\n" " {%- set ns_tr_out.flag = true -%}\n" " {%- set ns.prev_message_type = 'tool_response' -%}\n" " {%- endfor -%}\n" @@ -2743,8 +2743,8 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- else -%}\n" " {%- set follow = loop_messages[k] -%}\n" " {#- Resolve tool_call_id to function name -#}\n" - " {%- set ns_tname = namespace(name=follow.get('name') | default('unknown')) -%}\n" - " {%- for tc in message['tool_calls'] -%}\n" + " {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%}\n" + " {%- for tc in message.get('tool_calls') -%}\n" " {%- if tc.get('id') == follow.get('tool_call_id') -%}\n" " {%- set ns_tname.name = tc['function']['name'] -%}\n" " {%- endif -%}\n" @@ -2762,9 +2762,14 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endfor -%}\n" " {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n" " {%- for part in tool_body -%}\n" - " {%- if part.get('type') == 'image_url' -%}\n" - " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" - " {{- '<|image|>' + url_val -}}\n" + " {%- if part.get('type') in ['image', 'image_url'] -%}\n" + " {%- if part.get('type') == 'image_url' -%}\n" + " {%- set url_val = part['image_url'] if part['image_url'] is string else part['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- elif part.get('type') == 'image' -%}\n" + " {%- set url_val = part['image'] if part['image'] is string else part['image']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- endif -%}\n" " {%- elif part.get('type') in ['audio_url', 'input_audio'] -%}\n" " {%- if part.get('type') == 'audio_url' -%}\n" " {%- set audio_val = part['audio_url'] if part['audio_url'] is string else part['audio_url']['url'] -%}\n" @@ -2773,9 +2778,14 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- set audio_val = part['input_audio'] if part['input_audio'] is string else ('data:audio/' + part['input_audio']['format'] + ';base64,' + part['input_audio']['data']) -%}\n" " {{- '<|audio|>' + audio_val -}}\n" " {%- endif -%}\n" - # " {%- elif part.get('type') == 'video_url' -%}\n" - # " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" - # " {{- '<|video|>' + video_val -}}\n" + " {%- elif part.get('type') in ['video', 'video_url'] -%}\n" + " {%- if part.get('type') == 'video_url' -%}\n" + " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- elif part.get('type') == 'video' -%}\n" + " {%- set video_val = part['video'] if part['video'] is string else part['video']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- endif -%}\n" " {%- endif -%}\n" " {%- endfor -%}\n" " {%- else -%}\n" @@ -2788,38 +2798,45 @@ class Gemma4ChatHandler(MTMDChatHandler): " {%- endif -%}\n" "\n" " {%- set captured_content -%}\n" - " {%- if message['content'] is string -%}\n" + " {%- if message.get('content') is string -%}\n" " {%- if role == 'model' -%}\n" " {{- strip_thinking(message['content']) -}}\n" " {%- else -%}\n" " {{- message['content'] | trim -}}\n" " {%- endif -%}\n" - " {%- elif message['content'] is sequence -%}\n" + " {%- elif message.get('content') is sequence -%}\n" " {%- for item in message['content'] -%}\n" - " {%- if item['type'] == 'text' -%}\n" + " {%- if item.get('type') == 'text' -%}\n" " {%- if role == 'model' -%}\n" " {{- strip_thinking(item['text']) -}}\n" " {%- else -%}\n" " {{- item['text'] | trim -}}\n" " {%- endif -%}\n" - " {%- elif item['type'] == 'image_url' -%}\n" - " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" - " {{- '<|image|>' + url_val -}}\n" - " {%- set ns.prev_message_type = 'image' -%}\n" - " {%- elif item['type'] in ['audio_url', 'input_audio'] -%}\n" - " {%- if item['type'] == 'audio_url' -%}\n" + " {%- elif item.get('type') in ['image', 'image_url'] -%}\n" + " {%- if item.get('type')== 'image_url' -%}\n" + " {%- set url_val = item['image_url'] if item['image_url'] is string else item['image_url']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- elif item.get('type') == 'image' -%}\n" + " {%- set url_val = item['image'] if item['image'] is string else item['image']['url'] -%}\n" + " {{- '<|image|>' + url_val -}}\n" + " {%- endif -%}\n" + " {%- elif item.get('type') in ['audio_url', 'input_audio'] -%}\n" + " {%- if item.get('type') == 'audio_url' -%}\n" " {%- set audio_val = item['audio_url'] if item['audio_url'] is string else item['audio_url']['url'] -%}\n" " {{- '<|audio|>' + audio_val -}}\n" - " {%- elif item['type'] == 'input_audio' -%}\n" + " {%- elif item.get('type') == 'input_audio' -%}\n" " {%- set audio_val = item['input_audio'] if item['input_audio'] is string else ('data:audio/' + item['input_audio']['format'] + ';base64,' + item['input_audio']['data']) -%}\n" " {{- '<|audio|>' + audio_val -}}\n" " {%- endif -%}\n" - " {%- set ns.prev_message_type = 'audio' -%}\n" + " {%- elif item.get('type') in ['video', 'video_url'] -%}\n" + " {%- if item.get('type') == 'video_url' -%}\n" + " {%- set video_val = part['video_url'] if part['video_url'] is string else part['video_url']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- elif item.get('type') == 'video' -%}\n" + " {%- set video_val = part['video'] if part['video'] is string else part['video']['url'] -%}\n" + " {{- '<|video|>' + video_val -}}\n" + " {%- endif -%}\n" " {%- endif -%}\n" - # " {%- elif item['type'] == 'video_url' -%}\n" - # " {%- set video_val = item['video_url'] if item['video_url'] is string else item['video_url']['url'] -%}\n" - # " {{- '<|video|>' + video_val -}}\n" - # " {%- set ns.prev_message_type = 'video' -%}\n" " {%- endfor -%}\n" " {%- endif -%}\n" " {%- endset -%}\n" @@ -2827,20 +2844,43 @@ class Gemma4ChatHandler(MTMDChatHandler): " {{- captured_content -}}\n" " {%- set has_content = captured_content | trim | length > 0 -%}\n" "\n" + " {#- Forward-scan: find next non-tool message role for continuation detection -#}\n" + " {%- set next_nt = namespace(role=None, found=false) -%}\n" + " {%- for j in range(loop.index0 + 1, loop_messages | length) -%}\n" + " {%- if not next_nt.found -%}\n" + " {%- if loop_messages[j]['role'] != 'tool' -%}\n" + " {%- set next_nt.role = loop_messages[j]['role'] -%}\n" + " {%- set next_nt.found = true -%}\n" + " {%- endif -%}\n" + " {%- endif -%}\n" + " {%- endfor -%}\n" + + " {%- set continues_into_next = (\n" + " role == 'model'\n" + " and next_nt.role == 'assistant'\n" + " and (not message.get('tool_calls') or ns_tr_out.flag)\n" + " ) -%}\n" + "\n" " {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n" " {{- '<|tool_response>' -}}\n" - " {%- elif not (ns_tr_out.flag and not has_content) -%}\n" + " {%- elif continues_into_next -%}\n" + " {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%}\n" " {{- '\\n' -}}\n" " {%- endif -%}\n" + "\n" + " {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#}\n" + " {%- set ns.prev_non_tool_role = message['role'] -%}\n" " {%- endif -%}\n" "{%- endfor -%}\n" "\n" "{%- if add_generation_prompt -%}\n" " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n" - " {{- '<|turn>model\\n' -}}\n" - " {%- if not enable_thinking | default(false) -%}\n" - " {{- '<|channel>thought\\n' -}}\n" + " {{- '<|turn>model\n' -}}\n" + " {%- if not enable_thinking -%}\n" + " {{- '<|channel>thought\n' -}}\n" " {%- endif -%}\n" + " {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n" + " {{- '<|channel>thought\n' -}}\n" " {%- endif -%}\n" "{%- endif -%}\n" ) From caa72e2ac2c971a4255799fc8f498395cf799f8a Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 18 Jul 2026 04:09:01 +0800 Subject: [PATCH 255/304] Bump version to 0.3.43 - This release focuses on improving compatibility with the rapidly evolving llama.cpp ecosystem, especially around experimental APIs, ABI changes, MTMD processing, and latest model integrations. Signed-off-by: JamePeng --- CHANGELOG.md | 76 ++++++++++++++++++++++++++++++++++++++++++- llama_cpp/__init__.py | 2 +- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9654d354ec..b25d56cfe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,85 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [0.3.43] Better llama.cpp ABI Compatibility, MTMD Performance and Extension API Support + +- patch(Gemma4ChatHandler): Synchronize huggingface gemma4 latest chat template + - fix: chat template ā€” null handling, reasoning preservation, turn-tag balance, input validation + - https://huggingface.co/google/gemma-4-31B-it/commit/68abe48010cbe15293462fa11e901a60639a44e5 + +- feat(llama_ext): support optional llama-ext.h API bindings + - Add Python ctypes bindings for the experimental APIs exposed by + llama-ext.h, including NextN/MTP embeddings, and model metadata extraction. + - Extension symbols are loaded optionally to handle ABI changes, renamed + symbols, and builds that do not export experimental APIs without breaking + the main Python bindings. + +- feat(ctypes): handle missing optional symbols gracefully + - Allow ctypes bindings to mark symbols as optional through the `required` + flag. + - Missing symbols caused by ABI naming differences, API changes, or experimental + extensions will no longer break library loading. Optional APIs emit diagnostic + warnings and provide runtime unavailable stubs instead. + +- fix(ctypes): validate argument types before binding shared library functions + - Add explicit validation for ctypes function argument declarations before + assigning them to the loaded shared library function. + - This provides clearer error messages when invalid Python types are passed + to `argtypes`, instead of exposing the internal ctypes error about missing + `from_param()` methods. + +- fix(ctypes): validate argument types before binding shared library functions + - Add explicit validation for ctypes function argument declarations before + assigning them to the loaded shared library function. + - This provides clearer error messages when invalid Python types are passed + to `argtypes`, instead of exposing the internal ctypes error about missing + `from_param()` methods. + +- feat(ctypes): support ABI-compatible symbol aliases + - Allow ctypes_function_for_shared_library to accept either a single + symbol name or an ordered iterable of ABI-compatible aliases. + - Resolve aliases in order and bind the first exported symbol found while + preserving the selected symbol name for runtime diagnostics. Also improve + error reporting for empty alias lists and missing symbols. + +- refactor(mtmd): cache Generic MTMD chat template resolution for accelerate the processing speed of `__call__`. + - Refactor MTMD chat template handling to resolve and analyze the chat template only + once per handler instance instead of on every request. + - Add template initialization state, cache parsed media placeholder tags, and support + explicit chat template overrides through a dedicated field. Improve lifecycle cleanup + by resetting cached template state and MTMD resources during handler close. + - This keeps `GenericMTMDChatHandler` runtime processing focused on message rendering and media tokenization + while avoiding repeated chat template resolution overhead. + +- fix(mtmd): preserve subclass chat format during MTMD initialization + - Ensure MTMDChatHandler initialization remains compatible with specialized chat + handlers that define their own chat_format before calling super().__init__(). + - Initialize chat_format only when it is not already provided by the subclass, + then apply chat_format_override or fallback to the built-in MTMD template. + This prevents AttributeError during inherited handler initialization while + keeping template override behavior unchanged. + +- refactor(embedding): rename `llama_cpp` import alias to `llama_cpp_lib` + - Rename the `llama_cpp.llama_cpp` import alias to `llama_cpp_lib` to avoid potential namespace conflicts with the local `.llama_cpp` imports. Update all affected call sites in `llama_embedding.py`. + +- patch(Llama): Increase chunk preview limit to 128 in Llama.eval exception + - Raises the maximum tokens captured for the error message preview from 16 to 128, improving visibility into the offending chunk during fatal backend crashes. + +- ci(metal): get package version from importlib metadata + * Avoid importing llama_cpp when detecting the package version. + * This prevents initialization side effects and keeps CI version extraction reliable. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/86d86ed4396b4130922f7b9af26e3d9fc11a591b](https://github.com/ggml-org/llama.cpp/commit/86d86ed4396b4130922f7b9af26e3d9fc11a591b) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260716 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/e522cecb93907c67ffe2e339b7009c93d3fb0f59...a64128351a1d04c6dd644e3908070f7ea2002f20 + ## [0.3.42] More Reliable Dynamic Backend Loading, Safer MTMD Processing, and Advanced Batch Support - fix(loader): improve Windows DLL search path handling and diagnostics diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index c6676c294b..1695e37277 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.42" +__version__ = "0.3.43" From 79b8b5d0824f44a63e5c8c8d8943b2768bf65fce Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 21 Jul 2026 03:46:25 +0800 Subject: [PATCH 256/304] Update Submodule vendor/llama.cpp 86d86ed..91d2fc3 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 86d86ed439..91d2fc3875 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 86d86ed4396b4130922f7b9af26e3d9fc11a591b +Subproject commit 91d2fc387529940230555abd297a8b5e99737d3f From 0eb26153ff757498c22c4574c3edabe7e19b2ad5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 21 Jul 2026 04:06:22 +0800 Subject: [PATCH 257/304] fix(ggml): preload bundled OpenMP runtime before loading ggml-base - Preload the packaged libomp140.x86_64.dll on Windows before initializing ggml-base to ensure CPU backend DLLs can resolve their OpenMP runtime dependency. - This only applies to Windows builds with llama-cpp-python >= 0.3.39 and uses the bundled runtime from the package lib directory, avoiding the need for users to configure system PATH or install additional OpenMP runtimes. Signed-off-by: JamePeng --- llama_cpp/_ctypes_extensions.py | 10 +++++++++ llama_cpp/_ggml.py | 37 +++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index e88595f361..a9a2c02e50 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -5,6 +5,7 @@ import ctypes import functools import pathlib +import importlib.metadata from ctypes.util import find_library from typing import ( Any, @@ -19,6 +20,15 @@ ) from typing_extensions import TypeAlias +def _version_at_least(version: str) -> bool: + """Check whether installed llama-cpp-python version meets requirement.""" + try: + current = importlib.metadata.version("llama-cpp-python") + from packaging.version import Version + return Version(current) >= Version(version) + except Exception: + return False + def _format_library_dir_contents(base_paths: list[pathlib.Path]) -> str: """Format directory contents for diagnostics after library loading fails.""" sections = [] diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 42ff664944..9a7dac517b 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -6,10 +6,9 @@ import enum import os import pathlib - from llama_cpp._ctypes_extensions import ( + _version_at_least, load_shared_library, - byref, ctypes_function_for_shared_library, ) @@ -21,12 +20,46 @@ TYPE_CHECKING, ) +def _preload_openmp_runtime(): + """Preload bundled OpenMP runtime before loading ggml-base. + + This is required on Windows when CPU backends depend on the packaged + OpenMP runtime DLL. + """ + + # Only Windows DLL loading requires this workaround. + if os.name != "nt": + return + + # Keep compatibility with older package versions. + if not _version_at_least("0.3.39"): + return + + libomp_path = (pathlib.Path(__file__).parent / "lib" / "libomp140.x86_64.dll") + + if not libomp_path.exists(): + print(f"[llama-cpp-python] WARNING: bundled OpenMP runtime not found: {libomp_path}") + return + + try: + ctypes.CDLL(str(libomp_path), winmode=ctypes.RTLD_GLOBAL) + print(f"[llama-cpp-python] loaded bundled OpenMP runtime: {libomp_path}") + except Exception as e: + print( + "[llama-cpp-python] WARNING: failed to load bundled OpenMP runtime:\n" + f" path: {libomp_path}\n" + f" error: {e}" + ) + libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) libggml_base_paths = [ libggml_base_path / "lib", libggml_base_path / "bin", ] +# Load bundled OpenMP runtime before ggml-base on Windows. +_preload_openmp_runtime() + libggml_base = load_shared_library("ggml-base", libggml_base_paths) ggml_base_function = ctypes_function_for_shared_library(libggml_base) From db460e0c02aeb3400ee7776b19d60dca4aadb275 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 22 Jul 2026 02:02:42 +0800 Subject: [PATCH 258/304] Update Submodule vendor/llama.cpp 91d2fc3..846e991 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 91d2fc3875..846e991ec3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 91d2fc387529940230555abd297a8b5e99737d3f +Subproject commit 846e991ec3c7ccec49112ff2c5b00b710e5f551d From ebf6099b81cf67cfb5eec569466367c9fa04e9d4 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 22 Jul 2026 02:16:45 +0800 Subject: [PATCH 259/304] Bump version to 0.3.44 - This release is a small but important maintenance update focused on improving Windows dynamic library loading reliability. Signed-off-by: JamePeng --- CHANGELOG.md | 12 ++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b25d56cfe7..69fc02b25c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.44] Improved Windows DLL(OpenMP) Loading Reliability for GGML Backends + +- fix(ggml): preload bundled OpenMP runtime before loading ggml-base + - Preload the packaged `libomp140.x86_64.dll` on Windows before initializing + ggml-base to ensure CPU backend DLLs can resolve their OpenMP runtime + dependency. + - This only applies to Windows builds with llama-cpp-python >= 0.3.39 and + uses the bundled runtime from the package lib directory, avoiding the need + for users to configure system PATH or install additional OpenMP runtimes. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/846e991ec3c7ccec49112ff2c5b00b710e5f551d](https://github.com/ggml-org/llama.cpp/commit/846e991ec3c7ccec49112ff2c5b00b710e5f551d) + ## [0.3.43] Better llama.cpp ABI Compatibility, MTMD Performance and Extension API Support - patch(Gemma4ChatHandler): Synchronize huggingface gemma4 latest chat template diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 1695e37277..10e452d5f6 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.43" +__version__ = "0.3.44" From ffbd951819fde28a43c7354391b979035dd3b3ed Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 23 Jul 2026 07:51:08 +0800 Subject: [PATCH 260/304] Update Submodule vendor/llama.cpp 846e991..4310aa4 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 846e991ec3..4310aa4f87 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 846e991ec3c7ccec49112ff2c5b00b710e5f551d +Subproject commit 4310aa4f871c104698f6a6614a362bdec87c247a From 03124efa535f9de0974c36d54272be8303e649b5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 25 Jul 2026 03:16:58 +0800 Subject: [PATCH 261/304] Update Submodule vendor/llama.cpp 4310aa4..88bfee1 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4310aa4f87..88bfee1429 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4310aa4f871c104698f6a6614a362bdec87c247a +Subproject commit 88bfee1429a2dfacec65d1b0c0852eb327991865 From c4cdd46ab373393fab41d9a0e4f6faac55c5619f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 08:13:30 +0800 Subject: [PATCH 262/304] Update Submodule vendor/llama.cpp 88bfee1..8bb9093 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 88bfee1429..8bb909374d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 88bfee1429a2dfacec65d1b0c0852eb327991865 +Subproject commit 8bb909374d04d40621340aee5ba2245860027fdc From 93ebd83b540e1e0ccb0489b7a42b6956fde3143b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 08:57:32 +0800 Subject: [PATCH 263/304] feat(embedding): modernize the built-in Llama embedding API - Replace the legacy embedding path with sequence-aware streaming batch processing based on the current LlamaBatch interface. - Support string, batched string, and pre-tokenized inputs, token-level and rank pooling outputs, separator-based splitting, token accounting, and llama.cpp-compatible normalization modes. - Restore embed() and create_embedding() as maintained Llama APIs while preserving the existing boolean normalization behavior. Signed-off-by: JamePeng --- llama_cpp/llama.py | 262 ++++++++++++++++++++++++++++++--------------- 1 file changed, 175 insertions(+), 87 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index f1f71be8e3..f733d7afb9 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1950,21 +1950,25 @@ def adapter(token_data_array: llama_cpp_lib.llama_token_data_array): ) def create_embedding( - self, input: Union[str, List[str]], model: Optional[str] = None + self, + input: Union[str, List[str]], + model: Optional[str] = None, + normalize: Union[bool, int] = False, + truncate: bool = True, ) -> CreateEmbeddingResponse: - """Embed a string. + """Create an OpenAI-compatible embedding response. Args: - input: The utf-8 encoded string to embed. + input: A string or list of strings to embed. + model: Model name reported in the response. + normalize: ``False`` disables normalization, ``True`` uses L2 + normalization, and integer values select a llama.cpp + normalization mode. + truncate: Truncate inputs to the available context/batch capacity. Returns: - An embedding object. + An OpenAI-compatible embedding response. """ - warnings.warn( - "The `create_embedding` method in `Llama` class is deprecated. " - "Please migrate to `LlamaEmbedding.create_embedding` for better efficiency.", - DeprecationWarning, - ) model_name: str = model if model is not None else self.model_path input = input if isinstance(input, list) else [input] @@ -1972,7 +1976,12 @@ def create_embedding( # get numeric embeddings embeds: Union[List[List[float]], List[List[List[float]]]] total_tokens: int - embeds, total_tokens = self.embed(input, return_count=True) # type: ignore + embeds, total_tokens = self.embed( # type: ignore + input, + normalize=normalize, + truncate=truncate, + return_count=True, + ) # convert to CreateEmbeddingResponse data: List[Embedding] = [ @@ -1996,130 +2005,209 @@ def create_embedding( def embed( self, - input: Union[str, List[str]], - normalize: bool = False, + input: Union[str, List[str], List[List[int]]], + normalize: Union[bool, int] = False, truncate: bool = True, + separator: Optional[str] = None, return_count: bool = False, ): - """Embed a string. + """Embed strings or pre-tokenized inputs. Args: - input: The utf-8 encoded string to embed. + input: A string, a list of strings, or a list of token-id lists. + normalize: ``False``/``-1`` disables normalization, ``True`` uses + L2 normalization. Integer modes follow llama.cpp's embedding + example: 0=max-absolute (scaled to 32760), 1=L1, 2=L2, and + values greater than 2 use the corresponding p-norm. + truncate: Truncate inputs that exceed the context/batch capacity. + separator: Split a single string into multiple inputs. + return_count: Return ``(embeddings, token_count)``. Returns: - A list of embeddings + Sequence embeddings, token-level embeddings for pooling type NONE, + or scalar/vector scores for pooling type RANK. """ - warnings.warn( - "The `embed` method in `Llama` class is deprecated and will be removed in future versions. " - "Please use the `LlamaEmbedding` class from `llama_embedding` module for optimized performance and reranking support.", - DeprecationWarning, - ) + if self.context_params.embeddings is False: + raise RuntimeError( + "Llama model must be created with embeddings=True to call this method" + ) - n_embd = self.n_embd() + ctx = self._ctx.ctx n_batch = self.n_batch + n_ctx = self._n_ctx + n_seq_max = self.context_params.n_seq_max - # get pooling information pooling_type = self.pooling_type() - logits_all = pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_NONE + is_rank = pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_RANK + is_none = pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_NONE - if self.context_params.embeddings is False: - raise RuntimeError( - "Llama model must be created with embeddings=True to call this method" - ) + out_dim = ( + llama_cpp_lib.llama_model_n_cls_out(self._model.model) + if is_rank + else self.n_embd() + ) + + # Preserve the historical bool API while accepting llama.cpp's integer + # normalization modes used by LlamaEmbedding. + if isinstance(normalize, bool): + normalize_mode = 2 if normalize else -1 + elif isinstance(normalize, int): + normalize_mode = normalize + else: + raise TypeError("normalize must be a bool or int") + + def normalize_vector(vector: Sequence[float]) -> List[float]: + values = list(vector) + if normalize_mode == -1 or is_rank: + return values + + array = np.asarray(values, dtype=np.float32) + if normalize_mode == 0: + norm = float(np.max(np.abs(array))) if array.size else 0.0 + scale = 32760.0 + elif normalize_mode == 1: + norm = float(np.sum(np.abs(array))) + scale = 1.0 + elif normalize_mode == 2: + norm = float(np.linalg.norm(array)) + scale = 1.0 + elif normalize_mode > 2: + norm = float( + np.sum(np.abs(array) ** normalize_mode) + ** (1.0 / normalize_mode) + ) + scale = 1.0 + else: + return values + + if norm == 0.0: + return values + return ((array / norm) * scale).tolist() if self.verbose: - llama_cpp_lib.llama_perf_context_reset(self._ctx.ctx) + llama_cpp_lib.llama_perf_context_reset(ctx) if isinstance(input, str): - inputs = [input] + inputs: List[Union[str, List[int]]] = ( + input.split(separator) if separator is not None else [input] + ) + is_single = separator is None else: inputs = input + is_single = False - # reset batch self._batch.reset() + llama_cpp_lib.llama_memory_clear( + llama_cpp_lib.llama_get_memory(ctx), True + ) - # decode and fetch embeddings - data: Union[List[List[float]], List[List[List[float]]]] = [] + data: List[Any] = [] + seq_sizes: List[int] = [] + total_tokens = 0 + + def decode_batch() -> None: + nonlocal seq_sizes + if not seq_sizes: + return - def decode_batch(seq_sizes: List[int]): - llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(self._ctx.ctx), True) self._ctx.decode(self._batch) + + if is_none: + token_index = 0 + for size in seq_sizes: + token_embeddings: List[List[float]] = [] + for _ in range(size): + ptr = llama_cpp_lib.llama_get_embeddings_ith( + ctx, token_index + ) + token_embeddings.append( + [0.0] * out_dim + if ptr is None + else normalize_vector(ptr[:out_dim]) + ) + token_index += 1 + data.append(token_embeddings) + else: + for seq_id in range(len(seq_sizes)): + ptr = llama_cpp_lib.llama_get_embeddings_seq(ctx, seq_id) + if ptr is None: + embedding = [0.0] * out_dim + else: + embedding = list(ptr[:out_dim]) + + if is_rank: + data.append( + embedding[0] if len(embedding) == 1 else embedding + ) + else: + data.append(normalize_vector(embedding)) + self._batch.reset() + llama_cpp_lib.llama_memory_clear( + llama_cpp_lib.llama_get_memory(ctx), True + ) + seq_sizes = [] - # store embeddings - if pooling_type == llama_cpp_lib.LLAMA_POOLING_TYPE_NONE: - pos: int = 0 - for i, size in enumerate(seq_sizes): - ptr = llama_cpp_lib.llama_get_embeddings(self._ctx.ctx) - embedding: List[List[float]] = [ - ptr[pos + j * n_embd : pos + (j + 1) * n_embd] - for j in range(size) - ] - if normalize: - embedding = [ - internals.normalize_embedding(e) for e in embedding - ] - data.append(embedding) - pos += size + for item in inputs: + if isinstance(item, str): + tokens = self.tokenize(item.encode("utf-8")) + elif isinstance(item, list) and ( + not item or isinstance(item[0], int) + ): + tokens = item else: - for i in range(len(seq_sizes)): - ptr = llama_cpp_lib.llama_get_embeddings_seq(self._ctx.ctx, i) - embedding: List[float] = ptr[:n_embd] - if normalize: - embedding = internals.normalize_embedding(embedding) - data.append(embedding) - - # init state - total_tokens = 0 - s_batch = [] - t_batch = 0 - p_batch = 0 + raise ValueError("Input item must be str or List[int]") - # accumulate batches and encode - for text in inputs: - tokens = self.tokenize(text.encode("utf-8")) - if truncate: - tokens = tokens[:n_batch] + max_tokens = min(n_ctx, n_batch) + if truncate and len(tokens) > max_tokens: + tokens = tokens[:max_tokens] n_tokens = len(tokens) total_tokens += n_tokens - # check for overrun if n_tokens > n_batch: raise ValueError( f"Requested tokens ({n_tokens}) exceed batch size of {n_batch}" ) - # time to eval batch - if t_batch + n_tokens > n_batch: - decode_batch(s_batch) - s_batch = [] - t_batch = 0 - p_batch = 0 + if n_tokens == 0: + # Keep result ordering stable when an empty pre-tokenized input + # follows sequences that are still waiting to be decoded. + decode_batch() + data.append(0.0 if is_rank else []) + continue - # add to batch - self._batch.add_sequence(tokens, p_batch, logits_all) + if ( + self._batch.n_tokens() + n_tokens > n_batch + or len(seq_sizes) >= n_seq_max + ): + decode_batch() - # update batch stats - s_batch.append(n_tokens) - t_batch += n_tokens - p_batch += 1 + seq_id = len(seq_sizes) + logits_array = ( + [True] * n_tokens + if is_none + else [False] * (n_tokens - 1) + [True] + ) + self._batch.add_sequence( + token_array=tokens, + pos_array=list(range(n_tokens)), + seq_ids=[seq_id], + logits_array=logits_array, + ) + seq_sizes.append(n_tokens) - # hanlde last batch - decode_batch(s_batch) + decode_batch() if self.verbose: - llama_cpp_lib.llama_perf_context_print(self._ctx.ctx) + llama_cpp_lib.llama_perf_context_print(ctx) - output = data[0] if isinstance(input, str) else data - - llama_cpp_lib.llama_memory_clear(llama_cpp_lib.llama_get_memory(self._ctx.ctx), True) + output = data[0] if is_single else data self.reset() if return_count: return output, total_tokens - else: - return output + return output def _create_completion( self, From 0d3ae934e759afc6612554980cdcacd84d2ee21b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 08:58:30 +0800 Subject: [PATCH 264/304] refactor(batch): improve sequence capacity validation guidance - Make LlamaBatch sequence validation errors explain the configured n_seq_max value, valid sequence ID range, and minimum capacity required for parallel batching. - Handle negative sequence IDs separately and provide actionable setup guidance for Llama, LlamaEmbedding, and direct LlamaBatch users. - Remove the unused normalize_embedding helper now that normalization is handled by the embedding pipeline. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 4eb2df7db5..a41764d7d9 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -1165,7 +1165,9 @@ def _validate_seq_ids(self, seq_ids: Sequence[int], where: str) -> int: if n_seq_id > self.n_seq_max: raise ValueError( f"LlamaBatch.{where}: token belongs to {n_seq_id} sequences, " - f"but n_seq_max was initialized to {self.n_seq_max}." + f"but this batch was initialized with n_seq_max={self.n_seq_max}. " + f"Increase n_seq_max to at least {n_seq_id} when constructing " + "Llama, LlamaEmbedding, or LlamaBatch." ) for seq_id in seq_ids: @@ -1175,10 +1177,22 @@ def _validate_seq_ids(self, seq_ids: Sequence[int], where: str) -> int: f"{type(seq_id).__name__}." ) - if seq_id < 0 or seq_id >= self.n_seq_max: + if seq_id < 0: raise ValueError( f"LlamaBatch.{where}: invalid seq_id {seq_id}; " - f"expected 0 <= seq_id < {self.n_seq_max}." + "sequence IDs must be non-negative integers." + ) + + if seq_id >= self.n_seq_max: + required_n_seq_max = seq_id + 1 + raise ValueError( + f"LlamaBatch.{where}: seq_id={seq_id} exceeds the configured " + f"sequence capacity (n_seq_max={self.n_seq_max}; valid IDs " + f"are 0 through {self.n_seq_max - 1}). For parallel batching, " + f"initialize Llama or LlamaEmbedding with " + f"n_seq_max>={required_n_seq_max}, or create LlamaBatch " + "with that value. Use seq_id=0 when processing only one " + "sequence." ) return n_seq_id @@ -1487,15 +1501,6 @@ def add_token_embedding( self.batch.logits[idx] = logits self.batch.n_tokens += 1 - -# Embedding functions -def normalize_embedding(embedding): - norm = float(np.linalg.norm(embedding)) - if norm == 0.0: - return embedding - return [v / norm for v in embedding] - - class LlamaTokenDataArray: """ Performance-optimized wrapper for llama_token_data_array. From 29ad3505d2ff81838d21fa94eef6f8e39f1950cd Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 09:01:53 +0800 Subject: [PATCH 265/304] fix(embedding): respect n_seq_max when streaming embedding batches - Use the configured sequence capacity instead of n_ubatch when deciding when to decode the current LlamaEmbedding batch. - This prevents invalid sequence IDs for multi-document inputs and allows the default n_seq_max=1 configuration to process documents sequentially without failing. Signed-off-by: JamePeng --- llama_cpp/llama_embedding.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_embedding.py b/llama_cpp/llama_embedding.py index d8f6e6bf21..baa4b9f066 100644 --- a/llama_cpp/llama_embedding.py +++ b/llama_cpp/llama_embedding.py @@ -128,7 +128,7 @@ def embed( ctx = self._ctx.ctx n_batch = self.n_batch n_ctx = self._n_ctx - n_ubatch = self.context_params.n_ubatch + n_seq_max = self.context_params.n_seq_max # Determine if it is in Rerank mode try: @@ -137,8 +137,6 @@ def embed( pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED is_rank = (pooling_type == LLAMA_POOLING_TYPE_RANK) is_none = (pooling_type == LLAMA_POOLING_TYPE_NONE) # Token-level embedding - logits_all = True if is_none else False - # Determine the output dimension if is_rank: out_dim = llama_cpp_lib.llama_model_n_cls_out(self._model.model) @@ -247,7 +245,10 @@ def _decode_batch(): continue # Check Batch Capacity - if (self._batch.n_tokens() + n_tokens > n_batch) or (idx_in_batch >= n_ubatch): + if ( + self._batch.n_tokens() + n_tokens > n_batch + or idx_in_batch >= n_seq_max + ): _decode_batch() idx_in_batch = 0 From 6896e3dadef9041cf3fb9f0ad43c92155c02fcea Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 09:04:05 +0800 Subject: [PATCH 266/304] test(embedding): cover built-in and streaming embedding workflows - Add coverage for actionable LlamaBatch sequence-capacity errors and the maintained embedding APIs on the standard Llama class. - Verify pre-tokenized batches, normalization, separator-based inputs, token accounting, OpenAI-compatible responses, and LlamaEmbedding streaming behavior with n_seq_max=1. - Explicitly close embedding models after integration tests to release native context and model resources. Signed-off-by: JamePeng --- tests/test_llama.py | 107 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 11 deletions(-) diff --git a/tests/test_llama.py b/tests/test_llama.py index bf075845e0..df379df7fe 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -64,6 +64,32 @@ def test_llama_cpp_tokenization(): assert text == llama.detokenize(tokens) +def test_llama_batch_seq_id_error_guidance(): + """Sequence-capacity errors should explain how to fix parallel batching.""" + batch = internals.LlamaBatch( + n_tokens=2, + embd=0, + n_seq_max=1, + verbose=False, + ) + try: + with pytest.raises(ValueError) as exc_info: + batch.add_sequence( + token_array=[1], + pos_array=[0], + seq_ids=[1], + logits_array=[True], + ) + + message = str(exc_info.value) + assert "n_seq_max=1" in message + assert "valid IDs are 0 through 0" in message + assert "n_seq_max>=2" in message + assert "LlamaEmbedding" in message + finally: + batch.close() + + @pytest.fixture def llama_cpp_model_path(): """Fixture to download a real GGUF model for integration tests.""" @@ -365,16 +391,75 @@ def no_e_processor(input_ids, scores): def test_real_llama_embeddings(llama_cpp_model_path): """ - Test Embedding Generation. - Verifies that the model can produce vector embeddings. + Test embedding generation through the specialized LlamaEmbedding class. """ model = LlamaEmbedding( - model_path=llama_cpp_model_path, - n_ctx=32, - n_batch=32, - n_ubatch=32, - pooling_type=LLAMA_POOLING_TYPE_NONE) - # Smoke test for now - embeddings = model.embed("Hello, world!") - assert isinstance(embeddings, list) - assert len(embeddings) > 0 + model_path=llama_cpp_model_path, + n_ctx=32, + n_batch=32, + n_ubatch=32, + pooling_type=LLAMA_POOLING_TYPE_NONE, + ) + try: + # The inherited n_seq_max=1 processes this list as three streaming + # decode batches instead of assigning an invalid seq_id. + embeddings = model.embed(["Hello", "world", "embedding"]) + assert isinstance(embeddings, list) + assert len(embeddings) == 3 + assert all(len(embedding) > 0 for embedding in embeddings) + finally: + model.close() + + +def test_real_llama_base_embedding_api(llama_cpp_model_path): + """ + Test the maintained embedding API on the standard Llama class. + + Covers pre-tokenized batching, normalization, separator-based string + batching, token counts, and the OpenAI-compatible response wrapper. + """ + model = llama_cpp.Llama( + model_path=llama_cpp_model_path, + embeddings=True, + n_ctx=32, + n_batch=32, + n_ubatch=32, + n_seq_max=2, + kv_unified=True, + pooling_type=LLAMA_POOLING_TYPE_NONE, + verbose=False, + ) + + try: + token_inputs = [ + model.tokenize(b"Hello"), + model.tokenize(b"world"), + ] + embeddings, token_count = model.embed( + token_inputs, + normalize=True, + return_count=True, + ) + + assert len(embeddings) == len(token_inputs) + assert token_count == sum(map(len, token_inputs)) + assert len(embeddings[0]) == len(token_inputs[0]) + assert np.linalg.norm(embeddings[0][0]) == pytest.approx(1.0) + + split_embeddings = model.embed( + "Hello\nworld", + separator="\n", + normalize=False, + ) + assert len(split_embeddings) == 2 + + response = model.create_embedding( + ["Hello", "world"], + normalize=2, + ) + assert response["object"] == "list" + assert len(response["data"]) == 2 + assert response["usage"]["prompt_tokens"] > 0 + assert response["usage"]["total_tokens"] == response["usage"]["prompt_tokens"] + finally: + model.close() From 77b7fa1d3e7022a4ebd971171c9c4b4106db402f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 09:04:40 +0800 Subject: [PATCH 267/304] docs(example): refresh the built-in embedding usage example - Fix the Llama constructor option from embedding=True to embeddings=True and demonstrate L2-normalized output through create_embedding(). Signed-off-by: JamePeng --- examples/high_level_api/high_level_api_embedding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/high_level_api/high_level_api_embedding.py b/examples/high_level_api/high_level_api_embedding.py index feb0ed68d9..bf96213213 100644 --- a/examples/high_level_api/high_level_api_embedding.py +++ b/examples/high_level_api/high_level_api_embedding.py @@ -6,6 +6,6 @@ parser.add_argument("-m", "--model", type=str, default="../models/7B/ggml-model.bin") args = parser.parse_args() -llm = Llama(model_path=args.model, embedding=True) +llm = Llama(model_path=args.model, embeddings=True) -print(llm.create_embedding("Hello world!")) +print(llm.create_embedding("Hello world!", normalize=True)) From 85e4cea81442c4e89255916b51d04988eb64eec2 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 09:06:54 +0800 Subject: [PATCH 268/304] docs(embedding): document maintained APIs and sequence batch capacity - Replace the deprecated Llama embedding guidance with current embed() and create_embedding() usage. - Document the roles of n_batch, n_ubatch, and n_seq_max, including parallel batching examples, resource considerations, common sequence ID errors, and the required configuration changes. - Clarify that LlamaEmbedding remains a convenience interface for embedding-oriented defaults and reranking workflows. Signed-off-by: JamePeng --- README.md | 39 ++++++++++++++++---- docs/wiki/core/Llama.md | 38 ++++++++++++++++---- docs/wiki/modules/LlamaEmbedding.md | 55 ++++++++++++++++++++++++++--- 3 files changed, 114 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ab8c39d779..1345c06904 100644 --- a/README.md +++ b/README.md @@ -1648,7 +1648,12 @@ To generate embeddings, use the `LlamaEmbedding` class. It automatically configu from llama_cpp.llama_embedding import LlamaEmbedding, LLAMA_POOLING_TYPE_NONE # Initialize the model (automatically sets embeddings=True) -llm = LlamaEmbedding(model_path="path/to/bge-m3.gguf", n_gpu_layers=-1, pooling_type=LLAMA_POOLING_TYPE_NONE) +llm = LlamaEmbedding( + model_path="path/to/bge-m3.gguf", + n_gpu_layers=-1, + pooling_type=LLAMA_POOLING_TYPE_NONE, + n_seq_max=128, # Maximum independent sequences in one decode batch +) # 1. Simple usage (OpenAI-compatible format) response = llm.create_embedding("Hello, world!") @@ -1662,6 +1667,14 @@ embeddings = llm.embed(documents) # Returns a list of lists (vectors) print(f"Generated {len(embeddings)} vectors.") ``` +> **Parallel batch capacity:** `n_seq_max` controls how many independent +> sequence IDs may coexist in one decode batch; it is not the total number of +> documents accepted by `embed()`. For batch embedding, set it high enough for +> the number of short documents that can fit within `n_batch`. If an error says +> `seq_id=1` exceeds `n_seq_max=1`, initialize the model with at least +> `n_seq_max=2`. For example, use `n_seq_max=8` for up to eight parallel +> sequences. Larger values can use more context resources. + **Advanced Output Formats:** You can request raw arrays or cosine similarity matrices directly: @@ -1755,14 +1768,28 @@ vec_int16 = llm.embed("text", normalize=NORM_MODE_MAX_INT16) embeddings_raw = llm.embed(["search query", "document text"], normalize=NORM_MODE_NONE) ``` -### Legacy Usage (Deprecated) +### Using the standard `Llama` class -The standard `Llama` class still supports basic embedding generation, but it lacks the memory optimizations and reranking capabilities of `LlamaEmbedding`. +The standard `Llama` class also supports the maintained streaming embedding +implementation. Initialize it with `embeddings=True`, then call `embed()` for +raw results or `create_embedding()` for an OpenAI-compatible response. +`LlamaEmbedding` remains a convenient specialized interface because it enables +embedding-oriented defaults and provides the `rank()` helper. ```python -# Old method - Not recommended for large batches or reranking -llm = llama_cpp.Llama(model_path="...", embeddings=True) -emb = llm.create_embedding("text") +llm = llama_cpp.Llama( + model_path="path/to/model.gguf", + embeddings=True, + n_batch=512, + n_seq_max=8, + kv_unified=True, +) + +# OpenAI-compatible response; normalize=True selects L2 normalization. +response = llm.create_embedding(["query", "document"], normalize=True) + +# Raw vectors. Integer normalization modes are also supported. +vectors = llm.embed(["query", "document"], normalize=2) ``` --- diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 1f7cce206b..d768e43ec3 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -4,7 +4,7 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-05-16 +last_updated: 2026-07-26 version_target: "latest" --- ``` @@ -430,15 +430,39 @@ The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dyn --- -## Deprecated / Changed APIs +## Embeddings -> āš ļø **Warning:** The internal embedding methods on the `Llama` class are deprecated and will be removed. +The `Llama` embedding methods are maintained and use streaming batches. Create +the model with `embeddings=True` before calling them. -* `embed()` āž” **Deprecated.** -* `create_embedding()` āž” **Deprecated.** +```python +llm = Llama( + model_path="path/to/model.gguf", + embeddings=True, + n_seq_max=8, + kv_unified=True, +) + +# Raw sequence or token-level embeddings. +vectors = llm.embed(["query", "document"], normalize=2) + +# OpenAI-compatible response. +response = llm.create_embedding(["query", "document"], normalize=True) +``` + +`embed()` accepts strings, lists of strings, or pre-tokenized inputs. It supports +token-level output (`LLAMA_POOLING_TYPE_NONE`), sequence pooling, rank-model +outputs, streaming batches, and llama.cpp integer normalization modes. + +For parallel batches, `n_seq_max` must cover every sequence ID active in a +single decode batch. The default `n_seq_max=1` supports only `seq_id=0`. +For example, `n_seq_max=8` permits IDs `0` through `7`. If this capacity is +exceeded, the exception reports the valid range and the minimum value required. +`n_batch` limits tokens, while `n_seq_max` limits independent sequences. -**Migration Note:** Do not use `Llama(..., embeddings=True)` combined with `model.create_embedding(...)`. Instead, use the dedicated `LlamaEmbedding` class, which offers optimized batching and reranking support. -*See: [[LlamaEmbedding]]* +`LlamaEmbedding` remains available as the specialized convenience class. It +automatically enables embedding-oriented context options and adds the `rank()` +helper for formatting query/document pairs. --- diff --git a/docs/wiki/modules/LlamaEmbedding.md b/docs/wiki/modules/LlamaEmbedding.md index 3aa2427227..5aa3bd8e0e 100644 --- a/docs/wiki/modules/LlamaEmbedding.md +++ b/docs/wiki/modules/LlamaEmbedding.md @@ -3,7 +3,7 @@ title: Llama Embedding module_name: llama_cpp.llama_embedding source_file: llama_cpp/llama_embedding.py class_name: LlamaEmbedding -last_updated: 2026-05-31 +last_updated: 2026-07-26 version_target: "latest" --- @@ -38,6 +38,7 @@ version_target: "latest" | `n_ctx` | int | 0 | Text context window size (0 = model default). | | `n_batch` | int | 512 | Maximum prompt processing batch size. | | `n_ubatch` | int | 512 | Physical batch size. | +| `n_seq_max` | int | 1 (inherited) | Maximum number of independent sequence IDs available in a decode batch. Increase this for parallel embedding batches. | | `pooling_type` | int | `LLAMA_POOLING_TYPE_UNSPECIFIED` (-1) | Pooling strategy used by the model: `LLAMA_POOLING_TYPE_RANK` (4) for rerankers, `LLAMA_POOLING_TYPE_UNSPECIFIED` (-1) for embeddings. | | `n_gpu_layers` | int | 0 | Number of layers offloaded to GPU (0 = CPU only, -1 = all layers). | | `verbose` | bool | True | Whether to print debug information. | @@ -46,9 +47,44 @@ version_target: "latest" ### Initialization Logic 1. Forces `embeddings=True` to enable embedding support. -2. Sets `kv_unified=True` to enable unified KV Cache, allowing arbitrary sequence IDs in a batch without "invalid seq_id" errors. +2. Sets `kv_unified=True` to enable unified KV Cache. Sequence IDs must still + fit within the configured `n_seq_max`. 3. Passes `pooling_type` to the parent class constructor. +### Parallel Batch Capacity + +`n_batch`, `n_ubatch`, and `n_seq_max` control different limits: + +- `n_batch`: maximum number of input tokens in a logical decode batch. +- `n_ubatch`: physical token batch size used by llama.cpp. +- `n_seq_max`: number of independent sequence IDs that may coexist in a decode + batch. + +For multiple documents, set `n_seq_max` to the desired parallel sequence +capacity: + +```python +model = LlamaEmbedding( + model_path="path/to/model.gguf", + n_batch=512, + n_ubatch=512, + n_seq_max=8, +) +``` + +If the configuration is too small, the error includes the current capacity, +valid ID range, and required minimum: + +```text +LlamaBatch.add_sequence: seq_id=1 exceeds the configured sequence capacity +(n_seq_max=1; valid IDs are 0 through 0). For parallel batching, initialize +Llama or LlamaEmbedding with n_seq_max>=2 ... +``` + +`n_seq_max` is not the total number of documents passed to `embed()`; it is the +number that can be active in one decode batch. Increase it carefully because +larger values may require more context resources. + ## Core Methods ### `embed(input, normalize=NORM_MODE_EUCLIDEAN, truncate=True, separator=None, return_count=False)` @@ -129,7 +165,10 @@ version_target: "latest" - Token-level embeddings: `LLAMA_POOLING_TYPE_NONE (0)`. 2. **Batch Optimization for Large Datasets**: - - Adjust `n_batch` and `n_ubatch` to balance performance and memory. + - Adjust `n_batch`, `n_ubatch`, and `n_seq_max` to balance parallelism, + performance, and memory. + - If `seq_id` exceeds the configured capacity, increase `n_seq_max` to at + least `seq_id + 1`. - Streaming processing avoids OOM for large datasets. 3. **Normalization Selection**: @@ -153,7 +192,12 @@ To generate embeddings, use the `LlamaEmbedding` class. It automatically configu from llama_cpp.llama_embedding import LlamaEmbedding, LLAMA_POOLING_TYPE_NONE # Initialize the model (automatically sets embeddings=True) -llm = LlamaEmbedding(model_path="path/to/bge-m3.gguf", n_gpu_layers=-1, pooling_type=LLAMA_POOLING_TYPE_NONE) +llm = LlamaEmbedding( + model_path="path/to/bge-m3.gguf", + n_gpu_layers=-1, + pooling_type=LLAMA_POOLING_TYPE_NONE, + n_seq_max=128, +) # 1. Simple usage (OpenAI-compatible format) response = llm.create_embedding("Hello, world!") @@ -263,7 +307,8 @@ embeddings_raw = llm.embed(["search query", "document text"], normalize=NORM_MOD ## Notes - This class is in development; some features may be unstable, especially reranking model support. -- Performance issues can be addressed by adjusting `n_batch`, `n_ubatch`, and `n_gpu_layers`. +- Performance issues can be addressed by adjusting `n_batch`, `n_ubatch`, + `n_seq_max`, and `n_gpu_layers`. - For custom models, manual `pooling_type` configuration may be required to match model behavior. ## Related Links From df3ca79d9cdd3f9374bd1b821fd73b5ae931a018 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 09:27:05 +0800 Subject: [PATCH 269/304] docs(llama): expand embedding parameters and API guidance - Add a role overview and reorganize constructor options into focused, readable parameter groups. - Document embedding, pooling, attention, KV cache, sequence capacity, and recurrent-state settings with their defaults and runtime behavior. - Expand the embed() and create_embedding() sections with normalization modes, return shapes, batching semantics, pooling recommendations, OpenAI compatibility notes, and resource-safe examples. - Fix the YAML frontmatter and improve Markdown spacing for cleaner Wiki rendering. Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 176 ++++++++++++++++++++++++++++++++++------ 1 file changed, 150 insertions(+), 26 deletions(-) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index d768e43ec3..305624c8fc 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -1,4 +1,3 @@ -```yaml --- title: Llama Class module_name: llama_cpp.llama @@ -7,16 +6,29 @@ class_name: Llama last_updated: 2026-07-26 version_target: "latest" --- -``` ## Overview + The `Llama` class is the core, high-level Python wrapper for a `llama.cpp` model. It handles model loading, memory management (KV cache), tokenization, and generation (both base text completion and chat formatting). It includes advanced features like dynamic LoRA routing, dual-mode hybrid/recurrent checkpointing, speculative decoding, and context shifting. +## Role in the Library + +`Llama` is the main user-facing entry point for loading a GGUF model and +creating a native `llama.cpp` context. It exposes completion, chat, tokenization, +embedding, state, sampling, and runtime configuration APIs through one managed +object. + +Use `Llama` when one application needs a general-purpose model interface. +For embedding-only applications, `LlamaEmbedding` provides embedding-oriented +defaults and additional reranking helpers while inheriting the same model and +context lifecycle. + ## Constructor (`__init__`) Initialize the model and context. Note that model loading will immediately allocate RAM/VRAM based on the selected offloading parameters. ### Core Model & Hardware Parameters + | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | | `model_path` | `str` | **Required** | Model file path (GGUF format) | @@ -29,23 +41,39 @@ Initialize the model and context. Note that model loading will immediately alloc | `use_mmap` | `bool` | `True` | Whether to use memory mapping (mmap) if possible. | | `use_mlock` | `bool` | `False` | Force the system to keep the model in RAM, preventing swapping. | | `kv_overrides` | `Dict` | `None` | Key-value overrides for the model metadata (supports bool, int, float, str). | -| `numa` | `Union[bool, int]`| `False` | NUMA strategy (e.g., `GGML_NUMA_STRATEGY_DISTRIBUTE`). | +| `numa` | `Union[bool, int]` | `False` | NUMA strategy (e.g., `GGML_NUMA_STRATEGY_DISTRIBUTE`). | + +### Context & Batch Parameters -### Context & Performance Parameters | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | | `n_ctx` | `int` | `512` | Text context size. Set to `0` to load from model metadata. | -| `n_batch` | `int` | `2048` | Maximum batch size for prompt processing. | -| `n_ubatch` | `int` | `512` | Physical batch size. | +| `n_keep` | `int` | `256` | Preferred number of leading tokens to preserve during automatic context shifting. | +| `n_batch` | `int` | `2048` | Maximum number of tokens in a logical prompt-processing batch. The effective value cannot exceed `n_ctx`. | +| `n_ubatch` | `int` | `512` | Maximum number of tokens in a physical micro-batch processed by llama.cpp. | +| `n_seq_max` | `int` | `1` | Maximum independent sequence states in one decode batch. Embedding calls split automatically at this limit; larger values enable more parallel sequences. | +| `n_rs_seq` | `int` | `0` | Experimental recurrent-state snapshots retained per sequence for rollback. `0` disables rollback snapshots. | +| `n_outputs_max` | `int` | `0` | Maximum outputs in a physical batch. `0` is converted to the effective `n_batch`. | | `n_threads` | `int` | `None` | Number of threads for generation (defaults to CPU count // 2). | -| `n_threads_batch`| `int` | `None` | Number of threads for batch processing (defaults to CPU count). | -| `flash_attn_type`| `int` | `AUTO` | Controls Flash Attention activation (`LLAMA_FLASH_ATTN_TYPE_AUTO`). | -| `swa_full` | `bool` | `None` | Whether to use full-size SWA cache | -| `kv_unified` | `bool` | `None` | Use single unified KV buffer for the KV cache of all sequences | -| `type_k` / `type_v`| `int` | `None` | KV cache data type for K and V (defaults to `f16`). | -| `offload_kqv` | `bool` | `True` | Whether to offload K, Q, V tensors to GPU. | +| `n_threads_batch` | `int` | `None` | Number of threads for batch processing (defaults to CPU count). | +| `ctx_type` | `int` | `LLAMA_CONTEXT_TYPE_DEFAULT` | Context implementation selected by llama.cpp. Keep the default unless a model or backend requires another context type. | + +### Embedding, Attention & KV Parameters + +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `embeddings` | `bool` | `False` | Enable embedding extraction alongside logits. Must be `True` before calling `embed()` or `create_embedding()`. | +| `pooling_type` | `int` | `LLAMA_POOLING_TYPE_UNSPECIFIED` | Pooling strategy for embedding output. `UNSPECIFIED` follows model metadata, `NONE` returns token-level vectors, and `RANK` returns classifier or reranking output. | +| `attention_type` | `int` | `LLAMA_ATTENTION_TYPE_UNSPECIFIED` | Attention mode used by the context. `UNSPECIFIED` lets llama.cpp select the model-compatible behavior. | +| `logits_all` | `bool` | `False` | Retain logits for every evaluated token instead of only requested outputs. Completion log probabilities require this mode. | +| `flash_attn_type` | `int` | `LLAMA_FLASH_ATTN_TYPE_AUTO` | Controls when Flash Attention is enabled. | +| `offload_kqv` | `bool` | `True` | Offload K, Q, and V tensor operations to the selected device when supported. | +| `swa_full` | `Optional[bool]` | `None` | Use a full-size sliding-window-attention cache. `None` keeps llama.cpp's default. | +| `kv_unified` | `Optional[bool]` | `None` | Use a unified KV buffer for all sequences. `LlamaEmbedding` enables this automatically. | +| `type_k` / `type_v` | `Optional[int]` | `None` | KV cache data types for keys and values. `None` uses llama.cpp defaults. | ### Advanced & Chat Parameters + | Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | | `chat_format` | `str` | `None` | String specifying the chat template (e.g., `"llama-2"`, `"chatml"`). Guessed from GGUF if None. | @@ -71,7 +99,9 @@ Initialize the model and context. Note that model loading will immediately alloc ## Core Methods ### `create_chat_completion` + Generates a chat response using the configured `chat_format` or `chat_handler`. + ```python import llama_cpp @@ -89,7 +119,9 @@ print(response["choices"][0]["message"]["content"]) ``` ### `create_completion` / `__call__` + Generates standard text completion from a raw string prompt. + ```python import llama_cpp @@ -99,7 +131,9 @@ print(output["choices"][0]["text"]) ``` ### `generate` + A low-level generator yielding token IDs one by one. Highly customizable with sampling parameters, dynamic LoRA mounting, and control vectors. + ```python import llama_cpp @@ -111,14 +145,18 @@ for token in model.generate(tokens, top_k=40, top_p=0.95, temp=0.2): ``` ### `eval` + Low-level method to ingest and evaluate a sequence of tokens. Used internally to update the KV cache and logits. Handles **Context Shifting** automatically to prevent OOM when the token count exceeds `n_ctx`. + ```python # Evaluates a chunk of tokens and updates internal state model.eval(tokens=[1, 453, 234, 987], active_loras=[{"name": "coding_adapter", "scale": 1.0}]) ``` ### `abort` + Immediately halts an active generation loop safely. + * **Usage**: Typically called from a separate monitoring thread (like a timer). When triggered, the running stream will exit and the final chunk will contain `"finish_reason": "abort"`. ### Runtime Logging Control @@ -158,7 +196,9 @@ llm.set_verbosity(1) ``` ### Dynamic LoRA Management + The `Llama` class allows you to load multiple LoRAs into VRAM and apply them dynamically per-generation or per-eval. + * `load_lora(name: str, path: str)`: Loads an adapter into VRAM (does not apply it yet). * `unload_lora(name: str)`: Releases the specific LoRA from VRAM. * `list_loras() -> List[str]`: Returns names of all registered LoRAs. @@ -436,33 +476,117 @@ The `Llama` embedding methods are maintained and use streaming batches. Create the model with `embeddings=True` before calling them. ```python +from llama_cpp import Llama, LLAMA_POOLING_TYPE_UNSPECIFIED + llm = Llama( - model_path="path/to/model.gguf", + model_path="path/to/embedding-model.gguf", embeddings=True, + pooling_type=LLAMA_POOLING_TYPE_UNSPECIFIED, + n_batch=512, + n_ubatch=512, n_seq_max=8, kv_unified=True, ) -# Raw sequence or token-level embeddings. -vectors = llm.embed(["query", "document"], normalize=2) +try: + # Raw sequence embeddings with explicit L2 normalization. + vectors = llm.embed(["query", "document"], normalize=2) -# OpenAI-compatible response. -response = llm.create_embedding(["query", "document"], normalize=True) + # OpenAI-compatible response. + response = llm.create_embedding( + ["query", "document"], + normalize=True, + ) +finally: + llm.close() ``` -`embed()` accepts strings, lists of strings, or pre-tokenized inputs. It supports -token-level output (`LLAMA_POOLING_TYPE_NONE`), sequence pooling, rank-model -outputs, streaming batches, and llama.cpp integer normalization modes. +### `embed(input, normalize=False, truncate=True, separator=None, return_count=False)` + +Generate raw embedding values for strings or pre-tokenized inputs. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `input` | `Union[str, List[str], List[List[int]]]` | Required | A single string, a list of strings, or a list containing pre-tokenized token-ID lists. | +| `normalize` | `Union[bool, int]` | `False` | `False` returns raw values, while `True` applies L2 normalization. Integer modes are listed below. Rank outputs are not normalized. | +| `truncate` | `bool` | `True` | Truncate each input to the smaller of the context capacity and logical batch capacity. If disabled, an input longer than `n_batch` raises `ValueError`. | +| `separator` | `Optional[str]` | `None` | Split a single string into multiple independent inputs. When set, the result uses the batch return shape. | +| `return_count` | `bool` | `False` | Return `(result, total_token_count)` instead of only the embedding result. | + +Normalization modes follow the llama.cpp embedding example: + +| Value | Behavior | +|---|---| +| `False` or `-1` | No normalization | +| `True` or `2` | Euclidean/L2 normalization | +| `0` | Scale by the maximum absolute value to a maximum magnitude of `32760` | +| `1` | Taxicab/L1 normalization | +| Integer greater than `2` | Corresponding p-norm normalization | + +Unlike `LlamaEmbedding.embed()`, the standard `Llama.embed()` method defaults to +raw, unnormalized output for backward compatibility. + +The return shape depends on the input and pooling type: + +| Input / pooling mode | Return shape | +|---|---| +| Single string with sequence pooling | `List[float]` | +| String list or separator-split string with sequence pooling | `List[List[float]]` | +| `LLAMA_POOLING_TYPE_NONE` | One token embedding matrix per input: `List[List[float]]` for a single string or `List[List[List[float]]]` for a batch | +| `LLAMA_POOLING_TYPE_RANK` with one classifier output | A scalar for a single string or a list of scalars for a batch | +| `LLAMA_POOLING_TYPE_RANK` with multiple classifier outputs | A classifier vector for each input | +| Any mode with `return_count=True` | `(result, total_token_count)` | + +Use `LLAMA_POOLING_TYPE_UNSPECIFIED` for ordinary sentence embeddings unless +the model documentation requires a specific sequence pooling strategy. +`LLAMA_POOLING_TYPE_NONE` is token-level output and should not be used when one +vector per input document is expected. + +### `create_embedding(input, model=None, normalize=False, truncate=True)` + +Wrap sequence or token-level embedding output in an OpenAI-compatible response: + +```python +{ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [...], + "index": 0, + } + ], + "model": "path/to/embedding-model.gguf", + "usage": { + "prompt_tokens": 12, + "total_tokens": 12, + }, +} +``` + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `input` | `Union[str, List[str]]` | Required | One string or a list of strings. | +| `model` | `Optional[str]` | `None` | Model name placed in the response. Defaults to `model_path`. | +| `normalize` | `Union[bool, int]` | `False` | Passed directly to `embed()`. | +| `truncate` | `bool` | `True` | Passed directly to `embed()`. | For parallel batches, `n_seq_max` must cover every sequence ID active in a -single decode batch. The default `n_seq_max=1` supports only `seq_id=0`. -For example, `n_seq_max=8` permits IDs `0` through `7`. If this capacity is -exceeded, the exception reports the valid range and the minimum value required. -`n_batch` limits tokens, while `n_seq_max` limits independent sequences. +single decode batch. The default `n_seq_max=1` is valid and processes multiple +inputs sequentially. Increasing it allows more inputs to be decoded in +parallel; for example, `n_seq_max=8` permits IDs `0` through `7` in one batch. +`n_batch` limits logical input tokens, `n_ubatch` controls the physical token +batch, and `n_seq_max` limits independent sequences. `LlamaEmbedding` remains available as the specialized convenience class. It -automatically enables embedding-oriented context options and adds the `rank()` -helper for formatting query/document pairs. +automatically enables embedding-oriented context options, defaults to L2 +normalization, provides additional output formats, and adds the `rank()` helper +for formatting query/document pairs. + +> **OpenAI compatibility:** use sequence pooling when calling +> `create_embedding()` through an OpenAI-compatible client. Token-level pooling +> (`LLAMA_POOLING_TYPE_NONE`) produces nested token vectors rather than the +> single flat vector normally expected for each input. --- From a27aa9b854180069c1d725315678030efdca58ba Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 13:05:54 +0800 Subject: [PATCH 270/304] docs(embedding): add end-to-end embeddings and reranking guide - Create a schema-compliant feature guide covering sentence embeddings, token-level vectors, reranking workflows, pooling modes, normalization, streaming batch configuration, return shapes, and output formats. - Add complete examples for the standard Llama API, LlamaEmbedding, pre-tokenized inputs, cosine-similarity output, and cross-encoder reranking. - Document common configuration problems, implementation limitations, and the embedding and reranking model families currently listed as supported by the project. - Expose the new feature guide through the Wiki index. Signed-off-by: JamePeng --- docs/wiki/features/embeddings-rerank.md | 419 ++++++++++++++++++++++++ docs/wiki/index.md | 12 + 2 files changed, 431 insertions(+) diff --git a/docs/wiki/features/embeddings-rerank.md b/docs/wiki/features/embeddings-rerank.md index e69de29bb2..b5cb54ed44 100644 --- a/docs/wiki/features/embeddings-rerank.md +++ b/docs/wiki/features/embeddings-rerank.md @@ -0,0 +1,419 @@ +--- +title: Embeddings and Reranking +feature_name: Embeddings and Reranking +source_files: + - llama_cpp/llama.py + - llama_cpp/llama_embedding.py + - llama_cpp/_internals.py +last_updated: 2026-07-26 +version_target: "latest" +--- + +# Embeddings and Reranking + +## Overview + +`llama-cpp-python` can use compatible GGUF models for three related inference +workflows: + +- **Sentence or document embeddings** produce one vector per input. +- **Token embeddings** produce one vector per token. +- **Reranking** scores each query/document pair with a cross-encoder model. + +The general-purpose `Llama` class and the specialized `LlamaEmbedding` class +share the same native model and context implementation. Both support streaming +batches, pre-tokenized inputs, multiple pooling modes, and configurable vector +normalization. + +`LlamaEmbedding` adds embedding-oriented defaults, extra output formats, and +the `rank()` helper. The standard `Llama` API is useful when an application +already manages models through the main class or needs both generation and +embedding capabilities. + +## When to Use + +| Goal | Recommended API | Pooling | +|---|---|---| +| Store one vector per sentence or document | `Llama.embed()` or `LlamaEmbedding.embed()` | `LLAMA_POOLING_TYPE_UNSPECIFIED`, or the model-required MEAN/CLS/LAST mode | +| Return an OpenAI-style embedding response | `create_embedding()` | Sequence pooling | +| Inspect a vector for every token | `embed()` | `LLAMA_POOLING_TYPE_NONE` | +| Score documents against a query | `LlamaEmbedding.rank()` | `LLAMA_POOLING_TYPE_RANK` | +| Return raw arrays or a cosine-similarity matrix | `LlamaEmbedding.create_embedding()` | Sequence pooling | + +Use the pooling configuration documented by the model author whenever one is +provided. `LLAMA_POOLING_TYPE_UNSPECIFIED` lets model metadata select the +sequence-pooling behavior and is the safest general default for ordinary +sentence embeddings. + +## Supported Models + +The project README currently lists the following GGUF model families as working +with the embedding and reranking APIs: + +| Model family | Task | GGUF model | +|---|---|---| +| `bge-m3` | Embedding | [bge-m3-GGUF](https://huggingface.co/gpustack/bge-m3-GGUF) | +| `jina-embeddings-v2-base-zh` | Embedding | [jina-embeddings-v2-base-zh-GGUF](https://huggingface.co/gpustack/jina-embeddings-v2-base-zh-GGUF) | +| `jina-embeddings-v3` | Embedding | [jina-embeddings-v3-GGUF](https://huggingface.co/second-state/jina-embeddings-v3-GGUF) | +| `bge-reranker-v2-m3` | Reranking | [bge-reranker-v2-m3-GGUF](https://huggingface.co/gpustack/bge-reranker-v2-m3-GGUF) | +| `qwen3-reranker` | Reranking | [Qwen3-Reranker-GGUF](https://huggingface.co/JamePeng2023/Qwen3-Reranker-GGUF) | + +This is a known-compatible list, not an exhaustive compatibility matrix. +Support for a specific file still depends on its GGUF metadata, pooling +configuration, classifier head, tokenizer, and reranking template. Validate +the output shape and quality before deploying a new model or quantization. + +## Related APIs + +| API | Role | +|---|---| +| `Llama(..., embeddings=True)` | General-purpose model interface with maintained `embed()` and `create_embedding()` methods | +| `LlamaEmbedding(...)` | Specialized subclass that forces `embeddings=True` and `kv_unified=True` | +| `Llama.embed()` | Raw sequence, token-level, or rank output with optional token counting | +| `Llama.create_embedding()` | OpenAI-compatible response wrapper; defaults to raw vectors | +| `LlamaEmbedding.embed()` | Specialized raw embedding API; defaults to L2 normalization | +| `LlamaEmbedding.create_embedding()` | Adds `json`, `json+`, and `array` output formats | +| `LlamaEmbedding.rank()` | Formats query/document pairs and returns reranking scores | +| `Llama.tokenize()` | Converts text into token IDs for pre-tokenized embedding input | + +See [[core/Llama|Llama]] for the general model lifecycle and +[[modules/LlamaEmbedding|Llama Embedding]] for the complete specialized class +reference. + +## Code Examples + +All examples assume that `MODEL_PATH` points to a compatible GGUF embedding or +reranking model. Pooling requirements and output dimensions are model-specific. + +### Sentence Embeddings with `Llama` + +```python +from llama_cpp import Llama, LLAMA_POOLING_TYPE_UNSPECIFIED + + +MODEL_PATH = "path/to/embedding-model.gguf" + +model = Llama( + model_path=MODEL_PATH, + embeddings=True, + pooling_type=LLAMA_POOLING_TYPE_UNSPECIFIED, + n_ctx=512, + n_batch=512, + n_ubatch=512, + n_seq_max=8, + kv_unified=True, + n_gpu_layers=-1, + verbose=False, +) + +try: + documents = [ + "The weather is pleasant today.", + "A storm is expected tomorrow.", + "Vector search compares semantic meaning.", + ] + + vectors, token_count = model.embed( + documents, + normalize=True, + return_count=True, + ) + + print("vectors:", len(vectors)) + print("dimension:", len(vectors[0])) + print("processed tokens:", token_count) + + response = model.create_embedding( + documents, + normalize=2, + ) + print(response["usage"]) +finally: + model.close() +``` + +For `Llama.embed()`, `normalize=False` is the backward-compatible default. +`True` and integer mode `2` both select L2 normalization. + +### Specialized Batch Embeddings and Similarity + +```python +from llama_cpp import LLAMA_POOLING_TYPE_UNSPECIFIED +from llama_cpp.llama_embedding import ( + LlamaEmbedding, + NORM_MODE_EUCLIDEAN, +) + + +MODEL_PATH = "path/to/embedding-model.gguf" + +model = LlamaEmbedding( + model_path=MODEL_PATH, + pooling_type=LLAMA_POOLING_TYPE_UNSPECIFIED, + n_ctx=512, + n_batch=512, + n_ubatch=512, + n_seq_max=8, + n_gpu_layers=-1, + verbose=False, +) + +try: + texts = ["apple", "fruit", "automobile"] + + # "array" always returns one vector entry per input. + vectors = model.create_embedding( + texts, + normalize=NORM_MODE_EUCLIDEAN, + output_format="array", + ) + print("first vector dimension:", len(vectors[0])) + + response = model.create_embedding( + texts, + normalize=NORM_MODE_EUCLIDEAN, + output_format="json+", + ) + print(response["cosineSimilarity"]) +finally: + model.close() +``` + +`json+` extends the OpenAI-style response with `cosineSimilarity` when at least +two compatible sequence vectors are available. + +### Token-Level Embeddings + +```python +from llama_cpp import Llama, LLAMA_POOLING_TYPE_NONE + + +model = Llama( + model_path="path/to/embedding-model.gguf", + embeddings=True, + pooling_type=LLAMA_POOLING_TYPE_NONE, + n_ctx=256, + n_batch=256, + verbose=False, +) + +try: + token_vectors = model.embed("Token-level example", normalize=True) + + print("tokens:", len(token_vectors)) + print("dimension per token:", len(token_vectors[0])) +finally: + model.close() +``` + +Token-level output is a matrix, not one flat vector per document. It is useful +for token analysis and custom pooling, but it is not the normal shape expected +by OpenAI-compatible vector-store clients. + +### Pre-tokenized and Separator-Split Inputs + +```python +from llama_cpp import Llama, LLAMA_POOLING_TYPE_UNSPECIFIED + + +model = Llama( + model_path="path/to/embedding-model.gguf", + embeddings=True, + pooling_type=LLAMA_POOLING_TYPE_UNSPECIFIED, + n_ctx=256, + n_batch=256, + verbose=False, +) + +try: + token_batches = [ + model.tokenize(b"first document"), + model.tokenize(b"second document"), + ] + vectors = model.embed(token_batches, normalize=2) + + split_vectors = model.embed( + "first document\nsecond document", + separator="\n", + normalize=2, + ) + + print(len(vectors), len(split_vectors)) +finally: + model.close() +``` + +When `separator` is set, a single string is treated as a batch and the return +value uses the batch shape. + +### Reranking Query/Document Pairs + +```python +from llama_cpp import LLAMA_POOLING_TYPE_RANK +from llama_cpp.llama_embedding import LlamaEmbedding + + +RERANK_MODEL_PATH = "path/to/reranker-model.gguf" + +ranker = LlamaEmbedding( + model_path=RERANK_MODEL_PATH, + pooling_type=LLAMA_POOLING_TYPE_RANK, + n_ctx=1024, + n_batch=1024, + n_ubatch=512, + n_seq_max=8, + n_gpu_layers=-1, + verbose=False, +) + +try: + query = "What causes rain?" + documents = [ + "Rain forms when atmospheric water vapor condenses and falls.", + "A cake is made from flour, eggs, and sugar.", + "Cloud droplets grow until gravity pulls them toward the ground.", + ] + + scores = ranker.rank(query, documents) + ranked = sorted( + zip(documents, scores), + key=lambda item: item[1], + reverse=True, + ) + + for document, score in ranked: + print(f"{score:.6f} {document}") +finally: + ranker.close() +``` + +`rank()` first checks for a model-provided `rerank` chat template. If no +template exists, it constructs a sequence from the model's BOS, separator, and +EOS tokens. + +## Configuration Notes + +### Pooling Modes + +| Constant | Output behavior | Typical use | +|---|---|---| +| `LLAMA_POOLING_TYPE_UNSPECIFIED` | Uses the model-configured pooling behavior | Default for sentence embedding models | +| `LLAMA_POOLING_TYPE_NONE` | One vector per token | Token analysis or custom pooling | +| `LLAMA_POOLING_TYPE_MEAN` | Mean-pooled sequence vector | Models trained for mean pooling | +| `LLAMA_POOLING_TYPE_CLS` | Vector from the classification token | Models trained with CLS pooling | +| `LLAMA_POOLING_TYPE_LAST` | Vector from the final token | Models trained with last-token pooling | +| `LLAMA_POOLING_TYPE_RANK` | Classifier or reranking output | Cross-encoder reranking models | + +Do not select `LLAMA_POOLING_TYPE_NONE` when one vector per input is required. +It changes both the amount of output and its nesting depth. + +### Normalization Modes + +| Mode | Value | Behavior | +|---|---:|---| +| `NORM_MODE_NONE` | `-1` | Return raw values | +| `NORM_MODE_MAX_INT16` | `0` | Scale the maximum absolute component to `32760` | +| `NORM_MODE_TAXICAB` | `1` | L1/taxicab normalization | +| `NORM_MODE_EUCLIDEAN` | `2` | L2/Euclidean normalization | +| p-norm | Any integer greater than `2` | Normalize using the corresponding p-norm | + +The constant `NORM_MODE_PNORM` currently has value `6`; callers may also pass a +different integer greater than `2`. + +Normalization defaults differ between the two classes: + +| API | Default | +|---|---| +| `Llama.embed()` / `Llama.create_embedding()` | Raw output (`False`) | +| `LlamaEmbedding.embed()` / `LlamaEmbedding.create_embedding()` | L2 (`NORM_MODE_EUCLIDEAN`) | +| Rank output | Never normalized | + +L2-normalized vectors are convenient for cosine similarity because their dot +product is their cosine similarity. + +### Batch and Context Capacity + +| Parameter | Controls | +|---|---| +| `n_ctx` | Maximum context length available to an input sequence | +| `n_batch` | Maximum tokens in one logical decode batch | +| `n_ubatch` | Physical token micro-batch size used by llama.cpp | +| `n_seq_max` | Maximum independent sequences decoded together | + +Embedding input lists are streamed through multiple decode batches. The +default `n_seq_max=1` is valid and processes inputs sequentially. Increasing it +allows more independent sequences to be decoded together, but may use more +context resources. + +Each individual tokenized sequence must fit the configured logical batch +capacity. Choose `n_batch` large enough for the longest intended input and use +`truncate=True` when truncation is acceptable. + +### Input and Return Shapes + +| Input and mode | Direct `embed()` result | +|---|---| +| Single string with sequence pooling | `List[float]` | +| String list with sequence pooling | `List[List[float]]` | +| Separator-split string with sequence pooling | `List[List[float]]` | +| Single string with token-level pooling | `List[List[float]]` | +| String list with token-level pooling | `List[List[List[float]]]` | +| Rank model with one classifier output | Scalar for one string; list of scalars for a batch | +| Rank model with multiple classifier outputs | Classifier vector per input | +| Any input with `return_count=True` | `(result, processed_token_count)` | + +Token counts are measured after tokenization and any applied truncation. + +### Output Wrappers + +`Llama.create_embedding()` returns an OpenAI-compatible dictionary containing +`object`, `data`, `model`, and token `usage`. + +`LlamaEmbedding.create_embedding()` supports: + +| `output_format` | Result | +|---|---| +| `"json"` | OpenAI-style response | +| `"json+"` | OpenAI-style response plus a cosine-similarity matrix when available | +| `"array"` | Raw list containing one output entry per input | + +For OpenAI-compatible vector-store clients, use sequence pooling so each +`data[i]["embedding"]` value is a flat vector. + +### Common Configuration Problems + +| Symptom | Cause | Action | +|---|---|---| +| `Llama model must be created with embeddings=True` | Standard `Llama` was initialized without embedding extraction | Recreate it with `embeddings=True` | +| Output is a matrix for each document | `LLAMA_POOLING_TYPE_NONE` selects token-level output | Use `UNSPECIFIED` or the pooling mode required by the model | +| `seq_id` exceeds `n_seq_max` in custom batch code | A manual sequence ID is outside the configured capacity | Increase `n_seq_max` or use IDs within `0..n_seq_max-1` | +| A long input exceeds `n_batch` | One tokenized sequence is larger than the logical batch | Increase `n_batch`, shorten the input, or enable truncation | +| Local source changes are not visible | Python imported an installed `site-packages` build | Print `llama_cpp.__file__`, then reinstall or adjust the development environment | + +## Limitations + +- Embedding dimensions, valid pooling modes, tokenization, and reranking heads + are determined by the GGUF model. A model that was not exported for the + requested task may not produce meaningful output. +- `rank()` returns raw model scores. They are not automatically calibrated as + probabilities and should primarily be compared within the same query. +- For a two-output reranking head, `rank()` uses the first output as the score. + It does not apply softmax. +- The fallback reranking prompt depends on the model's BOS, separator, and EOS + tokens. Prefer a GGUF model containing a suitable `rerank` chat template. +- `json+` similarity output is intended for at least two compatible, + fixed-length sequence vectors. It is not suitable for ragged token-level + matrices or scalar rank scores. +- Embedding calls clear the context memory used by the operation. Do not expect + a previous completion KV-cache state to remain reusable after embedding on + the same model instance. +- Model and reranking support still requires broader testing across GGUF + architectures. Validate output quality and shape before production use. + +## Related Features +- [[Index-Home](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/index.md)] +- [[core/Llama|Llama](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/core/Llama.md)] ā€” General model lifecycle and built-in embedding APIs. +- [[modules/LlamaEmbedding|Llama Embedding](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/modules/LlamaEmbedding.md)] ā€” Specialized API reference, + normalization constants, and reranking methods. +- [[install|Installation](https://github.com/JamePeng/llama-cpp-python/blob/main/docs/wiki/install.md)] ā€” Backend selection, GPU acceleration, and source + installation. diff --git a/docs/wiki/index.md b/docs/wiki/index.md index 8e5dbed14b..bc029f739c 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -44,6 +44,17 @@ These pages document major source modules and related classes. --- +### Features + +Workflow guides combine related classes and configuration into complete usage +patterns. + +| Page | Description | +|---|---| +| [features/embeddings-rerank\|Embeddings and Reranking] | Sentence embeddings, token-level vectors, normalization, streaming batches, similarity output, and cross-encoder reranking. | + +--- + ### Development This section contains maintainer-facing development notes, workflows, and LLM-assisted helper tools for working on `llama-cpp-python`. @@ -99,6 +110,7 @@ Currently available pages: - `modules/LlamaGrammar.md` - `modules/LlamaSpeculative.md` - `modules/Logger.md` +- `features/embeddings-rerank.md` - `development/git-commit-generation-agent.md` - `SCHEMA.md` - `contributing-to-wiki.md` From 1857a6065ea4ed47043ebbc7868b95dee6bf7e94 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 14:51:44 +0800 Subject: [PATCH 271/304] docs(readme): replace the new logo with fork project branding - Add the new llama-cpp-python logo asset under docs and update the README header to reference the repository-local image. - the new logo which combined llama, C++, and Project branding remains readable. Signed-off-by: JamePeng --- README.md | 2 +- docs/icon.png | Bin 0 -> 331909 bytes 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/icon.png diff --git a/README.md b/README.md index 1345c06904..c07fafa730 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- + llama-cpp-python logo

# Python Bindings for [`llama.cpp`](https://github.com/ggml-org/llama.cpp) diff --git a/docs/icon.png b/docs/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..d2d754d746d23a777296c98c996e0ca949cf80ee GIT binary patch literal 331909 zcmZ5{Wl$Vl7bfoR8rksUHHaws)9x$bG?owL^Y)`Uh;-LLP1m*9SpZ|QSs_bM%>So2n^?5#Z7 zUJfP7_nu}rPgcEpY~tJ4z1`W(s$lxRdqdvtMf|pDzt~p0)}#$1R!Iw&OO05? zENdNdIA^5qy##7lx~O5fJ!x+>|GPT;4>;WK7Zwo4V@Af1$?$6t{<|Lu`x}=~{coOJ z5{u;U6D=+3>e6f0zpn~4DSIZq?ep#DSe5_w1ogJ3!AYI1xf{)>Lxa=DmkDPi41QKA zW5|%rwj1%!`{gu_U%rqWGyh%baM35@fcGa*gWWt~Zw=}!EG%821^^Nmx@qZ=8;%?# zkrlD*4Okz~@);Z-S9V$(OW%dl&1wo-S0{Lvu`vqr(LDo^&r8@o)5!MaQ2vJW0}M>X zG6M*Rx0HE#)vWpYIEwmQ(U9bk^xsm4D^35t5=HgxMwt#?yN7sLmX)@e$1+u-hGT3o zigx0m&#mnTJ1h}CV1HkcEcA_-@cq56As#|0LcW95pxv*VDa3}Sgi6z-Vy4Ox$mC)E z$|A4sr&;aN0e|(^9f9d#_kJ}@7AWKwq{)n3m>9kya&oZm$Ma1*cl$zZ$!kjHB2Y z{pEDr$Fv$|szBn<e!w`;yW8WquYHq5Qe2j*$Cx zofM5L#AUT(y!2+a?_K*qW4M2Nt%;tuG+1tOUyL$R!A8JNK8nfCd1d;<^}#Yi5c-p9 z65A=GB;#BZlF1zkLhg`L?u#(y^krY z#pVar!bu>H z@kt^QaE<`O@dpdiD`Rh2Qhllx+HO=wy=volm-)#*KSmLMb1J#xk!{oZTRYy&PQ{~Wleh9F$~GVrwQ2znI;`P+~c946%F{ zjJ2Mdv~GwkIAi~@)6t#!)J}SpwwgKq+eR#mI{#U{*&DiYa^e1}@p*)moz@Xtaxop7 zCRGh!$F?JL($PZd58E25K@UcMdJ&dT4dF$&y<)}TWD<6~$wcWo;&ZTkY@huSFWwJ; zBdoGt5(zCQ(?FhoNRmJqk}A_;;In5MYPh}q^(0CH`BzU+i2p$V+8Z!V=(%Lr!rIgs z^(1oTPp}`PJn+JJPO@VqRIJb85J{XpKAx6XDsz zC;v<*Ofeys(KbCj ztV>N#oBbG6)0yM#z|v1!L4V|;VAL8ixoST$Oo-tzA#}Fu41<`{5}abIt8QB-rQmhKJV!9F74l{lbgKx3AGNb>1{gI@7a2(}|_QWKh*v zqA9d3e3($dkJ{CMRzU-Hc+v+Ej-DwfaYLZ2K1d|eN4s1IEch>I{a>sjeKWAV_HK2( zw6v(x!($&*yT>D`q2tgZv29B3g!nn`PwGy2(CvO`N=WHmlg?iI?qi)IZCjsdx%i4S z7)&63*$p_~kw^pO^?6AS>Y z%rt&drRvc!Aun_X@umozgDtfj!gXRtUt(qvANUPk@q9JxNY17vM|Hw8cS}6@CG%_Ur*yV|wtz8UKMZWhiDlU3STpO@^AI|8wR@?}b_pf2J8@*zLataj^0_iV zz#5*Be}8<)`IEG0r~8iv!0Q5}Z^ZkmO#g-2Bbqny(FVE77VCJq*6py^UahV>H@I*$ zU?O#ih5WAQJT3jS@|1g)gW5)DbT^)+dO?1LibSnXQ3yPo6JMi^?l4|75pHpMfrx1! z`&zZ-vUP?BOGMgC;;i?BSugXkcf2A&Vjdn|EgRQRz-HZcNgs4TY5&R~QbX`-FEiTE z4MZ9^wesLl>aN;dF7($&N&i9b4dfeXxLRGJkn8PYpr2XLjpWZ7r&RMOn1l2$p`|HC z(L&RLWE6U|`?CA`Lil<$$q*E!;u>2q>7A^&W}6&$Zpn7uhja^F7+`$)!GTbKX2b8T(hhrry%iVfgaf=9#oz3zz~ z)CR8LVB0US@~rjckmD3kb(F1zxYmRPRh(GI$Qwn^f@TwxI$3fH zo}&CBAbdCG7u?qNl~)UCj0~Ito2526W*ObZJS0Zq9)gL(b&JOu$mmjc zx2Z>fz<%Q;Md8^6ZJ6AE$Z23vPA#ILYV34I@->a#l1eGQUusquJDv0E_&9p{%}!G7 zFrV+ri-6p_pIRgR7a40%--E@pP2G<^XJ60gRw^3_v2R5LYqhRP>%pscL?2z2d-wJa zr6+P(RZ_#HDy?Hnp%7x)vyG&*ksgE)OrgX`sayM*1=TE^qZ`GWB@`3KSn?RLaPJG!-#>%cUF|SFyzmfG~pj!{ogWC6Yg%X0SN7>I&=I^Mc~#6ysoF zf?r%kKG461ZqC5}7|?t$6>WWxcE9rz>Z2XS^fgGU+g#fyY}N0WL+95T>(vcvw#J!6 zrc^RhB?y7XsqsJp|6 zHjsWXaaj*j2mlSFo+nW2z%Zwy)TP}8mTRFW=^&CzNrts;ZG5Pyp1vFqqjo?(yYf}eP+<#0JEaH%ZBm8RBX>Z+KkRh~tdJAd! zW^s6s|FgKYQ6piRs7(^po<3nZvC!{0i=BtB=LvTYe$NbDMbuISTIJ$>m3CA%MGiDl zbl;bPje%U52g(kPy?Ie6*R-p$G{f$3JIh^NeZnBt&li3$#)4BWNB&Bm70bC=P2}SX ztdfBD?8JoCreS~BG``0+<43&E%Km|A1F-bm z8^*YY@=4>Bvh3_@S-BzjaJ?p;a&1^S|#uZymbDfCHA;Y=Z8}i+x z>YL={F4+Y@Vj8*IC8$HJiO@7qUiAjtV|M_zd1PevF(>l(MNi##ds`7H@r-_Ibe?#9 zKy6U|Et$#s25TZZ9b?~Eo`vML6BOB!$@*|&_vp{Vn|DRBJ|g~;*y*J?SRv1Y$&Y0M z=LiqNy9U3{6f4ibB;z8rASUXRnBO92M6gktA!B((IR&+$)y7dX*`1+Y zA-2bztYz&kq)U_T@djbQ%e2o`>W0Zi7G~kuXrKI`c0p5n|zYMNj{aO+lyM5B-2V zWAbiM=0=0^>L&achwh=DW25-7zam3U5Wn)ECb^`OY-mjFH1TiTZv(y z1#DP?E!_6rIhO$s%UcjgeEMzKSXq-542sK2bM;zw{uHV6To9`=x-OI~ek0fbrf)&i zSSa=W#E%QAWEU2q_&%h11MwCtM_CjOzNC;yc>hB%>^Ol+ZjxYT!<08DY)Wy!Umn=m6I1qtIkDOhiGwR+o z3POAxuiL6Jgv)-NuzCwq5(b*vaB5b%WT zT#`>96BY?y*~tUVc+^nuysK2csPH1S-XlH^=cSFHWHwuFeiRX+&NVQiSs;@EzO{yx z;e0Hae`o8V(YFJ3Aq+s+-B^Hd@Z} z^fGLF(q!-tYfEBCR-)ZhYGS&|5pZxm+euZLkH-L3^~m2wMoHjWf~e(8W;l}%aww4q z773**A5AY8it73nubtiu7n{ljO8A$*d-=>~r2nLRBk)hkymUrke{yGGd&rX{lB-zr~}*26(N*$siTpXPPX3l`sXXkW&g4v`5v<=D=J$ksRBhqcVY2` z6R!JT-??L;LQ629VluwrA^s3lcnKGC8_KTF+Qi`I6?8*&0^PMH?lOn?+k`?1X{dRoa~6&y~t86hehp+5WLKqvvH~?&&g`c z%!};wqRgg+if?g1v6@be#|&@iP_z0Ey!tu^xd<~+KMl0uEtaFK8?D|`3+~2l?;Vne z=B&+#aPjO^+#5*WQzIkH&=GXw#O%fTZ?ztHS8hpxV&jxL$m=`5pUmS!?fvk84h6nL z!j0HJT9_V5UuP&L0TD9jh|XkY{+i)tkU^W;Q$6mB*~#FgeY9HhH+cQCkmMBSg9Hbn zev)}y)Bdm6ZwV{*%F99$sIUUH7er$~py6$zuCHAc9ZMhKf!*99%S|F?LP=S4T5WH< znQjVrN`WKlh#!bkv9iYCBaG}Lp(6UgkFv0v?y#uX>sR-)hzJ_o1cO^NO|dKIwW}%U6~`N8;JP3 z!!3mJHXw*M%(!I~qSTZvT<0zJ4u1{$kLTf2EPA@P<8=793OiwKOsgw(?TTC|At9sW ztYJ@%Qp41bRLe7lGUp__~arw!uA^HW-H?IJLA`iN~j|y>9_h!R{YCvQ^f+ z)oH;BPLG|_h}V`+y%>~qIALe>z_dLJRK7CA*h8$27VlyAJVqBc>|=i>nq* zjHM$7I@m_Xd+=Q6>jC^8{t+&*N3Ge;BiYVqdQ_0BCE16Pjw<^(3s$QQlnIhzAGu?O zCUKJ?F^MwsRtSOjzDM#i!higdL#}|O3540P_$GhjZ?@$=%pZM^d+~@$a`u`BFwgT4 zgSfTKmT+1~^y1?ruEKi8@&bTWZBN#~I6c{7M8&YQL1<2%BxA8eF;2T;b^^xmph#7d zIuc)n&n{DoOzm2rA+mI|=H3dF5t%_?{I?v$Xbb>!8Ft6@H?Ja+pn8k+nhlNN%*pR< zU+F%ET%Zs6lLRIUn`7*ZrBMYkT-c7|*ij`?VgSvTL<($WYUA=l)CiTTyz;n??^>B( z-fMTmPX8!guGw2592TthWQ1*#wd*`Qs;6+}9%dp#oS$nLCv6@#XGo5n57GM<@bpsT z5(I1la8?eVWM4zltA7iMy3tBG&6X$W1K`y~IQE809xhEW71rH98)FIJBxJ})%5$~E zFzLh$w2gBk+D`FPb zbFLdYKKQuRCiQN`?v!y(VC92#CuW=VB0Z-7)A7ad$kgJr>G`sMp~TFK88W*Jm40?r zON>7D5jUMn=5r;0$Ml)R4&ol{>Z8TOE6O~q8lI}$JL7vR)u9`%J@%O79T7Qd9bOD3p99g2$L~wDl z>Y%>jk!F|E>^z&z!$GE`vD@x&x^X9m0-(uJCb$&yqrnqFC^E{jLieqk}+t)6X4tt#n&H&iNK{%L{Qm4746q`pMv=0Qi)?XNuklsLdl)kp% z7xRrKeh7lQB2?u1>~IK*$5wZgO(hXwU8eA`kR3I%&@B=@x>&v%rc!otAQM?ceo|cX zxkA>`_j}g@vJ#C?`k&bVuQ6A+<)|6?lZ84G87DK0e|w<#@L`63^K}b6rC(R((!I`K z2382hs!X8*06N|i2)|D5$G(%?IoVOLayEQ190<`9t-l>+E^Kk0R7dQ0Kc7Ch_8JXe zjVR303$4d@l|-WznF|U2tL(X~G>omdsFN!&Q>i^p>jOd6f zoozRDjFe{2+D432qQZX@ z;yb;htjq&4P5bP!_W^2}T&afedvUh=2=%U=|NV*sX}0}E$o?x@+ZV@3>3JLnv-QEl zip~`Is2~!kje##ka^ebN0O25>McY(9JZtzz~+zx^u0gxFGTOk8y2dZ<# z>#QTLt!;pSR*oy>PL;xtsXB9H~oYkhvc^Lu^aTe^4@$+aFe zyL%hnU_w;ILm;Cv^oDXt11a((3{@IPo3lr-I8Xw6^J$T36%Xz{F(Kg=L!e(54|e=E zV)Cun_n#zKq(4}R?Ix}|Vm(=}czDU2N?g3z%ooA9tfEb}7Mm!5ZTfOUaJWqIi#?jkRlH#1E zkwbi%gK)n@0&qaNxLK*1U*vIPrKM{8>E?aC|KzM4FE~2PktF?bokQh(L`ceM?qeuvT33aDIG7-DDzp;S7sg zM28W+j+dld@Y1W^`sRY!mmHlF6q%wfFfp-0?SkJI*_*+x@N+ay{-e*OAN1E5kTAI# zCwmg>U`#v!V1(+TM0o-)kAMmi?PcgXCGLsu-rcwe9|O1+Q=anlgKH-0+$Sq{V~CUE zC6vSuQE?{7g#hU6dg2?zE6jYP7Uz2uI~>j*jm(`pls1$MEuTdUugH1CBZDG$w~E-F$;#Ftru3wM(E@6ruVo`AWOm}I znf51BHAs|~bCMlJGcdMPMJ`<9%5i~9j68*Dp(RX-hpdS!ar39TQb&2_2d{FhQG(#9 zuoa$*3>9HUg#$Z)k3I(-aity2Jg^Oa>*D&h>|Z$rZNHIQ`x_z*0ZUrbLHUrtHuRb z1O1-oW-Bk+g4aCiDN6o`W^i@sSeS&!a**HU9J#R_1n-;RZe=z`LC(6P_k-4qe^~|* z6CzkH4D`>JKRZBu4UgEJ-X>fC<>KU_DWUhISlIqH2tNv6BCOXIuk!;_-x==M#?6eZ zFQ|60ZLC)ONsr5hZdx2A$d=HdBkI}|;jbS}tdy=XE(49CZlm6rbKtwNFc%Q5`>r~C zhX1fbAxi(`>lwWb6#EJI23(W~Tt5Q3X0Qr9meaFlQf_Xm>^9zSg~(rCIg%)Bbey?J zZues89<|x`7M+(pnAb+DUJPx|Gp8UIA2gbtcl0)b+TLpg4!xkBjZvVr% zGlUN}XgJ(^UpyE+5RbQCQ34jG@tnW2GZ$@Fpz-v`=WF8SOC?feVDmR>E(t9JkzvE> zR52|P;AS)sS+ekfw?6xVj%}(Vj&fu7%i|K;ey2sU8zE#ed}Jrp@RXDVh`ZC^%r{AX zO@w0)W$ng)pA;B=3HUD^9hl(Zy;I&a5&&XqtdV{Lcs$u)2a|B2R{XkA(~d9PrAgL`sd@ICB3g4{s;JQfEY=#3tBgPxqwH}2fB!t7Ck$(vd{Y@uINN#-71bE z1!I{3Lo9qGae8U2yc*cIA++eS{K+wb)m)3I&Kgek2YM{e0H#6e8l`ET3Mro zb5@DHU#u2UV1ntrR%u>Q7qu&bmlynbiv}vCAI5x?_+lmy=k3kvxsM9ejY4;0OeqR~ zM?}J}V)tfxJc$pvz6zm$V% zLOn`a*(l3tey%GcroKpQa$MneoiQO5^G+*gn1I{#E&Jby%=90+;|26< zRkS+%IvgBG5Blwb#`gh2q;F}S<9yP#Ty&%&6yB6zDY+PuReJu|!-{C5foH|v(eE|M zDn(dU!G@_{Oo0ioZ!G{%yJLDsNyOe#oYykQx~OJ>EY``8caO4}>$_5ITWI2M(m-PG zV_%ka*4>DvMhEMgJV)u)CpWqu4F_cnBY&Y}Xf%7Na(LVuuRG;5-#f&XUlsxGVoSMA zIiK`_h3%Sl-Mx-oAy@9r8Q0^*)AoP)VhKFCb8?CR@4;!!#@}R*DKSep|;Y) z_c`bF{%OUoUR_I;?$}LSkXCNCa)ZNHTG=e8Sx@N+T||1^?nElGw0gUK0g(quPOb#8 zeWYn!03{veGK0o=u4>=-9-a5Xyh&b4kwLo#V~OlZV^E|CR==i#N;+L!CQ0Di$cq+W zy`24AeAg&>R(R2Y5;|U0_uYIw{i+XH?{w((U%9%T;K7Bl}Sj4!b0e zvh_Z7e+l}v=1UwEOZXlh7iVm&2XinC2M&1tW zT^EmE*vyZg;!Bgj>>UaSKCml_J3F7zFRSA{kkOjj1SCmSnNDDo%+b#4KifA8Ta&-d zxhiGZ&)@W;NiuDf$Z7_`Jp0BuX;T;i{L1*#WZI>n7Oz>y7lXS|(*h358^MT2r0QX| z9r38By&LQsl80DpU`R4p$Ne$IWu3ej-J4KxP|IFo!n%XiE>x?%633LO>Z6@&#orwD8Xw z=Ibp0t3XkbDV5~2LX5>h0J1o0*! zKar<+yT;huN$J2rMvZ6cve=b7(976>BP{`;Ia;F>zVOZ=vzA-21WFYGXw7(8T@{i)?;uUEa z@%?e%n!0skw`@Z3?~wKx=yqih!iDAmD(B>E6gQrT=19U!HRTu*TJOtVs2S1-&WgRR zp$R|SJLWFTKw=?u;bZi2ld~C*UWIU_6J9vCCc_b`Z|CK8CXc6zOa;WZ;}$J5c(&ki z1dZ0?N`5_-`@G@aPjDPD9z;v;(JS~$NfeSqCq)MfU5D&Jq~YOiJOTRL_z>nQ@kgre z0I@&X&BrPxVoCg7P}f(4U?Z@Zri)Z!Z8n~eLW70QEyPLvm|d-f@@+a&ooMQ^7sC{HF=(_9 z5-o@6!Qy4tMZ{Sfhf44lGevB8J~sYxOT&|GEk_8^4zU;}(Fgn}EqtgA0@?|OC%f?3 z!ImG>ey1vvm7$&=s8^f~eiBmymT_T8o6Y1u8h5ctbFA)~BW(iJMM}kxXY`OO zWklfN7tCZQp?RKrb{fbFP)8IsNjdRNH8vQNh26ne}g82F&gHy47x& zp*UNDuPV22pWELCxrRo$>pIDh9LJeaIv(aXnzU>0mF1+l_)DyU#newo1k^~UL*m;_ z9={g`ggCuk32t{AJh30L&AIf5Xk2hn$!C*OPhW)K4#|T z+&6}=qTih(QVeM>+w7@t=d*SRj5yFNSQ~p;X?#|L5AIl zJ=$G<^1;~`vk*vcG>C}1vu5q_6K8zunUk0GTc-`QlYLK@Gv$y`tf=ZaM=NT?G~PB? ztT@46r)2rhDTf1h14y35OQ8>cR+Z(mvk&vx;6aLobmI_*UM4b$VFA0W#d5N$4<&?m zvyG9(LCPw)Sl$zX?v|b`ZD)}rU;xdYs7P8Q(gk1%3XTax+F0K>A7OlxXcv$k6-|=^ znE*s-s76qmp0Ac#;6isC-5DV>RE2AZ3)SfdtVGvCA)oQqj%TsX3V?&3p$~{c*JLJr zHjbr}iwZ@1D}MkYvRep~GJIglo1CD%A0E*I>DM^a%1JH>W(=o(R_3z^*r%K{WUhcC zp{S(91trU=YS5OUogwik6VTe^IfQ&co=FM*8!<4SjNbDBi>}LEfl~pTdPAbq2hMeh zO<$}#+RxB$^BZ)u&D>of{06^Cd+p$5e0^qtITodT5iv-r5-msW?WB`|lGD-S-M`C(ASE6e7Zu_ja};iV9&5$oplQf?YoC-Kl171gp5qd8@@}Oj z#pmuKpcu|YX6^$|=56ZetGOh6OaQ+JZ!9Ow)h*=LuPk#P)BbJ_`#=B0%=97gd-zpV zo=XxoRE|wXgc$NfU!ZK$7i5{ioCt3XqSMP)3m;Wgx{^rJqPLBik{PRdsM(W7X&0%7#%cM?WaF}Wid1@ol_n}950?_L5&W|7DM_E(`B01Za61Rw!Ug|6eRAGx}6r;Q|BhoGenFSZu`f>({zaikVg-)4mt zmxIIBMG?#vO{trQ3?W{;B&HP6@0Lgn^=FzY_;VcfGyO$rO(21>TZyXt@73BEDc{ui z;+{cAik@iB9{-n|ogVPsybh44a({Q-tT1dLNduaIhdcaw_z5HjNiQRx>{hs)^z4== z2Ea;ye<*0-mxsGMLykC~l7r3QTsOGK4g1`y`LEpHx9x_B}XdAks#*flI2u-UaP5S z2d>M1IK+r(+=sTU6D^7}A+hO{foGPw7`sWy(_7_rMeK{RzfHCK&DA)La=%0C^na5T z963$5>oomy$W*I9b)oC>`($ok1SOGsJmbScoQeUY@no43rM2uF+Oz=Ds;xF3l5W#m z5q}dO4z%e{s(t?6g5@=+lO7q~?Jh0Fis1K!H(`0%q?k~u?cb0{?5F>jZKeTY%flfQ18PKHW|$E#Om+m(Vlh-24>m95cBwi*u~6dMHUruDmy| zRCTx)&4%$t$loI$!(abPUM<9HY8p1!farZF2D*QXbUKn6Bqu=c z?Cuk$+=?lm>;8%69MRRP6}&x+!enU>Z;#&5mPm2ZCY~)G+pJ77a<9mNz%pD%Ut~5Z z{8qWKxp6tx3vJ0P_A1(r5HMpd->e>)Q2#>Z#wVv%B2I5m9xt=6L)oKhyJ5fRX)<)f zJU22DLiaQNq=qs?tIX7jD-@FIvuuMLQPtlQK=Xe8Xl-A=JQh^7J+XlC+vjw%@EO!q(iU2H>Y1(B*N?!8mSfwT~pxw1%k`6J;_) zFe7%XRMeI4WTjN8oNTEEAsf8RVAVPnLeUMf$pd=u=0hn<*VZ4LD`X@&>1f>-vJ%5d z1?1(Ogc~u~u!BG4#lmEsfr+(D(!5NW^YxgEp4T1M@uuBM#Dv~P=1ma?1#yR`;$5Lm z#L!2750l-H`~#oF_7!J82F6lX!NY~O0YVc!yt`!fCS=P@m)IvI1%==VFpUn7E=ZiR zDKqhl%Lgg84_;@^f;ARQOMkRN4&r;DP293sd17m~G^X@*F-R|oB|dktrXY5MhR8Kq zbX;urT;kz}s|rPC7n)rkrH6>~wR^rBf$Gaka!R9lw)l7erX%Bj?2-$MkyLnT;9=ZQ zd)&$m2|Z~0n|B5gtDXW)$Csy&XFax#b9)dm#@)DKL#g!y<8I?st2MOC^A;NW*bHW| zZzpdugo-q6oa7~YOhoA_&3bT!EC`CoGU&u5&@WK@S@WRO-V06hEdSH<2cxF9#!)G2 zrn`B|44M`mSlbVLSIdnE^(a7{UfebTkTlxaEZI0F#etT-STxYB8LvdkY0BPHZ5+m= zUlL0qDh$vya=td-lp^AN1=U z%oUGESfuvCS1D{7t4o!z-bHv0h8+JY+?$M?N zU&z`mr5?3sA}O5hCU+LF*ZpQ zj|ld}f6-!I-==D+hM>VJ6+W@7ouOKx9H=2vmWNbTJ5U|YUq0o%koW`rpaRI6LT zm%ItDjp8O|?3nT}noOOb!asKYv(rGec&H)%~Ts z?UNPYVPj6<6Y$%q;aY299LjG%mNnevWCc&B@_Q&1fo{W>@-cDmF|;{1k38J%Tt+*1 za7ZP?D=afuF|A?!yzmb=WyyFoNUb-KiZ11A`A$xL6zRlDVm*yGch~Xb0sp(A75+QH znG7$FCl%h;P=Li{CLIq(%rwHgzZc}vEL%q3^F_IZq?(H6a}1zSpeRr@kXfpdN#9Mg z>AGC{+joR?5RD`}#i}R_?_d(ry@olsMXc4voD3wBdB_$m1$RWyRa>knB)O^S#su5# zusVd!>vqDBYcB0U#3$@6uJO_ETFgj?u+&DgD=svOfk{eV;wwCHQuHKl40~;cB)=4pXJ6-34?z`=(hm8JbzB75dXF}U@$L@Xx@`MilA@K2!t-;^jlCO9eD$B zP8)3Ds%0}~XSr}_%f65?Jybat@ZtzzwRTsiDC-rTtEh|lqgc(P9Y~0Zy%Ei5a;<&? zS;eNL@TkknRme4X(7{SQ78Ny7{Ep7bdw!j^j`})?n~5sh%4Ya4zs$;mtVUAzf21-} zL7~!v7e9muqOvrd(5E{esfs&Q7G0NoG;Cy?WTqh(kc3VQV~;QTsxE6+)QsvDuJe$Q z&Rb{mP4loJyTKMLYL^V5I8TC*J%SyJs|ZIP1HC`C%-+0V77lI`+2@krU9ZD3|4_`p z&UUBq4`vO|ersy`d!W*5jiUdnp>K3cUcV6b2JrrDzQNG**KCU( zxI)nrgwF?>vLPPal#(+1ab5oA2DxVN-HI_`gp@Whzt`ScZr|j`_%O9IJgIFG$e^%x z=;XvFx8sl^^AN+zk=XQ)vmN-VP^GYB@OnT<`F=(Inzq~!Uyul*@UbzD5nxdH^= zJT=t8J+G}Q&AB#ffh9F?XS|SD+-2(Nc&g!HB%$I2`;Q6HvCei8pL(r+$zNw7FLvLI zZ}phZDjU@~FU}&`+tUOY&&9gMp}&3^3>SQ`vqhJSwLVU5^^%oE2y)<81?#U;Jv`_f z?zY%4pEUyN#w||FEHX6bgp^Y^o5}Jy{w~hRz`ef(aZ>L0q-s*tP6hNK27oD2p}sYh zy1IW<#0E=&XRAhA)WY#dc^l2-oN%3kCJ96uX-+=!J*53Tw0eU!SnysHOos1|h-1#@ zuGqA>rl*8nlu`*-WlXH{T93If%_z)?539xAg)>J0O$RPmFM!r%^LI8U5|oD$(;%xlO9YhrD*D=r7#p+@NX;oD(*e>Rm#pe+0V@&*o2%<9y0e}pP0aJx{cT?B{Y))b=KzVYjd7HXAb?VI zs@#0p=S&GWFmgYmb9bvEuC{x!KQ)B^8_;-bilzf<8NrTvBlRsr1b*yN>X<2bNb$V7Fy z=?z<))~n;3uRmLY^oq09N(XPs2F=L~@DHe#Ka^mGT9xLe6AWq^*Wb>JqE$r@gu`{C zMVY^i@pqdl|CCbJ#$Boor_ILc^Zsc7XJg16B0*H6cx)rtKA&}R8~)zMQ?UA%3sHW> z!@BM9%XUTJFHq`q4&MnMBND2zHTWHK&!mnmj%k)KQe|t&bMosE9V=B1&eW#aIx*c$ zz&IJwEUk{fIHp@cqF>?<{Wm}gZri?Z(CYI|!H<@*vfcTp8SCB{vTxh@bcWwzS|R2I zwJ%fgj}42<7~DHUA#|c5ZB|LCM^XcDg-$Vs^DyEwg;oDh1|ki%WGC6ImJF%&?3I*8 zJ!twC=HmfWILgzQbrHJIdO^V%LeE}Dp_-y2!P|{_UPb!U$W$Xc{>iYk4j-#+;dX%IAW)O++uVO3sVJy3pg}g6ekc1r1hSpou8ix zch~pmVD@q$XGkngV!CRPT9hTs*`nFV%|DM#VP*^_t$6JyJR*77;~}X@OEdcP*&%%N zrwl-BN`ym~qQtz^ZXm67v^m4J`WLrvksY$C_{L*MHyTUZ2|J^JF`LKHfmX9W7T4Yi z0%im7$Q#s>7V;F)qnWe8X=OkmPqE(!%J?9+8v;V>^$(vR%<6>vA&kt60+Gw)bnqer z=@b`>(;XGts9_g;b1rZmB^}`FTlVIoczypJB(gBU=vVZqbcu~qVZ>oAu9K|un(55p zGpASu1GMt&M3I{Y%v}w8NYs((Fe_rz6`}tyFafdYYv+;dfEYQd5-BEs6UUA?@Vect z6}DUZK9`EzH8kpY6|0}Ujbzg9J;nAhC)5)CKC+H8Fd+^HC8^C5P+CNvk-pM!(Ag}v z^7`Fl82Fx~6hXDoN`o|mvU4O5@d;9!o%2M+D2mQdFjOQQU}hSv*r>8Tri%N+FIUvA zQ83JYS}w3$tFNa(Dt0f;h;ywyr1VMGJqH>PGP}b(EPOCRdiP0uihsBpM_Zs4?<__Z z>Vjcy-1l?e%9c4)5Q$CIZfowhf(!CArhBT{ z{Re;O{Mh38VdZST4X$Ajee#)c=hi5)1YG=~-66QxMbFeuy^u@f;TCE|TfIb|q0Q`o zxNypX2@_`r_wZwOQOrq!VcDIa9>04i5E0;e=+TRR7;{Cm0^pOnW|tAQLKA`yRwoA; zq3l;umm>bY+yBdAS%@O&l7>2}hpmf73QbHF@2EzOh)oQOw&C|=_{`+B8@16tiM%=5 zE%*+2YLW5gIHm;>$xV#rZj zn@)C7)6BdpB}|3Vu+8Er&!?rtSAJcNl@nqJ-`);c(hr@BGYHS&M`iYaZI%-G3Cm6fK4cV+yFJdrUVp1c~a zq2HfB1%xGQ<|h!ig-Xr;+cUg=a>xG`mk6ooS^O&e5C_A;!Y01CQ#O{F@uz!d(D2e5 z?Us0h1QV0{RXE}f5tr)W18m>~Y=xX@Mmp;}!5{{429+RZe{hV{ZHnH)MOZFvGpY{a z%V7T|YSNwR7GVX-yl-{)TsI6t2)8@Cu}4ZsUMT`bZVT*G+cxc!5QxC26*+-1rxKLdd|D7R;{s^JKd9X zeiJJfyjcmvcr0_e;nKRJ&5TqVQ^JXVwJC{ndGG*v;GQS&{$VG)@NkE0Nk4kTj_&x1 z09DU5p}x?l(5PV-85*}FQ=-L*Y$c#qXuz9DSn?5)4m%VnLmFHw^yb%IIONOrET06P zb`qv>0$8E-S?CLu0I_G#4R>WOq{7zCIKJ+GAIS0l4@?;Blo1C77gT=u0*ym&E}z&d zE}9=zj%P#cL~0Ywifu+QOy=hLXuNl5vE7bd2_y%Vb~Y*;VQ8yQc>|stt%RHg%1L<7 zk7@~P!H0ci!N~DGoeb!?@~o1(aG^VS5$l!77tvM;5thAYu}>3M$89hixU(K+xg-q6 z3v-}EZN~m(Z{16ye}|jZX!&>S3JD@cik*Ka$;P23onOu+KHhFOlLgpqQ5`?}5ikKgxy z|CjW%ss2RGN1+;P4F_UP`|HQi7~J&KBa!$) z1|6PLHGjvEzG#I10uWkyQk`kFslPv#Jd!9fC+J~O{rcHKKGY?U+o|3H^NX*quV59# zj4oLT^Vfw_wIv?l!US^I8?f&T-dkNbq0!Sf#lm)+|K(?2vy1Eds*P1TS`>V&jN6SSP*kN3A*`S|NDo`>2yeD3O!ays2>vE&*T+jTHf#fS^p-54kn>o!D-^ z;8tZ&UIZn=jH&WIW?Y&>zS3G*j&EiYnmd0M)i)l_yZ%}prCG3D25f%$8>XJE`wbL`J0ohR>25Ka_Frfrhih<*RNj%eqN; z>D}nrBzP->MQRH(Ql-GO{uyCw-2+e|nfUQF*l>2v?o36F=Iazsvtt#DeH0HB_n4_)8cD zdlPJ)LcNu(aws}m{6=v|;wm#Su^`LgKkT)2RzdCN-q>~?VL=c^;4d{+lC9T$x5^z$ zk@`i8htKdNz)7?2?Ps?t|1Y^Sf*f^_>bd!IlLfdx#btzPR;t`?sN+e`i(iNT9O zL0_U9gI@?5VYFO3%w;s@Ti|F#Ky39+coc#;mvSSbT!1_{(kc8{psRot)jtH#q<}JU z?7lq3`Bl}j@y<9@Sh{Sn)RTq1hZ-mfc*WBW(Otd|hVi!v7^F$Jh7$-?x*tqW%@Y zeSSNK7>h5vJRi#r*CgXRjZ>$_ZI$K#uQoUHCy-vH-uE)0GXTaWryWm1Rt*2lHwjGv z*Nxs_Awr0xg)_)pI72=EkXt1m(>A25tG%`Hx*_cS>hy~`-kGTJxSg$DXTF6)j(dXC z9=rh|LSIh5R1`M8R686$_JHV^pub5&H#{meH@asu4-w0@;v?5+rJ9l&MAdZ)uvJt{ zzT0Vkk@3k}m4(w974B&)CnbyCe&t^sCF5s z$pR-{d+u#z_!qh7WdHYKGWm7RO3Vr>(Q>TXpnSc*4uqQ?qezAuJXZqytnZ-&5TC|H zWc>-t1vg;Y>EvD3yeyi1jo`ppFsGn*~4H? zKn?1e4nu^A0hs0JEe(qj^O8tv_Hi}o2*&6{o_s>3c0P2s7QQK2Hw8HF{^>Og(B*!6 zd`_WXCfAvs7iW3FAmEHU{UR#&;&IVLEH+CA@p6qd@?&iqifq~{M2@!I){HbHZd5GL80BsEV=+4 zVP@qQ=H>{s5uISxlt{8D9ExhyZ%;-hKR5wV?f^5rG84;|ncH*GvKiER3aU@DYYkp| zAOJb%gi96|!A!*>!v=@eWRDrLmPqi$_n6nuo)dc0I@X%{yup0VS%S&%;rtwnDH;JxrI+YupyKZy710^T{)X38AgrYOtNOZCBEDJy|klnYBaFQbg~VKRSnNgwJm|a z^iJil?YJWr4DmFFm`t{nM@GrQg+d-EphXS%WfHNSA2lz-75gAg z2Qp>$o?NL&G*ML0Gve#xnZUF3zxzGfjYv7u^}PGVKPo zJQbudiUJb6qm)8)!Pr3x7Vp{`5h9_Y( zT_NrK2xatbzZsNOc6D%vcK%z?`a=AZfQ^d|5NuhWBziI##hS?|XkjBReIqPE5f0-W zRfs+?D}$EvI6!gXyVf8hq<$sZYg7y}Kk5@F547Tu+$^b*2`lyMf4kylYd? z!)|{g-LNktQPjsQ>kaoE&c$gr-(9E9A9Uq|b-UlLET-6QkD5Z>y4NT#;A4zn6&^+w z|5mt8iX~%V4ZNQBCEvilc{w%bB34z6F#`Nb)Dm#@RpePZREGBdx%XxLQ;D3)Bs{`=YwFHj^ zW`UKZ8N$GE-xdHA4PV~opvzF%R1JYE%B&cB0C=Ne=$LhqlJo*c$Yt0LG}=>L(!u<1 zVUpB1$Et`X09Ftdwp`Q6WP5h%L8PyNepIEQh&<8E-_$foQf5tfSg%X#KSh~YMJnOA zdf-0}6^hWaWjSJ>baMs!v;Y@(<-MZCAZ9~fbf?6c+(E+d*g+Y(UWjEcErk^RfZ$4i z`y^0*4pNp!QW-149YW<{yo8?cjlg_o)^k(WDt@3!2BybdQ{L7)qo|Au^qLoh<89Sr z@71Z$-1rF8R{}+QxVaL*v159sgqu6kO9F3PcSU8J_k_JY#or9M^8No{EbDmyV-A?1 zWt_MYp1i3AOma+Khw<@<^R6hPI!n&*7{`f+@_PK&FgF6snS_-Xn;z0V)?s=&^cjMa zk=IrP4gAdWULpFxtY0&Rmaavo-Y-({piMdSGb3E+vpupnLBJ*;k9Y)7L`jPvR2~35 z9B%@16vWik3UT->BcIVmU~VK=0{#6oZsu#y+1CY-kP>2jw~Jy<^G(~&ondeDiFG-f zRC29as|ERxu!S9EvB*`_0H8~#dg-WY?SFNd7@d%^#WVR_eogWq8dgkY!`7Wz(Kj5W@?mC=8c4v&xt`R&^OI`alx4TLoFXKH$DoBlL{kfH0zjrImf} zRVB#wM*Ku?jjGne+TsOfd4ehhd&Iv zPa^t2-td3zQeD`wUb_@z!~c zha`yWO+N^nA6GA8-8#*%egIA(vVOIDXaKo2Nk5ao_`I6cyXL* zE8UH3kVlu#vEDA+<|+^!96v;qkJKfP;#V!Rk~9)N3M>jkiF8n%(HcM%HLXE*9o=sy z_D&vE)E;f!c?kP`)PetFVKs2r4RMY$MPo~B08=hkL64Z!jBID=$=L3|KyKxqu^Bw) zf8+9Na@@wOn#UDG%O9K{kEc^E(4i+Dm~_g&mpdtqD@spu&USyYC=9ff=7i5x`{KR* zP{tk7E+86$DTHbwM757X%7`$1SnxTu7w!#}DTX!fWe@{ZV`R$c#PZwBm1)2eBP!t&v8M2yC^ z&s=>`u(%X8mx3GQ{&SaE*5>;_dzD@<12PyB@01LJj2-!!v4&Ylc&(~y7nKP~_nI^i z>6Ipn__SORyUY`uR>s9JU!HQx1z}r(V>5R2L634WBX$}AF|pN%lA}>g4@<>=-ib$e zxwbdMZs$)Z*WbAs*7?_e<$UFP*7Y-7=~k!HZwp7%sZ|}5SPfLHaFb8c_sr5#O)sVd z$3d#X%t1dptPdVKnrPjZ4}d}*Ay-+FQSLQDZ%mUplvg^$;0x8# zHY$-wJEFhRwQ@j4c8naXMVcjZ=Ck!PhwSC$+wy<$QW=z<{Uj4s0)5c}3aoEOvxOe@on-f6l^zq#BrNTyccwTtqPjQ# zU`AfpS@UXZB@c%8Sjr+FDm#)m@4sY3q5phRGF!Od5pR|GJH{OT_z$sen0h`0^_BZ` z7gY6Uh6Aw+8WMUx*77@jVM~p_ZK9tNJPAd(n!vBv><(;!BldN0uljTYS|)*hCKzyp zxl#o{Hp_fFDrXtCSH=wTz3~r^%z5v9cTPDeLL^Jxm(XN1z`l33oRMh+X?MAub!rGb z#T?u>E zBq>!pnJyv^9t(t85ABnAC8#5V!hUt$7vtZCvk(B2Q6d*$P!Xx#xlB@4z_NuZ>oLO_z&eZS|Bzf_yCEFcjqOi1%4ucw zCh?E{T+@X~y?A?-@0s$5kHmbOa6)B{9>deQ)NDk;ow4u4`M04&#p(ZuN}=mHp`aDD zu=w->qsGLWijV^HXP zh1lP0$Pmqa{T&M5@3+)=4h!6;wh8+v+Y#n)GAF>APSC(#pq$NM--WaKc&9v6Mkkka zrMX~UGzWaSL^b);!5M^ocj3fBYzUb&x?e{Rkt>FneouQ?=xAit(X!PiJO zPOfm!fj*4Y_=^&1Up-xt3^*aI+8C^waeQ{m;ziUV7jrT!`61XQep$ybCEyFvAd~5s zplnT$;DHFKv3xT=**ovt4q2`*z}1kI7eZuy_|SD~hqHGN%LD8ro(x2=rp9|ZIP)Z=2;UNKxiiX$ zTXFj#aWD!PuWipQtcU)8_5Mrfe}iVtbsbN;_U504i(Jfd(V8Bi*u7<|$sWy7B}-&V z{4|v#Xo5BO^CW^O=)&i`tzj!nqk38GpeL&}VhkZ!_QY8b5mulx0}x?=I22QGsF0#$ zB+@!z^_LuTyNUhpfX(i4_mY}kMSEm#>5@k~V*O!};WjbmV)j7`V$c5NtQEG!x5^%S zi}qO{p4?A7sWbJtGXQPMI>||A!pHb1_sF0se1yxRVyHAO7ZBo0^JfCu`nZN)_j?rK zw7>qU>o|GBdH-bcx%I6VyC?LMsO< zylki_M)~d)J`R7x)4D?(!lWJZj0%b3KJn8f|su6Ks^?|lYdjQ zamrV-U`9024AV%oI1!;&ohM9n)!i5{8E z89PGGWPRM$%xD2zOn(;%x``^vBBo20B%e-?4?WjWTOcM+pExk*5<>WjrJUgc!SszK zfm23gdRhy^ic83?!|VOi!Quic9%ah~L98tr8PWw>VV9aIi^}?h6f%%}R2?_M3BoCA znYE7M2&B_i%a3G~cw8kh32I?X)L81Gc18)+q5d$|^pA>?NHg_9F(#IrsF@n8Q+?5{ zc=?K?BjJoc?4vk!Jm!^)dT19WV7)co85bv3=Leejan-ZO_Vt2?^}YDpC4>830vg3m zpc#e9;k^9a1Ds(vGbb)ab#{ByT`u={$~v|RkId6XHo~VaqZ3DcM8r@hoKoTww$%O= zPam`2B^9VDr8q}2RFDzs#wzTiQhQoGB>vO!UlRcGe=@Iq?8j5Pv${QI0d35P2m0)) zDn1A!xsk{b&NNuI-^0CRSQo7oz)msj-p2I)E07;MP%k_ZDD03s^$GU`44>m5N4VjXEa27nQXklPG&i zR51){d_mu(dG=g%Gfvq6n#kV4?gd*o3A?O{U#?%;ZI*HM2 zu>a=wxb*5cbED!gvQlbv*jRRadLmM+9U!w)L<)7{8vx#YSlC&Gb=Gp(KjrVs66)qs z$k>D;RGcyfnfxkml=q@cW+pNkLXwR~Dqc2)>t^#WSOxwIR+B?t|CNOYgYJ#%bZa&* zvYdg&CRP=^k@=aL{88LSL&4aXk1w!lUzbm<+AT}Dbfd-e3lYbZ%!9K$C)1el8V<*M z7pr)TIFd!N1g*iC^sud#2W#{5&Dq0{*_c;QPvw-=^L0t!WRzsn`^K#hrDLo06osP^ zk%XDJ&{YnWd#9`Qu_8~=_M^xT*`d*`rppm=T-u^EmAK&$Ap~`k1Hqv&%&!rXEjj6F zr$TfbcF-aPd&Fef#7XmliHx-wDtiiU@Lhc+aqDa~B5-Dfh(E!G@cJISeYa_NG`z}m zu%c6&#HMV_OYHlI)16EY1xYBGmzULDA5`6MDHbM-(bvbPbebI>5O2@1kdDu7GL9o! zrFkaO6rV)-LhXtOC-k9D^-(@7uB+|a+hF%j*DHZUep4_fus1Teo3f&WY=!O+Ts`+& zNQ@`~GW*v^k_}^>;#`iOo&Vk_;eQV9=CZUO%d58<_LXlSOeNb|lSz~CPPWm|Owh5$ zM5a(iCK?#*-&bNiol7H}Ky+l=&-8Glc$Q&V>*5A{vME!g)7Cd*Cn`94>}?>*%XP&3 zsLRP`+xJa0DMHhI0W?d~-syf^vqDE1Df<*dp&djM5cxMKI7SmgIv>w5nr^ER!f}*n zGL)!%9@XO+hyf)9uc2h4dZGgm#=)f0qD;L%R&N%k7v_R`^06TxXpS$>`WEfh>YObw zn(fyjviUZa8(kq^+Py)nyJ^UeZw$EDxrj4h9Hgr$kMit-0WpfSBrK%CYY^4*{>U@E zs7}!)i_fT`JS1v@(NV>yI8^$i{4sZ`-#P=p&|vC{5hSA{b~M>>r@q3lzw}`u@OMI2 zEWP9Ux8L*Y`h6&8dB0p++Sv_ER@2W1LJ46SUpB(8zx_&lIwAOooQ~MxMBmkxBb(#4 ziVfA8ExgrAF%2@W*}Ycr zBQuQL;iQG6-(3yjl;+dZcKa=**i+9a3wo&&s)^0DKs(Pd9TD?QW=VhUs4@Kd$;E;Q z(dB}zD1jXfr)Murltt<_??Vu9Rj7TdRIxBUr_sZ*=5BZ~IHB3$iIbz>35~`<*RBJy)tt&k1$-=eB)=Kg8 z!O{U+3_SvnTmXjW32(-Q&?Y5(olUFas4LN`4Pni;{`xfMVZ5Nd?r?k2dRWllusA|N zcn8Yq77e%N{rlZuGq}s5Ie&VEda_};B0Ly{+mb5BRF`~2fo^&Dky$WFIv=sW`F(&% zpZwonCH#NcN+}fRdgx8Yp%#cWY1~6J-pooj#^*v0&LcW3hTm=n&RZ`xH^}&vwui=R z0`i3{fs*01w8<1$6upD2WVUJf>D)yscART5{oz59E@4K&QKrT^-8%RCH&W2lzVSFp z0%mftsJy^_ks1tu=H91wN?+Mns>x}Q8!Rk5AuP~zD5XkPEq5(6+f7bRqH!@<7|Kq( z@@mzRI_V1yK^(II6oE)GHUteCESwsnW|{GOiJiy!OlhrsAB%_2f%f|2y!5$dHx}?B z0Sg1Kin5V4iW!}z;P)aT;EIHJDIA%`#VoQ*-Q3Vhy*A$8vQ*q@VxDVEt`KU^_3&mCF);~+4JcO^%q;J{dqLS!_Dv5 zSN*3<9RV*jbma4*2-XCh1gP9@**&@3x?P^4;2%Tq3XK}bNds!~0!LZdTJ1ci$`BQo3=!w-|F5}f$c{>Icza<>)H&cbT#v~*Yoa6}5{E+-;#8=5W~ zp`=HnK`lc3CwTlPVcqyWFO#`@RVd`8FXowgO`rSsw2&AbcVgnO=)G_dwNbsmnw7f7oXb#D80T?i?)Gs)IibI|*tyG+RSNhqH`Y4o~O3 zX&MD1ZBX8ghJaco#ewpsvjL6W9!@#*0b5>IEz<)?XD6zUBzdTQN_1XaE_ZfAPimkt zf1d*1S98N-#==Et^zzIw>y=eY0deB=%*174?nFsPk}5u>om7|K?S+(b!$BbOI*^n4 zb*R;6${~=pOK5wX(zrAB;YMz}EhOAg#B}+H@3Kj^B2hSZ0>~u9BqAc*0=A7we6rR= zr4kgeT8tP%wXhqVzQ?C$edTI6j#Iu0+X^ioa4mdyP;YEHqFgZ>>##M&(ZKy)7!yq) zLK(JU8KTvE?D8mjt{I_x>Fv?nseS5WQ%y2^Gjw)kqv9wMbRi%sIe2k`+swjJTy+=$ zB=AhOM}+bcj*3$&w1blB6Kbt-C!-p`R5}9|MkBm1M|TD-tNt73eAm|cmPkENRo(TC&@(JbTO!Q?r;Zt^hZtlB!#r&NIO5atw9TARc3U>NV`lr z0+4cjc8g&m*E#$B`7N}R~!W@^wTn~9Y1e_;H7u7%Nt zw}O`0x6g2?RH@=LI??g4oBQzs zRMVs$nNTdsgTb6QC1*rZ>W2kp*@oGKc?8Mmf?3@~3(QW>PgMHP=J(L|Rx76Z9xmzE z{jc_(4!-}1#<}YHwsE!?;rdew&p{^x@$&%C=zjR{g&?HZQCNSns^M^Gg*pEMq`HA@ z5~tgI{YXfH8H4ds2#y#x>pmdO22&ng&nrPF6=`7P1nKo6CF~84$B%Q>=AU{167;_& z*5fMI-No-bRL`JXz0iiK&XOAKry6LJp-Zngvt3f6bH?yoWp_;}#;38Tr8PAXtnY&H z9H9XX@nM;kSOEDFtC*cNwaH9ipTk(F>t{6}f;TycmjsXgi3#w`4J8LMO5PWw*Or;? zi(^)El8q!H?v&(EQsmQyvC64@I`niT>z6Fp2938+-(k6F1OmO5yg z!hx4`i28j`+9XwDyt$PHp-65&AN)b0N8qG4b1lln4b=Ni2}b@cXV3U$-c2@lF?KID zE9~2p04PqgU7xSJKs3&W2aA2{o;|GHGPRSsbRtFmvM%~jnlA2s9^xeQC*z`?)E8#| zN=hx9I-Ti(UW!&Km}Im|vKq9F-K7g?dyo-Buy6&N8i&Q3$%Rz!GSEEVxr0tERD3f~ z>#z41P7(k2A;L5N3*+=(wrAH>2L|gt3v-#FT_KouX+Ln8+!q%m)T;~_U)VlR9B`u9 zB5ZVc{XSN)aXd`eV^l5s6{OKEJc<}X^oA#&tII5?71Y_6--WI3VZ8EDw|Jwf+~#SGdi!>Ap?)Sg^++7 zmp(^sba*Gl>9_-E-K}@7b(=$9ziolMt`Jxc%)InQOuo-lw6+n-ufhvZ0rNT8o!-z5 z_9LR#_wr0Qv(Acd?2dG7*>@HTkqHWvZsHhdb$Q`e8I|g@v89y!bn{RqcG}T&b7wsE zH&UhecWMfr_`e^Q_wyEkhvLs8ftz2Ga#T(b^mv2>LEA0TX*iR%%V5wbivuqPsNWCQ z%x067GHx593smLqb^t&KkFRkXQ4A=?l#v0W*}Tu`^ZExQtRJ2!%YMnqt!;Rro)jAE z#<#u3ibvsHPJccALfg3sS^a%2@QB(-#4QxCH>zGq9IA||hTDHY{Baqwl()~-3l6v% z2GoU-h5=ml(~fLd3kEF3V#Lp;iJ+lJYGWj?MPLYBvZ;mSTsrfwj>gj>+gM7CS4M_* zUXJu4lNG842tIru{ccJ#ukTPQK&&Nr%0SiI2)WBe18OFbxI|o*Kb0`B8gkUa^G9}2 zN;F8Vcdk-3ZXbGHqpp^~>!wCW;MKn0@&I#hb#><6CWU(Jvncc5w&p(wEe(3`r8>j) z5`&%J{eq?GJBc_8Ij033m)~!m?vLAJ4M|u?R8;J!#o~?5zby9BfTNt6c9#TD!qjv{ ziH8%uCb#18FsHiU!Z_Zqbx@8)i`l56R{Z^`W{hQ)0CUd$4IFNL;Sbr67?()fa~b`4 zOU1ffPE=THPIXal6~1z~F=r&#$L)DCik)*7`J`e@Y2J9RxE%?`M~=D6{!uP!Q>jpT zgPCp%`iM*ZL?qiAq4XH0#}QPtXkCRJ+5BigcyKJEc#}F>PvLx*+vIrJuIpeK#ddN{ zD};aNv%b$+s{b#8t1g1&=5G^tbicL7ewT@UqDIQ_*lkE^UzOeK;T}VuM7BWNlHiHx zvGAhgNaDC6+u#6U39)3=x;-NMuk%_5b0oQR{~B)2e`AdK*XtA1|DRaCHtYq_f_F4lr#CXyjV5JNl0?*Vp zR_+7xj;?va^!~|l;En%_LlEA^3Uj^Z?#$|q)}|NNA9~p2RdjOBgNrw8VzSXAJ%|I# z?Vg!wwB9+H&L;D65_8+b(LuroB*93ZXQ>?L-}-7j7qn5$TZkVj(kZS`WvhERwQBs-_nba`bT6ej#aU|9Z_}V5sA^ z^;sJVi_cG~cSbl|oM(I5bl|$$hB{&e!q8-#zZ#U%KB$ zORonxy5Q*&OA6iJS%43BeKcrBD&O>GdP3VXrx%j&@jZW((U73AjW;mMQby9#F|SiS zt0b&>1z#pC*dE3UgEOIVm0ZZnjswc#nMKTpA;N0)7Yk1)Q_w_-6FC-|(^r}yrYz(W;}{z4JEbE%MB!yg5vhSEpe+cY2(Qt)-o$t#{T-E!=;RRZKYAXEQ2ktqp`#jQ`M0X>Ry=b{@?-#EZuR5oD z?q^5bh{Y(K&BZDd!rd3I6u^hD>Ji1MIobx3!Q+Zi$peMn(b=tMwc zJ#LQlwl*g4B{g(ad85g>g$ET4ab-oLEY4fpLpOt5oA+)bC)WOFWMV-Rp}zI4JLmTf zUi#}cIXh!IB_|O9IY%jj#PX{`<+6gpDrk$WM$GU2b2Qf#-+hp;Jf{Rq1}n8NmmA@l z0=}S*k#U9;pysb$wj`I;T)mYA2=>OKWlK)Z{S8(fhn}wWPKoi5uORifrT;TC8Xs5swdUeP7&j2Nc zEKXcgjRegkM9oiad!5(4_Q`xWEK}jmXo62qSPt#nG#;KEr}_2QU_M_5Tpbu_DXeh^ zGN^JGAs*7CJfikg%@oR1MSMU%UxI2WFb~k!FCOtVl#!$XhZ2%B7!AC(kGhgk5O?0t zn|<>IfkFBqeEl2iN=?Ov`_&Ht%>WJhJ}R_rO_%Rt zr`n@uY4#a@3OobHaRcurm$l1OTC2oh@C}{S+()Q^)@|_8cytWz88Ghrjn?md2U;>8 zJ4w#fRx5?KV}Ie2SLlDkJUo*AZIm6$~c6&KVg(#qfFYgTv=90Bsrf`m6-?+=bi{-@Jq&Pg(n5_`eO*I5>(NKq|pjxl> zt@oS5dO_cQ(Pr~NbN|Em#r#4eCY%zsX@FvygS+CofT%44o1CX0FVyCha!+p5gN}Rk+f+xWea(kG0&BzYt(* zFD9M5&+57~o@U?Us{Lj&T~{IHJC4ODSb3HHMRWWwJ*CeCm~*h+4S_^0`&E>TuW-g;T|brQFH` z_L)iV7!Q@GiH2;es8iy;YLXsL_bvTvp zwA(6n>v(T=w2RUGj?#5Khg0kOPM*ZGs(5Z0MDSHTX0`t-fW1?(Es)6J+fR?P&G%TB=*f zd|f`p-zaaWW>UehgN)~|mtTo{Q#lB(z{5L(Z&0kpxFLd)`6Yt$Yf)ni9Z(eb zzF=>^!Uz;CEm>a(75ey;QXQ2rnz+Z+m@jVV~B z8k`v%UH*k?Nkko!EZbQ0IkNrHg+0n;_joL05yVOZ;C3on2S@EyV1o%ERHo=;!768+ zqX8YZ>9UsXE-=q~TNFpzx&6%I=574X#8lwlCCBlG_)K=#RhG>^cQk%JHd7gd;Zr=* zug;VXA14%Q?k%#r5t5^9#7ZebTqvsavkOaMuLR!8vh-q9qn1CR%;`&^Vv??;2rccb zdedd(MHv}?VAjXjHnT)&?G}(PN>N49izuzYvVT^gU4d7v4heINgMMA!MacV0!fP^W z509(NKl+?W za6(@k_MUI$!rQgBkgB|9QEu&|rlL|U*kvk`NV4BBq{1zUqs<23c0L$mW!&$erGSwn zsF|abE{*Q6gVz;T?XNvN(ku!un_8(|Pe5^v|Ln;A)2H!{wSElg9O=H5!Et0vd5K-wFkX#a5>g)6fsUqXwiE*@AuX z85@|6<=4H}NocXyfz%S6Kp$!yvNVLvI2_Q4`x&^%^(cU_#+7WTKPjyk+hJZ9Rr;ITc7!~eA z@L%7)cfSVcXv6Eg%&^e=t_0TQ*!&&ARx> zMPf|QCv)vpI*06;s$P|LSMy3QS<7_a=e^F?7bwWf+eHhI((}gmS<*v@IwA!3t+Slo zgk`h&aGJ^yIEy+lFL>aLL_5Aj^Ou*Ih%nUQ42hpoPe0>mGXP8gW4~sqG5Xn1ht?lW#nes8}0!*%6AL%jXSuFk9&)PAoR^=&|<#je;VdOYt^S|%-55{%> zCTP9ub!Kd}!+>b973zsqq~+{w5+W0x=aWb&06I*Qx{D->lkJV!2%1SV9oU%Gb175F zlb0<+(Lplly~l$!K@FrZXOz0Q!S;GRZ(aJn4c-`8_pOc`a6OAN8KB5_>ubN zakd^M&~=$&;f9~_B|o#}s-K0YqqegGhsWD%W@9OCGeGDL0WN~QTy0ANad2?1w~!@ie6yYj4>WPOnxghVkjdv=5)< zK8PF)3TpLKTHFuUcR=(w=mVR(0WYW0w9PnvZJ{CMSCZ(k>NnC9@(cMMZrkqrhXgJ6 z4dW-7xiPHDnJ?{?g`A_`zMkRXhcDBqnKh0(Wa$RS6uLzrZs?1gRL zf<{&hgYW6a)2fBQ4>Xz7JohpAJN^cg>a$)twEcIHVx>BSuzE$}6)dq-nh1N|-H#a# zg-NU<-xkPA>Qn zP{&aAn8(cCRbv*!3|qCfWi1tPIJ^m&?YJb}{p2MCqAuRb=XW$yS~HmCml_uFs??EC?L zyQzl9eR!h`3nEvF}j%hUIKg8tGy#C<#r`Rb1$H5I-wVVw6%)Nk?;y!RaehDsy1B)6${4}x)lq%83dXO*2gbG+u}z1VoLj(+UI%52tz3X&c;x3q$fvF)$JhN zKTqHRG=k{9Z9R-EQ4{Ej(_lLM1L%ofOG0LOfv@;FRM}|6!-5mFx9#Hk^Nfd2#=H0; zE+u)@Sf#l>xa1RAu>l`KoZy+%1_^dJZgHl8xfwD^g``R8xK7$N?PkX!qpku{k4p4{ z3MWle!Mb0HKRGjOzq-2)2$G9#vLLdUe^m$uswRdL>qX7b= z5^>nC#RAN_j8YVdFVNkN)?piYY#DgWt|3K*5k#_Oa3dFE$OJDy<+5H^T-r1aoNtGE zL1q@&kp}M%S536=)E0Ggu&6}s8hyS_-0t5^G?xw=;s%EKALeU zgFy=m{C>Aq3nw&^YZt1Fyy6i;Y&XL6GD2+|qSH3y(oeM1p(e5bgGTLJKpKv)BsJg8 z_ffVV(!5_rSzcI(aw?-djL50G+NEmCs5QUAy(?1p71vW6z7}#GJDKCDJC7!7-LdEv z@;o&Kn}=8DW8>h38j(zwRzb6u#=<-y<+_rzWTe3mnZ=*+D@UOFy!hRz>3IEV%5>u_ z>KUJA8#OUPI6kf!<0Oa!FqL5i>RKh00kjmju|L%UIyU{%@zD#STvovli7CdcZuX?7 zVRbT|Z07qDZ2G^FGL*kWa~E3HMRZlG+8Eb{NQ@6;8r%|R`Cno^ZCu%-e;q0x!Ur7{ zL*=0use8+?l#{K}Jvim|7t3{3t1f^Q@|qhVlcY}Ge?}K@EkomyF(it%>kmLy=EWJ9 zOo?q}zPpjrY2n?n)_VH2KRG#t=Q|#l`SH?m1!WcccLDK${%@6oE2?|DL3iut>wbmM z$H)&CM##X|+4)e6F&-|R!$z-U!s}6OGxtA|t}(jKZV9(hlg2hrY}>XQqp?q8<7;f& zw(T^wZQFKoPkZ;T{L4D?&YGD$`vEMag;x%-{>KKxo6nn1*4-AG@}ZwI@6Y7M{FJDD z2*k>ObKJ$GI~ij*$eN*}?8Y?JNc;1G!(9%6&x)Z9 z$&oKdMa9TdVi<)Pf^C33Q59KXlTM2a3*nE-fP^A8~HPmpbhmx?xq!;>;<@(pE8eH=|_!FgbN7!?0x8r`1=1FqA$b#fHA~K2+1|kZ^u3)Gyp}yh`t8 zhJ(ka<4_1OUQF;J(3m+@{VQ{w5nBjAJs|xcG!iW{AcMzcJ8|{08`6vcy%GxfMQS1t zq$8aH$DT?uI?N(xD3cks47n>SW&r4;4NPcA1{QD&q1VPg?s1&Bq6 zlC3Oqv<+{xvQ%mDN9=g5;Ji4>6OAsjwW72m2F&&CMxBnOvWjc{^0$rjEbIq9m^GyU zdKuReMiF>#YGf3~jwTe8OGOPMo5iFwV)s=^b>I=yM30V+Dd-36V59z^CsqE501-8Q zy*(vqygwP)g63&wc_Dau68lv}Omyu~T?6IVe26el8bc$(`sjp&cgHCy>QGU%V%#`R z%#a!SqKbmvuM@D;2`q$OmI=R{(ywl@S;&37Mp1p-tW_!g`Hdnbn z7hUe(IN^5V?4WN-SyswEZ{M&fTNb*>E&Cq%-$WN%7(7J|w-m*^h4D$_*Y#ne`=GkLoTc^St%WhCk3xj+<^&GW^OAu0gzgg%qJ`CGZNB>c#-0CyJ43%;$ zzahXW1(Omsx$s{!i7$S1Qx|>5RtO+hXSeHIr=&|mwMZKUBbdhP_;g{gU_TV44RT=}(rQnnFi zRHmfSgf6(S6%NfA!{9H4%mI@iGVp6GnUCQZhyJ9*9E>2^p zZ!mZ=k-EXC7z80<^jg0+Kd;$64=Ff)-p48Yp=mL<{3S8&grp2@jRn0JGj@#x@7o`d z8eZAglK-PzDFfhB=&RM*H5ViH=~WY(Jq=9^QrFDs$be`n5E|FN>uZ%mw_1r!m|aQj z@)+BB?s^-o>e^r%3;FqXwGQ|HsDG(^Sx5xEpiu& zMi9Yex$Up=x1W1-z4c=E(iw#m%{u!H+ZP7*ChuvbNL3F_-&tF)SHnCC+;L85*tZV} zD(dRgaSPLj5!>C+7<-H1aJ$dgh1reS!PeY_G}!WJAwPRlcaXXVBlD!hqZSw3u)J~m z`CD!-Z?%&e;pl2G=N-F7=Ck%6b(cyT%Vrq4!W6_5iFlRvvPjJ6pY);h?&tUizj#%- z)WY$6*m~I1XIGR33^()7p-b!F-aiI}dnfFsJQ`R&5i9sxztKEO6`JHzgx*2OG;@Io zh_c;mx3EueTQE&Cvkzu_UOHIUyh6^E0e^8zO;9GS6enD@XS}~ttU11d zGt?Jmx?jF&ssOb;$6}h^4m`4OvF0KFO4N~){)_yXgm;_$ACLW}rjOZ3;HS~@0(uaX z1BKhuMw#h7U79-o(TMy-^JPq1rRV*s{L#gERwrm#^<_xBCoppe8%1~q-k&_3iFUBR z@?5w1`N6|>;h3T@*VnrzOZL~P3|U{iA|lNQ-^}sp+ys03Tk7al<>w>J^f}3Y3`I@u zKCv%Yd?85&(cGf(1EZsm!^WBlY7vzLDa>K@z=0)gr^xW$DdM;g)E-s}WAaa|!d*#f zzGAbtV~*Zn#E3N5g35v5f7r}UKOo1VI>b;5V30xA`x zIZI2;^;#9g($<`#$zq-E#{AzzpZ)^Ry~&GZ#50o-6qj5aLR!LkldFtoY$;c%J3@X^ z)e}XGS{(>h3g#N#Gv;Y6m$=Ys%WLuJ+QRyTPM*VdxREiWVLhBV_uaV`rU4}~q#lI| z5iMnbw|()8H%YWS(P-@9sFdE(uzmw2&z(a+;%=ki8VQ5pX^HKbu*re(rnLYN*~_K* zJoEKG{8hyxYq63Fm>8ShN0T&7I)mUewMWFlD;=lsT8NQiQIyxJ2Qu8rn9Q#!I@Dki zCZrM&{+(~H{vb6$@V2!zMbn!av;V zYqU&Pqrr3*0nVEg*1+HnlHUP!l|Iw(=N{Xv;UrgcEi3i>O5y3p_4+(6uyyy&4nA78 zl#142DPRr}f6sh4lKO7Lj;bG7d$DRTPNAxIhHNmnv1+bC*i0k20K)XaZw~6jkG~|` zpFFqUH%ML22fgz-ZnO0MT(ys}KY;X9aOG31y`=iHXrE54m-*W+-rO^0crDs)L<%?) zwK3no5O)@p<=#UHa__h31s{F=k68S!>6kfuePyPZ=I{dG<@kOdbHt>2r-*c~u4(~o z>t4}x$(WA%d15pSHVIlh@P}QSI8$JtaoQ3p<~U|m*n|LV)7tYF@7FQ`5MXd4WaD02 z(fXHIO&)lswxD8|*dzLi40UHMCA4Z7zwjf|ern$ZpiXwqY?+?*#F}cF(uSPoay>Yb z1G*%+)R^l51l)u%zeNh#_OU)W=ATS;e52R}0Wz&!d;Ydhn6Lj2@_&~C8~U#AO-=0; zZ47TNc)oZ{%e{ztd~Kg|Ed>~OI-(qFj)U(EuDc%ccv$6U(>}ZYZ%8GLsZKF*2eh$Z z(2^QoNLG~!aM|axva4LaJo3Fk$P4V7D=`}e8E;2z?~!lA_k-_t$W`zHerWyF>Re}h|Mr+Crq3Vxeis!O*JPL$u9)W!+d z+!SQPCw-@1?VQIMXkdy}=_So7i@{_a5C(J$svjh;0(HPx!i$L^zOdb6w%{DN$@f#< z8?4tH5Bn&8H4HaMgxhx!X^JS2=cyc$PB5B2+*=`-K@&{FpN+ zHg#0>s~y{F(fZ4p?~$>xxuq}`M0IhRG+yvM)g>7d+%f)0*iT0|@h=t5J?314C z96B!xigdglWpz6}^(|UeE=3&HZ0EGVW zEX8av#wWpH1>-n~U{KKX6Z88KVfj|)U{+#W3#Bx-Oy69kih_yUsy8$CjKn@Zo-r~Snic0@!Ze7yO%PGU)coToHz}qnb zNPDFBPh^FymJDT+Yy1aT7X}HOaoG~7kci7~;qok`SD12MI0&+NsJaP7n;F+n@z)ZN!H&` zK=MqH9s#g~*2{0lG3mGRn1A2zI%%zTG!&H>^Ao38=VV~Kg+<)n3XY;Hd0qA96fSLX z`e&EKt)|hpXuB*jYS924?!I_@Zm8SY>s0n7`)4d0D%ZKXWrL_;NLz)ogpL0B(2V$c&!4g^dZ1cF=F z@Ed#wi}IZ0I&Vu{jz7}$oElejtjAQNRI=R;qq!a5D)!lhg@>{3;`yR}jd0-P-`NBe z?B7e}8D!tvgfVnZHg*%=&}qNsY+ zKfG#r_Qs-2M0R<}k-kdt@kh5T>DGqa#GtwTCL!G+4~RQp-+L~SHz|(Ba`QFofZg7W z3L7?jM2rb{g_U|+N?Hm|NH(#+c&_@c0{It>Jx#b;uc2-Nex4WV%y6D4v_qSaB#^RZ zz0e+6f{;t1ou2E}vHfUU#_sj}aU0R69rBU)g>q^YF`kdLA#ppA*FV20pymiow1A=X zLQ2zX&aOy%CV=)pQ?Bi*_$gI18~LzLn5aS)lH)rMA5G;Co!29ei`GMwF9HM)hJQ64 zg7E%dQoeTi*jGQ*URpmd;;(w zu-Xz|dn_q|;VDo(gZu_?CQP+f^1C{5$UzfLQEdF`LWJT=MIX8yb(G+XN=-3uZA3Ea zhO(J!p)?bG;7Ju4p@;fjFc)rD#B27Wy3NNEuhri3)PJhFEo{VU8h*U-1=AtM6sv~V zyJ($^N;R;GNX|Z5*Oupyoq!i`%$@f~a$8gImprDJRhk%pxag*a|GX>5I9qnC(&V3d zMcxHqdtLbvn9S*~8)3=S1?>0H1{*5;q~}047JZlgvX2(fza-={@Uf0MyX0O|Y;>9r zNk8$h30vIx=7g^l=~H{~h@fhB*sDOQuyW5DNggyH9-C#D$|YL_A4@;vBy_GHVuUf? zyN^jX$l!A$(>!ZNR0@Zny)3xP{a}uuA?LW;enleY7uXM$mu#`fMS{agg#+nLp2fB@ z4|RD93_%v*AOC`0Ckr^HHcIqM);8yTsMaLDrvPNMTCGz>q!m*bT@9IKCX)$CDVyK$ z$91fj+d6R?mXpq@I%^@P`T}HBVi9QJ; zt@{XT_M7iV>#ZVcsr9URYb#)Ff?Nc1#}G&{csW!L*yl#`2x%ALcT00jCDl4%eJCjK zM)B67EKsknrmWl%pd=j2f&Dj680~#SyvNZ0^t1U~n8$n*!1JZeq}RcDADpBmP~Fb{ z@>cb>rB1W?w4s2fef$32^dpZ&0*7jvXncBTDH=r-F3p}Dk5GZ{nYFd`+tiFQ8a6h$ zg+y?@D-lZ#N6|*Ses(QX52J94>K|M^H$Lx&(ar7>DCRC!=sp~h;`oE07!+x16n1e8 zqH=?cV|gOsrs4F|7u)dxd|}3yNb={{7^p*orc?Oc1<4VyTA7nyLF2bW9~;^A1=sQs z<~_rEKKZ?-D|ei=;>P8Aj{+ z5T7U)o~VQb&4j?M@pbFN2<+9AO4-#CKw-PngP=!Op}enk9=D_sQobZyxCKUDwt#fR zBz`sPOz%)&EJSfwUp8ePIV9_E07d_?JCm7gx05(B9cS~v1tM~vpz#c+M3!B2I13)S zM~f6)ra_6hv^-z;Fr(hyu-hq-$?M$pi}#qHN%_Q2PwgXE^bji5maAF^bCttpAmfoY ze3YISyrH;IOTDo2OGHXYG@%LuH8b~_Mk_llZc(};?L^m;_1fJiu2SYS41`~Bu}r{s zt@>>4AH`EMXMs4EUGD5z&$6Gf`Zh}!fS2Tfm1X}qY?l04tf0)5B2?#R)h;DXY45OS z8nf?M#~%@-8CXVT*1^2k$V7)5LwG?e7s*5Ves3o!xPxfojANHAHTM$2Pq78~@H`#C ziQPAPBr$TWG)L_5bXrojvV_2?GziH>%P{Azs78~pxU`L0rF0I zp<5MQs#sJ`B%{vFQ#e_lYD;IDXQdWHJ`Xe2Huaj+!hjld_5}evlFfMjmBn!1D*h#H z_gBQGj;1z-Cw0((%aRv-*FyY%_iD=d0S zkw${2lu!^H>m$?PbCd2YCK z78vh47iwSitHy;4oxd;c(f@QSlGkk?lGy25-fKRxM3p=_5twvVd)nwTb-XqURCERh zK;x_RdZ)v90fW!!Tu_!y!;|1;Sf-HX^JZUP+izYXa{alp*aJ5NOXRCRvsUrYL@dlR zjPf4I!zC>hj^3%)%za;Un@x(cd|4(nBUgQ#z(?X%wpbIkG%_6pmJ*Mwh2kb+56QLB z4>_dtHVle1svEeh> z+@}7B;mY{vN5qp>eq%A>hx2bkl%uhyqxhpwde-ag3+RVKh@Mm+Xu&@gQq4T>Ro>>N zFJ&=M}g1l zawFm$9K_`x;-Txh(A?eO9g6@zYCP7+-IKwtyY-06=O;`|Jjo>SK+hJji;^0y=_EpI z6_(YnI+(Z%Ha#!nJWOV$6rP*v`;^p?B0Ox&1?be@h?(s2XB7>W)#EHj&dh(=^zi7v ze)qJd#IWIbZMu}z!7<#|LiejH%5hL`GsjK~#aWgrPrpqSDmfN)?vv`!hj z7+}nm9Jn`V<~%8E%A@=>oRrz;W+nmt)r8hONt%64wTS}1Q*;mxdpy42h`owS#215= znT(Njpg8ttT!rc8C}rrK%dBo*lMyugK@0MY-^ABxe5vsFgr;)d2Szv~2#`6GSdAjm z=-O)^wdd|Pj;+AW&F~@)puaXPAUA>8xwsy0qNAcAr$q$|eHR+w6ay?MuI)87a%aO}>&^~lvrFEobs6Knq-A!2Z>x;!@VP`s0l z5n{>CFeA>$09f_(K)0ar{*EaA->aVwU(~9!f;zj2pAx=aJHFSBPV~xA*cC;Ueu6GN*JN&ktQA?S}Q9BB|tEB!?h4`IN?5z+iGQbRoWyDGOB znJVaY!x0Mm1iNIR`UNZCRoHqK&1sK^20^Tv&P(@-cbY9A8%WjlvR?7DT=g05LFkTF zt&66e)?b@zO+8#|p;#L4pD}~DLNPz0;X^UfX6@gpZ$ag*t?$_1e;U+*^tbU4!~E;q zQ2Om{O>dD|z*osl!7DX9-*nElU{C{in93~)U`8Oxr7ZQ_bJ$#Z>7Mkyhh87K`zaz% zDhGO%`v}`CUIVzzl#?6mP*+;v8|zxb?(ZtYvh-fz94Skr0!(2&Qmj15nL)>Q(LV0&DwKe9*9!F4pEy+WqD= zuZph4N{4e;V?Gs;r!fIZkPM4ilHGzQ{X&vE(0ns>!-(61%qdCoH(lsF=gCK`yAV1s(WZuuMz~9;;si-##8V07a zhDsTegNv9T1``tN@Q9PaT|KFksSd)SU=3Nstc#N~GGU5yr30f?5k&4g|`yf%!n17O%QMnolg7)p_K`1|Bo1 zUL!>TsXl4)U2cwETAJy>xYf=nlw7Pv8>8KUilvq^SPHy4Mmxcw1nVad?F)d6?}dNj zDI$+YJzf0+J8GmtQZ$29S?!{z(iJ5VAw#wd5^?@Xl#5EVO&vv9I00GYvDOC0OuR%@ zbr6>Ks?T!mb||j-{^8YjQTK88x@Zxs#!pxRcP$sTOHE9*1X_w@RY3gnJO+}og#{!T zqdFXV;w#LsYERvuRngYaaD432@$95;VMVtn-v0AnMt}R8(Xy`mZmX3y-@F@c1zw?g zIku7Z_PEGAKY!FgMWwpiI`M|&nqTn|e$G^WzU(v|MzoL}d5fn?n%83Ws-eYa-G&IFiC!9Z>Vu7r+jHyMq@OI^KxVK)v6F+R~%FUp=02dnTXo>G3MTu;2^ zosJfkKRZ5l8%Jc$V9;6p9jC5*S6n*xv;IuJ9;^RLZ{_Bd9%EbDSz(8Dt+aO*ijsl9 zlY|>w-brb!on5fN{_EZBHjhtnR53)rLn<@%hw~%gX-(ZHyrr&iXAU*0`z`Na_bNq2 zi5^pldid=-yvb~Iuo%9uq<{1-jTtzMY3`5eHE^X!QV2A61w+XNN@9Cae-9CAyt(7P z`|ms6Geu=>O_+%M*Q^Bk4f1D)8BGHJC8V+E`V7kdxmCP464apw6bBT;Kc8HRYn_pP zBIJ9g-*rr*XsUNE86)QG1Fl^doUI{*md>9)z$ZH{dVcXguBi~<%!?U=)`|#Je>do+ z!VW8Di*#+j=BJ`|u!8bscD>YPuQ%PgjY|bNL2q&tY8BcZ`(>N#F5`i84z@@DaMmB+ zkeRw`4S4qIZISyhgs+Qv#)qSO{HE=M%K>zEcYE(Ca%r}s8Hihui#nwK@JwZ_4v_V? zfG)y=e1)-;SrlL6iK>r$m{QlGTb{9#@4Y*}THoWUW*jG4Lz+`!av*(Ue1+p;#`XiL z(4|0NtH{*ReHO|d$*#~ z?%Q+SrR${t2uKQm`e*SMM7z-a5g!h^?CM|DXuGg`9jWsOYzUye0}zDddCL9&4!=`x z!y7_BoKXU&|L}b5I&@?`uOONcoPEBB{5VBAj*OmwO?^>9Rrj$)QpFG?<9 zS*#DV-~`SJCb-R^_nRxx)BukFG2Pj%6O@ru)@uA=?nl!_*Xx8I-}OIU5NoX7RRm8} z$5LIe$mLS~BNNGG&(HqMS;R59t&lp_H{|AZz}K0M{|UFqpmfiq#J%SI2DvEKeXn?@ z(;Nv1fUnab3!R!@4Wd}I?yI~2#w^x)_mY?EwtdxJ_$8#TkN9^>7GcGqvQvEY=4S|_ zrww~%D8!&T=c<7dLet$9d(&&aJO{vyNS`E*6!`w@ z`A<<$=hbP6@cS2%fiV{_9aJcsl72)&hPLL=UGK9Q zPty$EVVi5Ns8BCZ(@ZAc%MFn~6d^wp;f0Wh%2j{od?y={zL#mz&Z#BCk!^Q84^>kw zX6_?yP=zMFtJz(#<;9Ie?+Kym0tk}!@s=Xlfg~CX2Sv`B(T-F2n4~;3oQSlRrugJU z4>9gF?Y;KBXm#=LT&VoT5U3WH11R0O&rUsV=Br$XCAw~w5&5}q#QtirBRFw_oO+XC zx@SK1d>;4|5S0@LSn&+w`CtiKEw#?+4rRHIxHH$@r*==X9#>q(*Wbn>G4T41gukZV z7pRz0^ASIOkvq~+eSuj7}c{YY=?Nxj?Wg@s9b#%yt005PM01!U1nG2zd7?=cFKjCAOQ{=~B#nD!g zA2Cxn7&iq`Z5S&M>dK&>NHqo9guDidMi}MET2c-G>f7edTigwNM!STTq9oz{PmK3>;y8sEIB0cfzP+-&7UWO%iDhX9knF_8+oSD4CL1(U%O*UmM zq>NrGk`XTcPj;)aqIw+i;gU3x;y%R=DjPDZ*!N{7GIWGJ`lDT0jL{A$T@W?zRT7g? zpIup9Jok~#A}lSDVh>7sfv}`ER8N5Cu?kb!`BZW|p*=XtISw)wPX(xt1LUaFnPHXa znE6b3ja#Egd5?Vz_p~PH$wQM!7lq8OKoWcqlrN9Nnnw(#p%HxE5=BM^UNC4r0*B9clZxy6rVnrn317 zyz67*a52&|=I^#EqEJ!k4soHPJ9z>jCDfN zHTOvo+zIy$tHKTUNxyFi9;8v}Z>LgCZ@uM`NOnPy5tPuswcr`NyMMTD;_5}{^63_f z@oS|@bG}t`H9yuARbNaGZa26P-O(6sn5k3-YSQQWqXvn_YQbk3?d~t?Q<0aeQvc*) zFLsKcmzefQ&K&`d^IG?5a(bMpyY)%$2$n>2t@T%7ZquS$+@COxc}t~pvmOW_iFj5}=Z!3C+R zle;wTf*cTxhD#`VsW7XZ$F&?QbZI&n>g$Qf)dP^e=Z^|=bL@?x0AY#qg1a!q#{#x9 zr#P~TRh0C; zLlQ{t=1>gLQlI7k9h;b?$+( zm%{+KM1j_+*L;^}^Gze7kMqi(EhGd}V+@0Z{cLUB(50`b8zBiTpIcv~_en|3-NKreM$966?stRK=RPyh4t*~7_ho5Gb z+*_nnrH_kzcC+XE>0FV3HM_lZ2XR$FML{HR>Vx#%&GKY1CL$8*U`p(-QRyrao$0+B z7dEL&@1H-M)bGnqPL^A~a5T*od9cjpD64b4cfM?R=px2be|*Ik$F!24`ssMkaB@G2f7*UVt2KsbAnix^e^JaVaW*ed%$ zFx~gmmz0}X%h!xMfF|W>3H-nk9$69n9FUu!uD|r5_}-0Xv)p)b?awT4BtnsJh{v>sY|t>Gb2!$OGzb!`U^Y)=*~vrWl*8GI-+DgnWKtX}%jWcT4lu(w zI&XE&yl$(yKGYO;wlw~pb_)Z)AT13IxbL3l7w{sTKBFu8cj)adhDbZr*yB&P&%0r$ ztFv6JNG~t2x|p6fX(gw?77s8SJn2x?8?jXVZE=F9xPnSrw3tV}_SZF*tA#woo*9$T z=*j)&9pB%`$Q=RRa2CMs*GWF`$F;{ViF6vicfs4m)&E6|+%!asegUrZ4{UrV@V4aN zOp~wMisn0MI z`;#-$uNZ<*Ql70T2`3r>S5GAs_&Y93vI(mLKPME>sH-MQy)V``PiUFOdLtPqMtp&9 zY;H+Rl~L)e$jPFUvejod<4N!Ict@Nyx!qGI-;Mn3BBbl{6=BVGecbgbr0Bm+<{N-S z_oOkW6lQUFPjD^uCXzGYOtKl!4r?socrT5l-#bTx>>)lTlSrsGH_F^u{f>~XiU6dZ zRXIBHWSgDwT$kX#>df-|ge9b$Ct!jFoiGlXa>L%99>cal0s&3WwN1hk&upFR$9^Ia ztBHP-UxX@e$EySRtG7Urm}!p9_dlV%6RpN`sFNM>VgL~i@(}d0X4v3)w?uqjD&|%X z+rH^>2PDyH3T!~~>X%9D z?ORuzeP#4#YaGmUCvN2wG!ALa@Q7wy5o6Lp9bwWzp=3mSR`XzQegTQ9Fwcc!ei9w= z*prCmCKAbf@AA|vHNR%_QgJ|~eX&quaO+(&Uf~m=wB}1PE*h?V6FbISf!M*q{F?Ti zp`)Sbrj2SiL=BT~rPwR>^FVDfBW40qp7_?5*W;hBAW8e_JPD)mR$#eKd;GWWHoyso zBn$#Y)yCy0$!}GGr9sr}M^001M3Q7LR3Wk1))?hUEm_93%dzU$ZPHc?bZLwxGnbvu z0Z|`ku!QbrWOjTCM%-2MAR zX^ledX-h=#Xa>~?mB6_$WhzDuZA8x<)`$`zRpc0(ki>@)M$zM?O=Ph+z6TN3b3CN; z;IG#LHL9e8>KQ|-XXE6N zt{781IXS)w-A{_*zlkRN++5`U2y&3;E6uE?#`t@F8EX7L!9^r$cJOSMXHcuWtasn{ zL%TSjuvhHqy}cKfZlWf!GAu{g;-!1LJ{sMuA~|3E09jtw4dRyScqF|45kKYakdTB6 zAE5NiNSW%WhKaG7DL*X*ZTQ_P@Zp3dRtoZOio$UVKyz(XCqw=Iszsl~nBM@&Bbyf5 zcHsey%fn8@ksnYrbf=>E1pMxO1ni-eOjN6zSq-C#8`z!ldpLli0I-`E!v!o&y%rV2 zjakA37)xRm=FgRe*6535AQwE$-J@;04VdSb?g$bH=?WWmvou)a271qdv+;zO-wW=G zLI|V4qf)W;j1fSwM-;jgtcT8Hpa=<#X>fNU+X=6h_Nm7>uo7Y9$9;#@8{KVgrX6BH zWaKtFaZPue6I!x3Krpu)F&vKFU#e~dFOh{Rv4)CN57ee}VD?3JUxTEPzDJ!0>`k}5wf1j&ZP826K3Wq#~WF$`T zdRY|Qv|TrE!#UK{xK(=kYk)%Fnyo$(J0x^B`=m(8j>Uj)o@hg~0j{eE^OwLogwAc- zWnNX)f3m42VlXS1C6KBlLKNWUj?hN5a3W{m%E`QyYUHQ^LsO%RqqA@tn)G~n6zuh! zD3)!@_rzZX>zOHCt&vi-5{q&6Eb@`CdjZ6P#Y2K9Y*LNEllerKa`kjLgcTjWAb0Ex zBKlkaqzZEX{g#t|oJJykY19*~yr+}Vd4bm+93FAhmo6=_c_W8K5&ooYA$v{f-L6en z3h&RTG~2;X0n&5~R#YCDoj>UA+N`^coxN>so(nhSoePE3-(cR}hwmGG8H zR+8|gFX{t_FJ19t)uGgFb+~M*qJ$^g97}bb?qw+kIq{@VvDMRPFnSL_Ffc z^t1RHpWbIBig4u$t`~haJ1>qL81R`s69V9EtRbBB%?)D|zx&Q*Z;|>*o{sop*ESqF z;`C?gjjh;Ge_VzU_UWh2^!N&+L;e5_gK!5E!iyj9Hy>K`lp{=R zE`uTPWJg7XP$lPTxxS^W;nXnLW+akkrFWGR&J+>9;f$mqprpU6OYiyj`7(jXU<`fS z5>oJTjoGMjBuB-{LAFXtFBKWZj@&Q87L=)hKSF;B-$O-V?m`;-rh$}vCRl9ux?g|p z1#fFKdGmp0gT7sdz0_J;4MByfp38*)*wFkiDcN%CyGb)D2Nt7a)Gs9XAI`jOmD|_i zcVDV#brNy7t&e(DC?Xr|*#4FM6JIiR&>uJiS#2igQ!XDC`#(mxM>fAqDM|-c{6=-Q z=*}R_ROSVdFe>$WuOb8`4hJSw=wDP1B)VR$v`civ0f6eJ`EbnmDd_O>A75qi+lYnuGYECN%0fR9U-1!fxF?x1@=}fG;$8I+eK{Tv07`F8!Yb zK2T*CRieNnMS@>`vy?eAe#k_!4KdfhQ6u^A^28B#%MQly$w)ZH2H20I`M=w#!-%;5 zFjzx~7bmR&&N7AAthC=)C2C7}^w|vn0>1Z%a{Bv&{cxu&g$po4sv+xRilL{p3?k~9 zOH4Mh9Z29hkVm-E&g2eM-JQ76pB<1~o64t%-pTkUY$A-XYU{ zZ@A=<|KUzPLAk)=rSDosdsVBved%9(DtX9y9Fp?IH{&@xKr_+MMl~)X0(#xTDXiV%gKK7lzq4gfitx7_98UG#@9nDc<8f zweY@42dsM0QUXvfJrbDHm608svm!d2H(~mPu}jb#%)h3qnzO5j`N3%+am`0ViIu&P zFCv4-Ai*Z52o@&maHBP$ZY$@;kky$f6L= zT%lY!`U5Y)9&+CjGHx)cTS|g3bXcV@yQ#}xtcq69XHnevM$WdDD=~?4k6Gs;6s)ij z$@rVYh_tJv=z2bN6r%*4i;dLv6G6*@>96zBhLY7wZ}W#9F_}kcwubiFQILwlfim5{ z^#A{-$bNdWIh6oOSkipd8j9?f`6(6H-2f{-@Ch?@_I;zRO``*GQ(5VS&DK_xSLQHhZTGu2N!3V+q)y$a9_$ z#Y^}9k^2u4*$JQ=FfcEvEUuU!NCzp(_iAO6Ct?mquLW-TO8HEpgNt&qFbu03k^?bK z`fkj`VVwFS06V)Q_z4aWhJp+e3JYsfM7zrR5&O>AsAOFxWjzP3J7<`!n5$O0+kb!y z;DOlg-Wfm(g(O$(S<^M$O72Dca}|?UZQt)+uG5T_sV>&pLX})(9i9Q_8D#Pfvli%@ zs=4f|h;MSHJw$!uMi7geV+AV{n*PhiYSfg)DAUdTV}j%TGDc*vLSvT4^Qk%jbs)m> z>|0y_==8Jy{(sH+F2k;t?&s6ESMllYx!Z0(pXDbMxIIqw_OW)||0{!+dOg-eCL`kO z=HvUO&U;m5+{Sx|K^2{5rv*RcFHY574JQs5UrWg*Uq= zN7%-1Sa}{O8TCpb{v;J%wmYok(Eo%6=%LjzW>Z)dHdF2K&Ujonc)J|2^<2-e_X6v4 zwCH=TdGphW5=>hZ2d1X;>=v*ovbA8C^YUj(e$H#F%ZrW$GUHgTO!_0UKoYS*!m;5g zoGR&v<_anB%d7krUEN_D-E|dI_d^k3@be~rSaWPotyLTHiXFi+*O_lYp9~a+z-1dW z$!d7t&*D~?)4-=5&Q0n*^dIDeN}9Y@F7eM13EIkjKQNs1dH}1{q}6Z404ab2 zGMr*;fZKggVS>H$xV*rK4G)ElA`-=_;Lq-i!Q8Y2RNUXGeNt|E1;Yw0HgX~>t6}c} zZ{Ux7%pbo8v(_6d!y9a?K{g863<~@ zaHNN;*=$$N)IPgF>AzuS-Etk^op3z9h>Ji7=|_+49?hma(J8|Z zYIER-!6sMRj!zVSzz0jC6L%=6(PYOjbAI_MNZL(M){9AGsIq#zh_?@oH~ z7{2lnDu`RTY-;{I9Yi%{R&hoqwJZF1x=1o|Y}70sD0lNj*&dLq7&2&sydnQpsQZ9e z#RM&k><&FNnaKoCp6h{8cHQ7xd>KCmgI>tR1kgA&HG0IP#Uph0+i%z2H;Q!L_)_>x z^!_>EN$wt0iW-=`A8a~rm9H#++(Z*1cB|*{&34nr;|4xPd#kO^=oGPxODccCAaBqA zM}PMJkN#TV!^Zi(#HFN+vT~TilHzlco23L83JzL`h9eQJ^Tfzu z*^dlr4Fq|91h{g(d+dRq)DlCSWIMsZiHZkNB??SC{@(MJGL*#|c9rK)#;4tda!KqV~E_|G9*Vsu07p_jpID55ZHGC%9OdW5 z;kW|s-A6IK@Z}-@icuO&;!<^W^-|%}iOs2a$N3{omF~3CGw&DEX?W^(Ct#Y->o3J3iUq`0;YAlOrNH}X>fGddg%dctfQHeUSdbfUfsJ?OQ ze@uN-c%5C>b=%l>8ry1Y+qP}n*|BZgY0}uX)!4TE?LO~+@E_#LdG4&WhUS>v%^wIS z&e$i7AkCTq!&M^56G!R-ST%(t{b3g~qa4#M8nE5+Ms~m+b zR^x|c_UITAvaF)Z&)E%eb|?XAvL^Y5kPh+qohq-p0a5xfnTixSrt%R-5ubhbHhT1Z zCx3SGCVs&nvI^#Vra|`;3p>hs!ZGyXI)Wlc3KQeaZOjzBlx!`E%wC_3l3)<0*7Ud#RHL9J#%7{Y)2t$chXskp-+xvyQPDFcFJ1){&~%MMsVhSWTx3 zAyGFsXW6!|8VECddWA4yGp4jXOUQK|T$gJX#{ zmDnG6>`Ccy_!_sxD`V#zsdRs3cHehDeB|OcI3HNDFcCE)Tc(w_s)Fn(u+9BY3{-Cs z$dI`F!auZ7O9FB_VYvH8b*Z`yj0asCMq?7TNu zzliBuEEgZBS(SM~^5W;RY|*k>A(Skn>8=7G;hEG zaXq1?Skc)??_eGnfzd?n1q3J))O$386_LJjm*t4*xhIrv;6w~B3awI=FxPGu zi!cezuZ^Ns;-K(8!?-CYt5<8cKG``>y__IX(jd3BeJ376cOdZSu+U~fyl#LQjV~nI zEhii$c0Jcj+2Dp_|78KOxF2O`kb#QA+!eE*QNn>EO9oQk!aB4jiV+zG+)!vz0tb>L zET0w|qZr~B<_po{#Wz?1imsVgF%ovo*6>qQA?fHc?mL(K3K%_bn&_`sH!U@`8BXZ} zWKOk*`UF2yHbN%I+fsQeaTD0_EFx3_ZV#yOl3jlj+1m=d5*fVM<{W!aKrD9n<)to= zBp?=d@$TPpJWue1AmJLk!)f5xGLHZXsHFm1@#ErVQwR_#+e_8Ab(AHmd`>3nc7!U6 zZwJJ#ysW#u@Vs1Yn-sk*`dS#z) z@<%$RG3I>AQ0_Fo-si3ZvC>P_yf{_#7@ObY@a`)9tKwP2;@QkKWaCLdKB=l_u_e&C zfGKO%IU9IG9*xrwJs&b%&6sThK!itj^sSOw`` zzH%dwi6ww!{TP^qI6+#h*jx9=Ipm6ny#hC|xgg{h1~NNUX8LjtX^tAWY%O=P%y zA9{Ly4^&u;LG5SK7-`Mniq{kkuG6uvxrN1G#KBpe=M9rG5B_Qi4Z_kHQ!|@vTpdBa zfTtQ`EM>ibShm>+QZC6ML>9|hMvbSg6;&!u2F@I-_rn-kq21e+53`=HiGi1^_}ZL~ zsr1ES+keM|QQY7^xWopE-|ld+0#mDfQ>&7lEO$G2CjfAI#M|BS94$J%L~(>eliC|x zMtA$^b6Gs;*L^Yt%bbCO6(Qtb_Yy2Zw&!1Qn9j171xk)C5XN40PsKJMiQ~oKy{_(k zh=Sj?#Ml(|29~kC zd2)a8VZXQ97T8Wc#wMaDv@bl9?KWP!Bhw2%s#tuZOw^uMZ!NjC$OV-!gw&P1!f1Nb zpIQ#|*o^a%LzR{^F&0JCUVs7#aokltrRJ`UFaYA!>!38RF}h>QlsfMl==&PIP-i-{ zvSRN;#FQUd3@Os$wqFYLG4dBk8_{&$%TVJEFTN##qX-r-y0Ye@$PhqMXCVMmZlPU<%}qys$ZI_aGS z$)%Of+eV7!DN-vb>ik>h$#L(o@jgxl?2i%~C0O)I9>* zAVru3TnS4wk)IH3$iE0t@#%2cz@{bgcL^^_*Ov~F`ZFs+94~V`5H9YoQKqjhIn+5F z&W~gMEFJz`9{s0f7`BJ}N9A{x9N-!*OcuZtJU|6Pgv zX?o#L?{n^h$y@-p!Q&&~sS^Jn$gKKHi{qMimNh~e-}4;GHmN~w)feMLIitO#2p2*O zw}C!GGjw~i{g5zKcBk5o2dWT&@IBTK86J?dG4G(zR)*c(Mp%VY*Yvn7W)jyRd~~K9 zoaP?6T}jC*8rVRz^o=SP>)Wpdf+11t5%|t*7Ze=L50JJ2%*PW4c4;34Wl_QVIC;Gf zI8|!-L0aivB(SgjyX-_q-q+Vxj+35-Qe2l~0sA=(8lBYuxXPuQlWe=Gaj#{w94~2T zK}?FNN&AG@IBb?`nd7m(B>aFV60qE13s-f|73DU{5Gy~gm6Nle)Pd@%6P|Nh>8ICW1bS|L7+N5R~z4jh9^raa4A0Um#V zbRutPFE7)oRQXx-Occ(jvODY4)YfDraUfmdUk$=%Fnr_9NA*cT+j2-_~y-zZC)Oo?0(T$yqmFO1!8qx&Hp}#z>oi~ zCTkV#*V@@|)njFaF|kq@vVt+4=4pv8VQi{)2LNRu6EJtESS_u)p}oms z87t60h&TwR4I_AzYwi zvEu798(se!-&g1R>l7UelR5vyb*5>jVm0rt1@mhWqEK>h>1!ntFStM8r0(|flXDkD5&oj;z|Q}1rvm=Bf}BI>m(g~i%jj~#t#^(d?44I z>k=mC+c^1I6MN$!u7U`KG#MQS7*;d+J@`=x6|9x^dZ?%{;OYrdUDTUPz7;Xn`|H+d zUo00YL3T#>+Ju4*MKxLZn~v=@NfRlo7J)(8a+K(2OAKVrEq_m<1wo1$av-=+;nW{# zKvwEI_93{`Ib@xgGCG?lRrFe7-3}D z-xKmvMaQTB5z>LBt{`t};IM3&YCjn$iP!>O3ZCjZjwT{vd0qG?9CC%dEAfTet0wd0 zYJPf3N`9oSF3bCt+j{reDOx2U8k56%c3~9d)l!${zjFZie-{(xRk!QIGs36M18rqH z6+Vds0|eO0+pQNo=IPhb2TJAdAr{r4M2KAXnxir&-Ua73p4gV?nz^{iMDSXLEdmKp zk0?qcQS*_o{mcY-!XeDeU?%gjy@J0RK|W7wD7ha!VB8nRt$j>IMUWndf1*d8`eh>G z@mh-pFcucY@hvNrPEY}jO^$^zI@IXq@ztixXb^ANqsEMu1NH@Rh@$kr@PtglnQDj* z%9ccZItuW63;cfGiKtmpbQ0TgWP%EHK%8<4#;z&PT-O`8qr6~z_QRM~KKJEI*bI^_ z{A|;H&hHa3?cTX#(y&IdbC@Cf@C+l%96**#hYA+G&(!8}=!9M7g>|FGRU?_jKyu2@ z5JTiuTvzOG61R}b)0HfpFH^0^+QwwGhK%8H)|*KG%?w>kq)Mw1So5G`KS$X@iBY2` zFh=c!fyO$;lmqUPK4&-pDxl;;nkjh?;eG&(X_D}hljMRO^K(H+PP+RppWGD~*TQ1+ zTu8|82dtb^{sT;Fl>h14^{Z*!ZE>8>K)mENeAoF^?U@!R02u9OYI!UOPmaaqrC9{> z2ElsewtFi+N8bAb{6kh}m%^R=SE00vJo#_|_(VxKm2D&j=&U8kXz}_;UxOh;;$L%= z7>l1jGPG{puiB@F!b8tP7tfXmGsh)N!hi{^OS)YpQVq*BM-l={cUvVTJlaXx^%cWl z=r^MQfVhNaxvIyE{GYk%lC0=vNva=>TMLlIE@D3AxeIwm3$L9V&YBYs#IF3e9w1aQ z;?nVLYe7T)WX}pfExi6>z-K6nlr;T1Vo~yhEgO4P@aBPmBRB~{e173VYd<7PjP zWz^tCvxy^1Db$v2Uv;*!eXe_deb6aiv+cGkRYRxYA>s2LiJ&wRt2Lye`tRTU0_J~r z4^79{UzJ3@>*mGgnhkZIuYb7bDlh*N?kfBEf`xMphYGVHX!y7<@FWaA6B^bfbq~_mlH}^tmz& zw^u0l6lZ2btQd*&+qK?VD3C(4tWu0b&Ai7i84@>}+1v!MXobTYA`jCE;vkXdf-qWiVu=e#lNbo}oZR4{>e_6Ywp#3@EI;1a_V{E9N{a zG0es}RBmMj`rrULlizH2c24{%BY_BT5A)4GrNse^jc7tJb-t^_kg zNw!~wr=Y?hM=V9ul4;Q*6D9KDC@zXc5f6ckJFVfnUbO>7@WMF+&E?q5w}=JeYD|Z=l~?U(t?; z5pozkjl4-&&4{u51${wlF%*WLJGv~Vv0l~5ioZ1L$P~5l(=RQPm2CGI!?h4V=~+Q> zD->D&cZ?lEG7NS& zSyyCId4N~K^H1zc0IL)cVSuyvJ8iMB<$Dr|eILJ*M1)QzHcf1}b0P`em65z?v9^Go z-*+Q1xIu@|%TS=8C?uu_C52JN@2AGrojTuIrEFO5^r9qC_D2v6qg2T!dj%4Hd)0W5 z!+;fSOQO_!+RjnC@mDi~g&URbXO~B8LpFCimkw)l8nje`J8mfBx8a z6?p$d2f@^6{5r`HYyZ@sdh;@h3+U9>bG&Bw@+yse%utt|0q^s>@@l;5lk=Sx^`j zOYKFb8is8!7UPbFfaVW|{WK)6s`3ueNOAz8jzkJNdpJMhQ`+uhB72O(K?r4DWw{IY zA%M@mRpCv!ASSzCyG~t?d-^ChYQe&GtWf*beiiL&K(DqHPmcQzHV!;vZOva9`oJ>p z6Y^3DNl2(O)L0wMgD*4!_y~r*#U_f<&0V`!t~%J@5%Q`#qaolP zDsw7g%IfE6iyoY%f#Q{>e>f=Cj1Mv6q@-ZGxJTPK^p~u7(!LXZK&MNOp}P+Z{&NC zKVdB%aT2&Cc5sJT_zNe@%aF+nkPEo^fgHYg*Lbb&asVKdfJ#u2fCx_d zgdiw+)_)=_s+*t$kg-bgO~WVUK-7YUlA}*edzGkaJS-aab=j`RAQabs>vMY?b2CLu zPY>xtS~A=U5$jX|#z$2Zb5Cfcu>-ARGQQqA%8cOe`P;~W6DF&587yQ4OL|qsvdJ5@ zB!r7F1PNPTqTvltfNY>92(e4lfy-3uAtaN#} zU=;S-%rNMqBH)sY&@ns%=V9=E36Kpd`^&+rel)%}qK2rR$_7|ITR21Fwl*dg`@Z$a z47o?8rJ?kV*fhBT(sK0J-f!H#%toMjwh)1eT3!h&o{gw zKn$Jnn$}Jc^K}KT`PA4uDuOQ`!CFoiv3Qz8gh}9A;!Ob($tILad1d!#!ycmhnO0 zaUAuB;|3p73TnF&sKCJxpL^cS(vs4a_elifT>9A{EECGB83zH}C{DEy#@EcvnP zZ}oV@-<|g~gC_~sMe~TDlTagYKqNqL?*+0X!)X%I#U6jj*MbV`xNj76{BFoGpIH6U zVb5BCRLp3=L`KvQ1~nT=0xf6pOx>U5ts-@_Q5?YDSx@i|GOsLxH>N@2>}HeTA)Y@- z$W(IGRgo8Y4=X3DzS*UDT(`$9%4WH;^Ye?xcU>U#y6x7}exTQWNIkoGo}bQFz7VOQ zMc~`(t4&JxgXde?t8KphA&J~ip4(vR%Cmn*(IRUtsY3mEZCH$ND+Rym?euvwoO?6% zHnQx>aaKeeS(@X1Ts!SC4u!CU$@?GPW5fIp?}hE0Q>#h0f0keRrBHur!!p$k&H{1t z`qv|6;kz~`Y||FO^HKE|wa>K_-$=Da_xEV{d@Xq}^w|hHjl5+Su)p6U?0d|=f=g_# zH{9iucmtbI@C1My*LDe`puGe8BYGNPbFGft}%jP)pkZj_sC@roh?(}%y-pv;8S`f~jWiuUG+nVpAyq!~7ZhrG1YFEr7wULA9h+XCaO|en zeZTrc*BP(k`S5&JKD&9JXosuIe`o%C{LpKO|D!#GH#2je6I`ElyTp;p$=zK4j$O1V zRrF2rN6CKwLB|mP(NNB=NbZa4rC#$S%$y4g_Yb`qJ_R?C^UuAOxZ$B3Zl&Ey_rJz!756WdgDRKqIoPlIRUDpa6Z2}LK+1c z7b{0^3vTRZsEnP!fW^hRIcr0~#_hbZdM*ZSGjI8_0J;ZKQh#!W)`2g(`6~^D3B! zsye0m$HC^7^-03#u`UQ>5yy0a)(KR(7Zp6O{#w+X5+zJFOA@|du(hPo%#{Pwde*)- zDe%L^ORPm3;YfPqLc6}ZpIBjJREfsz`A4Ep)ds$Um*H_JXk)1T>bGgc>HMwAG|7yn z_x9ev&}45oaKs|_uGPJA{cDBz4P~v}!KH_;DaC+nUn6vWZGmqV<;kalKs`iTbFBBB~ zpR&n5^>tS9v(}^Wtj%#SH(CJv(D(C(81SaY7hcJNecJle3@kr-&#!%FSe{*dW;`i# z6cqexLImJ9N0ncTYDo~k6C68hT09)Ng=9;7Flvpcp_94mi8R1b7Y#9#qj$~y%`D6J z%5eYq04~Nz6fnN97tC+`&1$GhfkzM*xv{ap+Kwvukny`>9~h~@Xhu2gsmW3SAv_2! z*=)6<>J}c-jczg|?_VoQ<6jXRc83ngAMQ zjWOg8R{(kB40bpj(hhlaqM&%qF+~tEA)dHsdg*D2auKotNw9fpfxLXJS=@NP5LvS5 z)=+dBs{@nknguvtiDEfH@+1`_{r+N-^!IAM(~w-@jFTr43%{a;2%?!X2U=&uTPCOi zP6^X;@tX-!N?o&v_Tz6Mf=~yRSfTg~7{DJ0zYb5o_ft2uo!Q}>V7)RTuA-m3#LOBg z$qaNc&Vq@-a(Ocfr-gfjLo88lM80=#GcHQ9O1@+7AyR-Y_ZK}e-sY{GH+s#kTQ~gH z>#AWZm&Jd||L%j(X{FxB>6v;*NQg+`UP`Lhyx5oFYU!0v>l64i@D^h_=O&BH6k?!8186Es2bBB_PdvR+4*5 zaIO;ovyh+3&tC=E-%w5&yR?KZqapwX5?^nlXe>`5fcdb)SY#+7$Uj|X`zx$mH68bR zC$A^oMdc=|k)OG~N;&_`nQYhR&32K(PnwY;ekZ!n$}cA_)f#OOwn?Tfb^%QljwJ`` zY)hvC&%#Y%GLH0SVZ?lv6l8Cb#0dId-v@pQf`g-V_oGvN7wVTYUFa7ZdWe@Xo7q?l zVV32FKI*(6G5$^IjwV_XvZ~zYCGbeqL1;pQ!4oM*t0**;EiO@<5KwU998>Wy44&nU z=0y2P5k5^{(`LV_mnV&cWr_)$DSBVy^*WD0iQ&ZUw68OL;j~tt?_ES+^(GrzAs&IM z;UxTSB;rPKLwZQrPFA(~?8mv1ecqPtycv+_b+9;Sl;i$5Xr$-uF>6kKz}++tuOtZt z@{d6q+;H~(ividfc6`WDn|faix!kedt8M}PwZ^mUd9m2*T8}4CzquJ{p>meb_h~IE za=qu#^WXGcfQui{>4?5+7(q=cJNhC?ilB30gy1^H)N!zs`{Zyx5i=GC5RKnJghK5e zcye}u>#>PV&l%A0y{=l*7#t~BU7rY(O;5wO#Rr>02xRFX`^lItgL_HgoO5OZUAz4t1>L7!qp?nn$L)Jm=c@@VD1cWu36}g3=nUCjVf|AY( zn=2uzFNm*iq^5>-f4l8hfXQ06(C}12YFAcDkE|EJ!`U#A4hShSn{jJu^VDLl9RM=v_2%*#@J{*6+BKhd2|ciqKO^ENdWzRSQ00xh~>rX1zRtQGZo z7S(DUJ@2Q|uH^QQoKT$4RjG~x8Ll5?=n_d_B3v{H3sytUg0p=nA}eV1PQh|r+P_Sp z{UCT@ZjV49@LR|q*KEj^TwERdW6ATkkLD(bBQoQo&8WpL*|}E)BL_rM#-0lL49i6D zc3sIk7Q!o$kmQ5^5ffXrx82d46-@lN7L>?Jvln9@RDlK|lFM`NyT6Y>ifn9#LI(gG zbAM$0ooOLGppmFL8AQG%ndJ6d_pdXym0L}U6wL3L@5ggn$zgxZYi&7?=-zws>w=%| zey55}pIgs3xTMe}b(bA!tnJ`+i2m^q=?vT+-*we`9-Esz9_91?l8WH>cp$br9YL^ZP&MVEu3QVK5W+NVm!HXj-@1bQ92ZnvNMv6+Mm5)%P|2vd8FP zUSgow(%`nWrE_O@)fmZ_nEo{;$imG67#F2y8!0y(;sVJO6xn?f!A(*Qq?r$4K|-76-aS8t zH7Da$_%U{3-hT`6f^0&Q{mH`>PSMu+%Y0d_tr+qnFf4bsuh!Gt&a$s8{{oZEkfcN= zJ$U139q#I1ybkz5_1BLLc77RI5}Jl(4ogk!tRyY`n5wt~Y@Y3u!e*f4u5l6yN@S;1 zra$UT;Yu>aVUqVUheTz1lZADa8%-xE+ERrmmQMtg;dNZ$;`$s19Mh)O+rCf%%anV< zvy?eeKv5pj8fXY`K^D=*g$tD%Mz?p^n92!ru3|k74npSf{a#=8&}lScoXPIiqsmdn zGgZp|MGHlA>=ShRfL0Ya#^;B&++qK6&FA}E%x#`A^HJ3F&e+{hD)9F&W$0y-%B7^u zk5N05E6;647S5swwV&8txEYDko=M%Q>o`XXgFF`s33F8& zo)~}-`2#>D4=4B5sKAVzL<| zS^pRhE*6cTt5p$raROq%(zI=szs z;Kf^!3_Z3;rK#ac^o?!Z6~Xs@t$VjV6d$O1BFXXwIWL7Mm0y44BnBLnaMU-p76CT- zM7cZ$75TXBPP*E!ZlArT{MMefoF2>NY&~}VqY`8+9k_pilIHkL$`<|BONp7+r%w5+ z0L6zuur!sth4)SilJ8#^Y{0^`&(_zX%Gdoro88pw4cgo}OhlucXhs}VBt;Has%*!U zX^2=<0Mn5O)2uOrcD@@>JffnfVX|K-N4-mZK4G>glfS+es%?b>&!Mjug}VeO9#|P8z;xymK zj}A+;)ubS*Cn4krr1}neXAG7)%f?%Ln0JD8)G(xJpwbwVXdul| zJN}awvz;n=m?mD!;PYGFlEmYAz_T!>c$2Ln7-^bTRE1$s^shvIb8YTcX6a8iiWqKt z{P%r1zhS*ES7KfF6R;#H6_#IS?39}50p`s^18@){Mtce4`#VN|iRF0stUvpTQSf}v zTdiVwyu0~2KK2Ve|0v4*_bCahs3SrK35I0oANOP4R}<*@{*+z2O%K}x5h^2NY$koW zGLKU%VU5-rusnQl_n=bJ&g05#zWa0h0iL-98>i1WoGFR{6jaPT_uPxSoHxUBoui&7JZ{E;A#;y+=MV081-6#kC4sTZx=~WcODqy5%EU z;W(g+ws%9=G+dm2uDU4g(q-MY!EJ1}dnv}A=74(^&Nkyjr~+9z1nRd0R($U(m+L;{ z`&mkRy@L_k-0Jo#gaz*}lNkGW1u_qod1J4Bf_mjFgG-OBQ>5+!FT%gIxc$SyNN5_$ zZC(!N;bJ>=LXQVyQHrfJN;%*Gg8j3*>4MsSXO44N%7O#E-15|MOKd4URQq}|Zmdve zlY&=fhN5}(Mn)EfD~>0L=&TCt8s_5o4cF#Q2)9_QojJ=Cv1DwVVLApo(*3I`J-_*?Qksn!b2nkF3r9(_qwZB?e4XB~+`40KmqVS$ZyFgVlQf4R@_7I5{b=kHnnpt=LmkoYxFOI zuZx4@GtV=+yNQN~VQ5yC*LMrGL{h=u!p(ZR2bjI&GX$ekk=Q7TaI>!z(Ga*hE7b(B z@wjWDYc+c%N^ygnWI3>LIN87{bv0mr7^G#qR@y{b6SvVuG$CQe2y|lw#7~;e0&|nb zb>b1h=EoVEz*zx&D?{!az(koK)eM{|iUcCH-VlxiFFn$Ou_+_20TIi99iE#IV+r^8z&Y%TY5doNw3P66qYm6S=iyl&fy0_1 zgR1`j*gH7Z=c}>XC}2(Vu#eiRCj|=fs=SU6X_P z&AXEYG{3i4CHRj&IB0X?i31|$m^1ru-l5MkCs1wI+4?QeME-5JR^@CiKZ0+`dL`+j zI}vU6G~rEGnAwsVBMs#chN7?o5IW9i&RKk*S)FZ!$+8J67Zlpdn1U0~$0~j=vDL@M zLkXaV6A+77Y2ngZ;!x$DGT2q9_(jI4T%#NJ3hm<3cBV(2%*i{Ho-;rym1E^J?(eU($v&Zv0TxnxCE-5;mbj0Mf1Nb6gYqzazCb7{Jg{o$yX(fYo`@#$qhAO?;?1#=h?$v_L?`{M3z z9N4Z`Ohmo~3oaQGP50x>z=2wGSosvL>Gm;JKpLWI=X1{wdZu)iI{qxE0 z{7JT#K#1;Q8;3ml_RId#J<0+G1e@g>_w=2w{mbIizo@b0Y(w%29)IyiL@N#?<3R1e zqI`M@(;vfHY^;R`;s}|1tTN~1^cK$n6VP5uFMV?6uOIX*Gup~tZw@29Z#%nAn}LSK zeQH-h$(La&3JY)w%!P}dhQzav5pXoBAk0jejvBu3N>;E~b0x(N(abh|D@$S$=Pcl` z#e0&ASY&E}*tVo457(2oE7<($Nc5?3pd}P>^{y4c8j3iHn!Vv1%LmKrcivGyd2n%{ zRo3r%_BuYTsR(*kRwn`^+Ju8aNJPX0k@8ik_>P;Ro>FizJ}V!&N%lzUM-J`Kuxi7k zv9DoBi`Hp)A7`I-%Rmkby1T=i$p(|fh*AJIuc1Xnl>HMW+HQqbC|D>qM(6EHD1Tc(TQ-Eu7pf44G_RldDjn7u^_CIiUyC%lbxg zdOAjbMC@^gCyfDB8uN@h;YWJB-k+6`ubZBIJ93@AXu!@9by(kj7Lg(1b_#~qZRnTt zXmhTnV$0YQBWw`to9yV`9|YN&9NyAs>V{bY_nW4{gMX8{uVa3jfzWq1cKY&20PQ0* z#~<}WG+`hHNt8t(^_P$!ee;YFMolWK`dX;(@$HQ>57vD-;&TELnS#+cv+{CdPw}n_ zl&Bu62~N`-KgIAn=s8)P2@_=#dlIApQ(YqU--$pN6>T7cGH5pMqG_Hfq%g)IqyH?C z-QPbL8M)z$#^ClUvIB0AAhC#}w$cJbl_^Wkj7Fr9A9Q}*n&ToL{Dp2tj&Dm!AX(Vx z+ehbr1UBHk)x_w$yRpcKt50ddyd<$8F89^Z)h?Sv_4ls-!o{E*78tDDy#|&-bcp2) z6{PBku%-r_F*L!s=_`7(Vls-7w{(zF%!|-#6NRGLh4|fe{Ct0HDZ9mF ztn%rkUfL;BlItzT$PW2)$ertEA!r%)81V_dtR>@tjV_03@)@LtXMgd$l=lGhf3jBC)QE?{kbP^vMJ5JzKSWzm#ww6618&Wj9jmqrq)=#U)^6n?Km#Omn#MToxR=|z>Q2Z&^Y4Gau~ej zk^6YKO;vSO;v<`fi13DL)rSpXY2`d*YorcnlajD~@jnV^+0J#epx-A?hL zV#H*Q`@obnbr=MJC)DoaSWdm617vbih8rc6(p_*HIZx|`$q#_rKN)U=G%YDOw7LD4 z=oE@+1Ts356oS+!e5^_PjrX&~ zwpQ*JgKHWSlFtU1@&@jm2!DhX>&iRfB@@$Uuq9sTZ{1rb$+vqu;TOU;Janoy98o~P zxQ=`U_qUqd@%$ML;k6a0mhJU$rU}hVPSmvg>h% zcOmbCe+?mNaF75BoH`sBBM)+tJJ>oQ&%RA?mKPB?nB`L1d;Z^yp19sF`eeZNkzgh` zEuZB(E`o(_`ls~*=VSPAjQ~wfCM#z!xxY$QWm-{eJI=b~@a^ZoFgmCHcW5>LBlWHk z)c6-;+0SPtTVB=7F0NW}ka+^v=TfyQvR&7S<=vle@XC5m zshhMQPeyF~H4dkGur(C%`9%p6d?BAnPmm#`~q-j+nn`O4CRYO8xyEu@mxjH1%rwsZ0xKKFT6ZvCAA$UX`gpf z>4WsDcb#(EKx4+Ej6$hF&EK-yMxd*wK?c|v2+{ZYv|+-38qoE-!@tlegS$YeEX?Il zRUVai!qsq^24$xd+m_Z#Rzy7q21yR~-;WwG8sAf_J5386pInNSa6l|m-|h0$EI=+I zDmDomIdON|=Y5m+6kJydTMCxUD*H?He)&q1DYp$CP!Xv(EUN8zGU$HAQ99Nt6|^Pl zv)Fe>p2U8_wa8X+Hm83uaMFtDdT(7YJ4iWIF!JCa6N!XY>n>lAtV5RQ+( zadj?dZDf=?Y0OYm9a0HwZQ+3#SF06*7|&9CAlkJ*3zDg9%#=M!rPcfO^qP5;yy>@z z72A!6_HQ&6))aJH6G$D_&|=;e6L_vX%fA2>M{>aS38kT4HaWZlzx(SQZme(Epvjht zZ$oll+rCd~ezI(y_VVzyk$@5cLb~Hm%s2k*Jv58qlT+^?d|4)s7u8U}|Mb7PdH(KJtBHYI}cTncIxrv>?#P zIM`FyF+$=66nJw^2a=uI;!@QY8F%JO!rkiRP%f%d35ZBYIcPV|OLGh?s_k^B)@a>I z!yFbFotwS?O+4!>jWQNW%7^3~d539+c#^ozrep&&H|JNkLd}y!_fIjG?5Y7}x#lS^ z%(JB$q5M;a5VdIQ(M%X%4APV>H}iS|hThk4S!n7yr~4_ld*ig}pGFG%Uk8sNSD(xD zJ&VuvRb|(2Q=1KQGyTq2O1La1Ovv8q~tugpj!NQ=E38A2b;a_B=x`txb?~h;c>%B%-QXUD0A|f^#*OLu6^u!!QWSX&SHoL38cLDFQCqtP(aoXHRvofkO}4#PSK1~O_G4)v=Qu%UY% zx|@?n!<}{dH>GY|=Qfv4cQqfAOI1DpVrUHg5}H`RB!)Rn*8Q5v+s3E&6(2AZju!+w zse!S@Ym)S}BNV!r=)VI8q+gF+uY1(%_@9rb00ekw!&*=}iU@NaO%Afk+1tUKFIcsz za)VJ=nhe*)@KhX;^%vx+0xjq;3$0uh-25jn>U^XHeF|s7EP+ul= zn!@t0DG{eM^`R%0XH|qceq&o%g;Ac--W%~l5U1M$VBCiU0IE{H@CzO|!FPJ&TnbNFF(qXPZe<*i;Zhf6rcsONb^swF6c(PXekN=<(B#u#k zUqSE9I0V-n`G@JM$ute^`_R)YKmgEh%CZ-OAX|ak>?BnYFCHzHlcUFT6DU{o>GH;N zjh*#Pm78AhEhJOzHrE~>%1+?x`BFZV9-Rn8h@Mj=jEe{Z#ek%I093CyWegUnU$Bwc zGO{YOQR+-Z+06L#&YHrOGg8qHX*I<(>nmp7VI;??4%jNBk;0Z(vq^ozCE+iDj@hUw zK$b12rNu0XXMfZ`y)(bn{-gV4QDvph6YlSD41Yy6m`=71CTrDCy_a=TcKu6f)Cf{z zCM;R&u>1JNhnq!H!!%3PvR#%qe#nIg9mH{Y!3}uW>aeCLQx--Fq5HHSZTXeBT>N?k z&xrMN2^WGg$?}y~3cpzjf*=6SGs{{725*-!C(LdGR`j$33B{Eyn*Ec;-^mB34TCj= zH3vW{!9hbXmV~r9k}}CMye5FXG5XrWd%>bq$)c^wBE{_(%x+s?-5)p04t>2&vun%K zIt1z|@aP>WrM}3$Z2eQ2K{aH+bGZl>{)B|@)P#TElEJ6}y9_`!@cbQ3q9M*yc2c(Y z!8#@3=tp_S?=cf+hU9TbvH=2n{y9}2B~CkOZe!awP)}TcWXl{sd06jx0e)|N4zD!B zsl|5)e0%g-4G-4EajXZvNw1J2BI9(V{DVCE?_HzG5ffUoR} z*G0`h7;IHEn$`#>_@0-EboYi96e|xAn+dAwFPDbf0RbsA6cN}s+c0wE=x#Af<^8m*wkNgq%8@eR2PR`QOti5HJ4~Iw zzBMh|GkC>;liPM|b9{DoR3u7;*3vQ36XbI1AQ6Bkl4)hb9FN%MV26IKG#jz{4TbJ! z8xzr}Kdc+oF#H0>0Xx=@-nhTN(lHW*7klys(3K9KM~ZY%p+(MltvF-2b_r;-=A%Vw zX$9)LdoUr4=c2+OjS|9EM|i6G?)$WVk-SI$NFc6|IhI!k{5*_6rb=?304M05P_7O9 z6vSGsh$W8U#`J4D$62wPf(4Q{a0|=+b6iPJHO0s|yXwO?AqP}2EN5agN%Vm00gmr z#CWG8NPwp=sbJ#d?}%wct{~T~h>evhdIj|oYZ9Ijr4!u~WXemXngIZ!;@;xn9(B_$ ztI+P~i(R_74*LP_PwAzF?X^AQhSYZo-6~jN!`9;rK2}mrD4bre_KR>|gJRwHLhEj4 z8|JoFqLvft%IZr|ivP#bIXKkW_-%aEvTd8ouI*&owv8>jlkHmO$+cXoWpmkFw*8)- z-}^6|``q_e*Y&wF7XGd*fqm+{a9`gCN*^NyY=NWF$l1Kgr;NWC<$|0`D=3aSz^U=m z8#Iu;n|^2&no*kME0QUQ(hl@Me(YE|Ye}&SB^ObcXsjb7^u=kT#mr!-nO9@z4*odU zExF>qWy$4u{3dV{{I^nOD?p>A8EmaC3u1<1$gW~MYv{Dl2Y7gXFe5tGeA~EMiAmq? zVZXL@2&3gnD-ZDgxC*BV`^!Fs9U;`O#T<2KFsA>RMHsc~X}*-rPfg$n5!!4fgKW)8 z%BrjnnDK(XLg%R}3lcqt*AGqjOnqR7j3KM{gOBt^t)s*Rx<>wdX0S>Ct^3i0Ni5=lg$T$%Out_zc>)FpzbCF=*uXT~(bk9c!@}2l z65a-b=Tl^WfBu2*nRl0qzw@3uU~?E4X2UX+X8lCJJKdJ`=}{9ISL-;)iVb>bDN-QR z=|>u4;WP$_l{2LT_&#*rAiULvW`KfY7+jT8%Fe<7jbqu{-`wR!FYLUftySwro$0ca z&IoC!iwY&p9$ZA9CIR-(=a(Ax-1~VWpZ&h|A8IH{AY_A_lfgD99@&9wSrSAb&Q51D z{#^Yqe(!XV)1D8GB}uWs*VIyy5kjT~sYdjpb=lM@b0z<3eb?tkM=LTONIBQiUv>_< zh~I-D+&Los+sX$1OUuY=?0K%EiI_~;*Arij9q46Xs2yw0uF{=BMcH@%`6rsvuQ6<1 zXEto}viZZmx9Z0WZd580lp@Tr#U{IX?5GC7u0?KzOWO?-J&~xxdC*sNlMMzuhzcR%s!9lOW7RB%YZ-9 z3R2U|Y)-9Na&}4uqpWFWF>bjMv7kxR)3(efiV@8HBs@Ych0V{7vksT}LVp(p7hcFQ zIk4C!%2oCls#RKpprcZE-u*nO!5)~;ZQfo%40zH_?wf@wOT3ozv6_PJ&+560I-&oq zw*G_Q6aX%gJP9-3&+pvRs!pprVS~{I$85T>_oFKmh(>zecu@Wc-4yvxTXya->9t|R z#0LBchh~H`xHvp2qT*I_0VG1ZP_KwtD$P)yO<`0?!5kt(W<{Ve8KbF`LndtdW$J4X zp+9C$Rmcj&e9Sl_NTh4FEJp zfCyw8o)V){nY0}ik)o*@oJsGF1HB&p7Ox-~D`iKz%=Vrp&gO!DE5wi0XV%m&I>72q zKM%hY=ykaI3nwmURm)m7@OPC$Q83s3QW;dgG_ablVl`^XX7gf2j_^XdbmKss&s4!@&_Qge(*{4=U~DNdGxxMi(rVhe^5&c#bM?uql=R(a*9O4WPu zF}n+9FG%?NJSqJIrRsFQNHt2PQRy~CI%TRL23S#1tn;(wtYdY z8Y21r4?@xj3NZ!=DMdhSsvvI}N*%m`@_zhD6qAt?kgRY4hgSJm@@O)F^^LpRPUSDf z?#a$O_~c0H@PNaYGka`UuWuA}p12wU+x0-ZP=G+wXe~pMaAkU({^7k<;s38P;(O&r?JZaKvDpji`OqwSs zl-9cJd96NP?6mAp(_ZXiP8yHTsMFov=Yi>)j&&gjSzjuNTny{@mVS;?q&bQ_4<#)6 z=B_z!=Cto5gXwK!hX9O<&Nn|2C?GT%w&^h0YUWMqUl!+HJ-A`4TXCPUBXXh}tXAn= z>}*6YGIec@pdRBtd^PbbX1}O3c-}KAoy^=KsmJT%S1)>#=LSS!S{Bj?%Cs2r3o8h} ze9;Q3sP4k%+3@@VR`m7kn;p-S@%Vhn8n5zu=PNyb+WF#d<_-7B{6kg-?D>$Dt)TgP zc!(aCI%?hjx)qv3WQB8nnP7f6<=g3iKOH-=BU*Jnr+fp;;Povpuhgx4&@=;BX#y1u z`k?F3oFLpHbeKX^F#$zx$o|vna`iIw!*CRT!-og*c-D!R>&arUZmO75+p-$|C6P(i{1_W zu=jPpFYMN&@|3^=wemcj>H-Gof~lb~3#CJ?xWlBuok&A}(y)`?glUAS1n{1@`xg zG~Da}*CPs^1F71_9+e*U+t0f0mbbfF16fLH^MC)w?f>hFR6m|K?Y>I!xC>pV1*0px zo}ZMqY{+o!tC;Zwz7ttY{EP%)4w$47ml)q?V44#%8PTf(Ywc(cIpNVVIp=M7xcViE zfEa_*i)iS9__Vd$p^Ndie?1wOi(9ELv1*euXCe{^>h#C1=f2qu?UUdm($laB8k(6} zlPS-IrF;2jO_SFcgL4)6ExcfX9KV$Rlu z&rJ=x#$Ivmg z6LL6>Oc%)FxUB6{^GH$&>w_W*vp&gs$SvXgHg&iwa)%4VW_WYnV~>FhmD{W57z`}( zeDW?#k>L5#B>E@k1_k{+^5BaRo9#fL=@nbmTKRL*a#LBoN5VEDEe{43)y)eYm!ukF z^a~NU8(gMGz%)%X(Nb)eoAp#;DYs?-Rv_Z}W3sU!Mw?8OJ_0 zC{F*=g0gmcYf6sZpJ}~UApf5GVT`ujr?2DuC@}D}v+`Hh*wf~CEA?FnF15$~!^7u6 z2f^_Gv`TwOWNlNi&aI8pYKmm!V|Vo#fkMed-R} zHw$23^<5S_5wxf9@_ugd27pu~2HdAt6D9rUdb`eBP&Xd?bL%cHE#pboV@oazl^Ae( zHv`6o+{_>8CG2XUy}BSvYX}E(eXv-37w``lZ(yzURRMx9#@}e9Pg((DK^iozkwffA zeL;0&pZkDc0qUkC*H6h5(o)NWgrvcYGCdi$GZecIfjr0|bcoV!3l~0g)rcF{4-DS^ ziloLMJZ8Wdp%6^u&V^B4h8zLNjUjJu5v^zmzmmi+{(G#FZu}wvdKY?Ecw13n>@oIn zGhJl7{*&J=U%X%o)Z9-_SV}*Qth$J#d8N#rSTRzWEH{Io)~r65R6)n%Y)Lgws(+py z%uCln=!D?KrKqcL*uN4%9i2?4zuET%U25GpJs_{SeAKU8?~I?;e;0@Tx1Xl@xB_k? z_XUDc_$FSBsT$KFyooKl%H#>R!4VgQ#q`G^nHoAJ z3GLlHGGz7LXPN8q zB{GxxcZgN~=n;4V%s?1*tOYSi#eX$4lxDW(8(D@bhjLf zW?90YtC|uRD){YAUlsR>5?G#CLx^KfOU`Q8;`P5RGjiEa@ZZ%?7%atH8vIo#HN1;(7F!!!tazrj=Nm z>b|RP3%}_!$`>hk6)c7$=PbKUp>p518@rzFuKxc`yZ8^j=J<)lzt(kV|Lyl(8lhLk zJ{UUQuvGPMyJY&C%{~SUlmf1tcT*ca1-9FqdT|OiG|)`j%w}zYxdw2t!lCR#9}La4#+t>oV`vFvd|x`+4uyU5Wzj8W>}ImF>mXBoXtqx zVJxkmU{4zvx$kZ$awpMwVC6&OrEI7eQTw=NV@`F>NV9_F=Y*ARrSz90A(m-J;YN!W zbcE9ANTb~mp{9*iC8T{ZmcP*uD2olry9vnhRJzg2As@9|K}XD+muq$zLP94y@kNkY z;z638@9|jFBEC^Ocn`v}dzGK0SpPhdR8hq?8Rz6F%IuE!%Iy1-7dH4JZibi`Ph|`8 zm581~P*mSeMZ=)t=b;d_Vgp-yO?&iI2cn5$e_Kq7fb<5QsvTCZr%VCgrpFWTL(9Aj z&iXZReewQS^0cFU)G+|ehl93bf_HmgHnx2~rL*f%*yB$`cgP2q_Qu}y1FaL z|FpXDN84{IzJ9NHoFumL5yOeH5tNF6nl1z5a{!-|+q@VR(6v>QQ(R;ul$3f^Es2v| z)J2uw;Gkj$h{#0PMd;Xy0F>-L1cF#@y%GN$x00gv1ej7-Jy+>YxP39we(Ya)3f`^S zK0Hmn2f#W$lg^Cu?UqIdYtsGsa?|ZRBN!xMWSw~B(-e8;%!8>W)Y8wOAoNVt0lq8) z@|C{H3J)T$Sf!7#(6y-)rsmU`Duzk8I8eD$WgD~GnJ&rNK;y`}n8U?PXZqG) zVmraz&%WDCZO5w)w@s^qQy(w-Byn;sS(Sn> z-ca?cZwnzA+*fc5Ao;}I2E!FjDozfl&X7 zPl$v5p-;Kw-#TutwtYVAK<+v|sLF`|gg1{__ts319LG7B&l~S-lJcyD%62_b{uqB6 zm;~-WZ}+EGPXf=-U((trq%x3c@iTW&r^OiMjGi1lkpGa zup^=65`)6;#u(}iX3*}$vgl}|b9Xm6JITHrzR!)SH#BF=A8HGlh0_?pME34?y2Bi% z>Gz=71;W#~1SG1t)AM7jN+;^m59zU&@t9Z}Hu^>cs$1(yt<^O})TuUw0+@!iN1psD z$++OE-AJcjvOKu42Y*V7SN266>-;#0gs^jTKM`$)ongs4w{n+474;;XqT{s&na_bpn)^$~VUDUk_m=g~EK2aSBw&E1 zjlBa%J8NYNHyUhok6+s2rqf~f`m79@O$F){#?jCj(|XVlk>>v#il$rj=4NV zdy)w{Vh)*PgnnDuJ9dZS=0yDdo)v*i$Un_(1}8Wv87VFH8DxwU1j=GHW5UJ`J33RU zsnZ+%v`R#`e@RxJoVeuD_b;v+ZtcnoW8uEhYqoR1Cl5)}F=~L9+kfurh=uadxvdyi zMh~%9FjXYJXd83i|EC!~n#2$O{BP2U7a@ItbRxN$xA6Q&s)leTQQV3GKn zO3d*7)Dl4px$N-Y`zIm#7H0v2xxRH1n{5$89T&?cI3X{`x+>uxDm$a@?}=AaJtoh8 z%&)nTUXR9Y{a5DBeK?P2tVECvf^TK@&`$>_`>7Z z!1zM*NI!uu4b`_cOu@I-z$7#UM$5rvz|pjZxBL4;KpH$aqmvA44#u9B<1RQj1&4kO zB{jk`K(c20k}Ow1AuHglnuW=*19H4~MVUTrD`nIL`X|Upj0W4_Gb+k(Nd;ny9a9JE zG4!Up`wsCrFLu|%fTc1M?v!v5@}XQN4|R%(HdEqY$yo>~4K~b@!K8n%DF7dt3Tv%o z!IIw)sfO(h;;&+v8o_jD6pB+fs{2vCe%CQM@{zmQRU%&R3T}hnV_x}HmP1E-DFWcZhdGo{`Lky6dOQTx* zyz3ot!`-3d+RyH52oNg`cBS89&*)Wl+snq+-IWB8wnoS>p8;o2o#I%ajS}sQ8RDE# zn?7=yP6idlDN=k(Gm@MN1FNdUE67q6jJw|z>9!2;>vKw$-3!Q*GVg)F*^+GK43>Hc z)*_9*lH!9_6%Au%Ce>^hq|4ZV49mlK}-DcEDIqozw8;b<=V^k3}_0kVO^h?+Eg z-Uhf%IlAgLpTcnO-o0593C-tt;ptbm`7QJc6DV+yaZ# z9JXbadl%BjS|SIy@K1y$rdob@$A9oJDN<$UcDbrgqJu*Y&?F*E4ni1E(1VnP(qJx- z;xJEnhI;8bU-ny?i!I3K5{wo zA}=LLgfj>?JG>)ldao^DSBD#O?#(Rud#5+4xfbug=K@gyKF2J|RUU)*2j5%{hZVnk zd6~PP2E(M-?5$IK=Dym0x9vGzYwE67w|%+yJ8Pe?za|ms4KC45W(=6+&7%sma7K8J zNe0N%rzWy@N(p>YR<_0V3Q(o!p)j#dMk~#Spcun7){2Ooagj6TE&#y!%S;1#+2O=g zsx(@!g3Pfl()C0MJR68EsUKW>Z>k*_pu6+}4ey+fmIk-Iyw$u}_v$0G&tYVFq$10|TKf*n>0wOG$>{42szVe=`Y+eexg^*+5w zgsN_XygT2NQH#2rn_(+I*0}Xf5&c(=g}ovJFZ*HbPvC|#^gl5Q%70oUi~5ud53=-L97l_r&)4{VoQ%)egJ^W>BcLuMULabDzL~dgW$skG4_`9Axu9fi#bpw7bc;|oFMRP zib-GzYb7#gc^)uB6Iq2?i zlG#NF^I!F)o~N9MfbBS=s1NdYCQ4zxt&9se>GDBWqEtM7?d&2(eVgc9JH;5v84%WF z^32Z_5fdn<`5*4t79Rj#M8uE&-VQ_KyL$Ftzvi2p!8EMuD_>62cw-;$ycu541@N+& zNvCP1>*3di^Vx?VOSo)ShhyHoPbzk32+2ed?GWY}ju{?3Fc-xAPzSua#dR|jh&@{f zrW!Fc7GeW4M9tNW(a;EVGz;N;*{fmlp&)aD?4uswXHbgC*kQ_bAq!d6aKc;fLJkE~ zt+Cp?r-V)D)_vbr>W&+vwL0wZujKi1AgHqx5}det;$Ed}r6R;K&6PaKVsoMUP>riUcxHwYNk=sO< zW>N!{(Dl|#m89rQ(a=Q_bkk)*py~A+Vd#BhR7U4TqAO>HseSdVR+A4xHdW(F0jml( zj~P;_bh|32uTTKRI2}d>IWZcs9b(xf!VsjbNr*QeR-qan)SRY-wB9DqXm)oGB$BN* zvz!5AnX{1Ua_|i%*ssdtXRZl)iVe1Iq)MxxZxrHxZQ|!zHgFX^>MM27;&7X9HfVpW z>ksuukxmvE1U;_BF$}`=Aaqs+V`*i&q;`sHrt1Dg8xaWQtLMY3(w65;fI$=SW>Q*?V_rd9`Kb1T0v-` zZs4)eLQF2^+m{s!I)x(*&;bj#yNT-nPpY~3ru#EL*uN0xPkVbDOE&$NJd%{1%#Ej1 zD+M4_SCE{}E$urMRC)A#Qx(kuqe6kF01Ik}a^sI_7=2S&y=Cfu z)5EQ{^Tvas^ig2F$6P=kgw93E8H7aO!06leue+P#WrA1qM#rS_=o9xf>yFQmDRecR z#G&sPT8tLFU_Q^T?T7x@4%zM3MuNo=5VdXXCnWEq=A}Zv3OpLmDdPY@_)}}Ww^-*l z`1S_QWSN~t>e1Jzuh;$)9bvg*y}hU!hNcn)CO6Px(E5QXXY?lK8nW~RN#9-PB?^U+~c%@}WRIZZ5#u!D++W<&W(W#@{sK9R z-;fn`&r(**>#&X$;ZZcAfpBKApErZ`C>b`l7!bA#(J;qn0iVOK8xZs94Evmu;m{UD ziEu`0#wH^V3Wh#6jX3qIW4c7k+u*U&Vspv!;>+E)(Z!*tGI^8%`}p#Z^9AL)xVL_V zpRuQWVaJ_D8#`%0Cgk3{q|FhP9F+JFgQPG}(k2_!R`?T#Sej|XTfq76{FP)(=vR3aUR zz~tg^J``}{mJzD->4xW39H~z&yeBPZE$W#l9ak(Pw32rcvgUOL-G9}><|$>LU%^5r zcR6zyQB|Z=Gy#~qOC*uVK444pz^4YsQ!9VcRvgC=qcM`tmx3HFF0{y7Jje@QWVa)< zQlMhYm~A&uW2j%Aat5N?zr0a-dYWCx7eNW2RT{l_p zt}#CN4Xk^s`$ZsId^r*0JSI7^?R1}86{prFvUu8Hsi~>}niJ{5)mr}J;?<(~jySUhj5|G zgM<>G&xR`48hw*RB@UAu8oy&2Gb2#^qjbPDrOj~2mbhslfx|g~8-%PjxQv#r4(fuT zCr=@5R3p+|x|K1AT1*Z%q{Vz?9B^F*2cV$ZOGZLZ8(HTZw+-0zXZu30KKHr4#F1x> z8y&=RkEIKO(@$FoQC7WC?iS4@qlvj7TkRyGOpAoCA9j65lf{{*$kGG+K+pv!K zr;Y!t9NO&>n<8&NmOVB4-=Uun#X0$B)g5f$5(9D}m&;45xl#3Gw=3f6S>A%iG(KS> zVdE=^;CTY)cGHPPX5q_;M4aOyV!5=4BbZ~4;-XKj^u&me43fg3@Hj$(x#^-R*dq9e zIZDppekv0+EuVTgbuh%`D<7QoDEzY6hT6}L@ZpZBu7+gK;7m)i{H49?vNVM%e9~-P zc?13LGY@7daLY<3u-e*4#f6en&vXT2M17VZ`2yVr&m0LfD$_UXJQMKwnNwAjptS&h zQQDpZTA}zl)ey#4w?~U1g9}bs6sl-|#Re}4(1sfdtefF3S74<>I)IDxr=u>!J6`@x zh-;g%uR7C&*C;utw;^F_k5tqltu!6(aB)Ju>pG_6p5B_ElRzDbcR$$wFJiYAy?o#A9 zj&$v`uU<LK?0Ok{z<4TZ5iTH(CBSE=2!>Blw+E>Tr3ZjkM)QL?M4Jz3#ab^$9>B~mhUXEq zn)f>2Ueu_v@~!E?m|d$2i6>j6wTl!~Li5vo-kR4eF1ya-kloBx+e2N#ix-UG-ma!o z|01)-Fd$p1%#Cd!S_-Tyg>_ktltBH$QIa2l^!$z5Sf{Y&R%q)evdOk+EP%DOa>!HV zT}#3ee_77dfyzx3pVm_!X8U&gM^&9r{<5VktCD<)sr-FHEQG%7kdQV1m&4{o8mS3h zc<}AKg3T>Qvp@bH4N=nui3gpUuAO}C6Mn1neQ?~)60SjS(qvc2hZ^nqtP_zYjx@L_ z)q>=gBpC&l4~eSj;RsjDaUku*bJs>VZLW91SJx)?nJ+AQK4<~&_NPJ2AC(99qw+xh zNpmpZxf}fPLvKyjpYGm^Y*op}s@8#s;%+s9jptkShkuiAe}c=n1gY}T zwYIyxJ&kUZJb!ELX_ycXv10}P=8a%Fv8m(p;5)*1)=Z+SHL@>^C}SV8X_Ct#$YfSV z=X^#;n6!li>TnxYdI6H?7#C+Nm3_VG!l;fS^>C~_AeWWmU=hv1zW3kIKb^LxTP~Ps zjc(g+ecJITaZD+5M}b}Hm;Q3!Ov0VfiAoK2sEUD|aWDY2SrQqB`G<~(@#4=j->g`R zNP{R88&z$ zgNB)pXi)pNg-{gqe`8J*4>NdQepxaQbZdTJrO704e`wy%m|g`A*t4c&e|T-(c>&9# zvv5|L@N@jO)b(x-9k+(}4J8UGJC*Q2v~m)@YmD_=tBOI9!{oqFi((-*1TuUHhtS;e znj~-IGj>kS^~Ton@U zx0f*scR0UAS%?0(;inN{lyPSMiYc5iN?d(^-b;A^Lkx#iFYQs|$vKdm$xI{&rFEqy zBk;XvJxsxgqhsL$ogyVtQ7)J(#01TO)cM@{P3p&ri!)L`ReI^FN|BA*F;8tk;ZyVo z{Y=z+>=DYuROI2Y=$B;R$hb)8B0XHoj$1P#|33y=Z7$d@8I@{ZSj{-Bb0!iTlE!B= zVzQYzdEIyH@)n^=@UXJP4ClBsDeehVl+Zo>qyk1SC>Ufyt*rf$Dl`u`wt?6J)u|%g zMRZdmarjS120J5PL1a`HQn_4nFvW&9nN~DN;}&t_u>IkugA<-B2{-s(9g{b&GC`7R%%011k2NoGZp6w~o`$2JV zE=l&9eqeO2cnW+(rW${!k1D37O)d7ZPEoLRqoF3$EJD0B7#(En#$b-00>eO_7ni2g z+Q*V*&D3km1G`i7i$_nKC&8{$+5`jjI{_i(Ojlcw+>ERIjwO2wGJUcsc+XxAzhB|> zLzx#a{B0!1Dh1C^*1NMQhyv#iM_>orZP>XYzvCwiN)3hVGx~^b{YGQ7&&on0X;uUH z)tVRLuTh_>7cfuo>x~X&Dz$!*pb9+-f`e{6H@B-} zgbof)5FkyuCW>IfjujTq8}5rpI=6n*IEH;x5dy4_C!xB<@opgSwtsK;An89WfEJ7- zmv))Z8@7{08+Vn(UK8wkX%W;v`S4_}xe1$QIKE@UDVVIT-4=xRTJc~)h1@8Z(Xq|Q z1~H54v1SJHcK37Tw^+qlkO}8@bXZmi%~zhl72>AXv6-Vpvg0>F(c1_<){K{KJvct= zj#s91p4TolSq@>-YBRxB9Ho2S_8cnuHDvC}wPTv@`_ibqI{PUDD$n5_w?~EAK;1@e z0Z*0h&TGGiI3XzbTVE<$i_27PMR#rb{zOUa{I_oAozt??sla>HxSvLwra6R^UvSz#i=m?ZRlL$AO z(A?KcQjZ(Zfr{5)Cr%VTNeH-zAegt`^D|Ct6yp#l4|Aiq>H z6s^C7VdG91YCp|^k@I6j<7xyuPE$8D9}+9%CNK9(%dVKm8YYoK(eScfJU1$*BI^{j zGnLVJ6}$4aJdj$*aRjg7XGW}U9Ih)W8Y}&dP5fDAS_7KD zL!Uwp)Oyq!MDi(CdiCY3vfLO|^!ur+5{e{hQ=pM`QY1JE$qe7~!S6Ddo43KE0Wd9CJK_|B>&HMv9)RDDiuOrb%;*XL&-6A#|&E+4(@&_3_?N|50)Eeev%BV=C+7 zv@RX%g3f35 zB&G?$Q>_M`{W?2!+$%w&GE|<9NTzYyZ{N!t=-3uAA=-#UW3h?pXl#s5~%JdjS^kl(!VFP{}K!okz3Fl|w?iZ?I(8TOPf3l`fP zKmN??g6-xGmn~hHAuMhZ8*k$@W@Cr)bigNZS<=kM8zoNvt1~ ziK?{bis;`v8;FzD9WK>XP^Z}_G(!EDG=s&s#0lUrivi0yO8(nNEa=)pTuszLkr=M7{{AX}P?e7NQ;O9U)8p=4-ff-S- zrvYdmhrecA(W^x}4Hwx70%ERNL}DxdxJ&v>9jzx+O7#t6cj^#E@R0$k)`73=2r9xi ze_=yW4VS7LPw%f$3!|@s6RNC2D^UvNC@xGp6=NcR6kA51Sa6_5zHgRE$RLf0;gT#9 zDr3xBv@$`9FH4lw9?}`Z-z?#=jM1xSEodzUuHk}ZZl`~0A3otnkqQeCmAL0uO07{3 zI3&II^j>g8^(#m7ko>q$gHWCdNI{Y^ulD=rNZYsVHTFG<`;dObW_MTcCwZOd<6Dx~ zCt0m4ba(~JU&EOC>dQJ5|DD)pduZ$7D1!1;ug%%yLw9`v*{}Pah!2^D1MtT*62UTNhbXEd8MIiQE zpw%Z=6yDm^2|M={zd`-E&!$JYlH`qj4)PyG*|=L9YhK+kSVal41MhL|CjWarJv*`p zAlGWW%QsChL40K0nsuo5xP>ULxb8ez@OcS5>2`$W^mz+E+cRm^L#{wc-2~tzd%U%o z7db<*LjfSUj6K5QqN8i6RUq=K{ABVW-EF07Wt02OTYEr+M|B)XQ&mvoOsWVJOo%*s zwu+Lz$hoPw_{{ZCeV(-W880yRtpYeQ=DZ**FuDegi2a9VrG6LAlABJUx8Pe0h-*f# z4&9laD=c4Oc0xtgoSfSXT!a1>O>ne`0~V~~x>!-olI@1; zZK;n&9oEPLL*ZyfqQ!(n1ESdp2*lg68P@crej@o&Cp8WJ`CDdMz>*9zKAkGxZ$lST z@h#e^ID&>8Y5av@RUw?1a@@4a1GKFaPW?x^bb4btbj|WS@>kqfqCFRgJsIp)xzy+R zV-{q3uxY6njRFGJ)Rl({+AIka*lr8Lu2C!Q5(U?cFJCv{$U(;2J{w$;;goE)aaFCJ6 zv+7(J`e}kNB)wwCYd}I7G&1Xgntw`uT}-o7*dB}*&QD!dR_&DkjIm61%mxxU@S9-? z&E??Ajh|9>T6pjsY+_qzu=9rLU0z`|KCwFCb|p~Y_3^I+6Vbo?ZVD--^1mxyxVQ?U z)~d50o^+{S>jr$IvvYl@+M;cTlcV5)Qh1U$pC_&f&V5Yx?s-98X$u7d8G=#4ZRFzF z<-?4~)7Zur|Bas8JpgDXokKY6YlrL4uE9Q^9ae%*XD#>W9p~2@A1t@2i>YWo9G}Ea zCfU=#_;p&Wophur)?)J)Bjz^fnxgp;*VV;{p}?VLy&)v4q0boQ6x08d<-S$D&!UTR zRdMx7oTxtFa^YJ!eTGkEXnJ;s16Y-s{&Ok9h-ZjagTXGDA2R=dZ5YLA|E;s=g&~m) zrsuTs?L2obhLL?8(K#7Tr*m!$o&Jnek5y{hwl4Ne)zC69$ZQ(rdiyA zey@1XXN|nE*9t8u%?~{C?>}8W#x2L&aQfim;vy!z(Ph%iCtJZ;$#PN7{)Gsz*5Nm8N zsi0qg1k!~+Jjh6<-dHbYA##fWhFmc(BwEY#@9c64^tW~jZDj_RFLqm>2tBTOhzmY38dkiK?v``%+$&`>L{9H>HW*tco0x+t-7A z82y=O2jCo3)T5`TXkWuYsI$Ownl_wW zRmZ5Gj5H~c006o0Eg#h+TSKu%tF$k7xtz@yqEM~fg{n6<6wM;Pk{Pk>aGi)OU!P*9Kc(930P&if zt;tQZ!^rjw{%th%g8Yh|K;AP~Y0UsA@Tdku4qhA8>2I$?zaB=t)nCnME~IvT1rzjL zX?HK5f5h(P-ZE`>D!1(Uqi($uJ-F(gm7CKpz;k5ezXs^81(if0nxHjPU zXmm8x4={8rrnu!}tI8xBQikK5@+3^;=DzLZ4J@nm;^ESU1!K>mL!YZI%Lxpl#@)#V z1>K0q$cO-`IH$3p#4JSe_Y;=QT%q|e=0}fA{jTX!S{O(2!-&I$eq;p3PAC+q9!lxw zHD(uk%9sBA=PVP6oDVI;i7zdY1}`EryR4{=AyWz)9*TAoRa6EamypIe-@f@bSzaXd z%McgI-1o2{@Bb!W1SQgow>sY-yBRhJaR=XAg8tr8r1uQR$x%ut zZx)zPu-^PB_u_-HKuB1SQnj#R2^KFUs*Mv-V;pN~2bY&8SlkmLHH%_o$ zUqkRLbB^_0n!BKR>g)FvOhTxD_R>u{+BvZIadg)IG1fcZ`z5G5J8A*!xN-kT-fI%* zc)cGjec5l#A(X}T=RsmOQ$^`?T7|2gHHwUp=mu6h_?!@?ZB4#+K(9czB;ZU!q5~*V zF`ChP<#@hPizgZp#Pj%ydjO*3`6HOxB^k?ssgd_S#xX~*^Ylw;?Z??JZtiM{1Y?`> zI@mE%NK0H-Lj3WmlEa|zRX9#U0K?#0jDGs>YX#odMbXv@_)&*HLs2$dteZu3!~=hw z*3?Wk||;hpEXy$RlroM-}gU6hnxlEMPBLf#od$xXI_cTeOEk zEg34e@KMo^m(x9c!aU=@#S2a_z?Ev#6V31TEu^>jJggK)Avy3ebyPW*qf-tHtS)oz zI_y+%VMM=f!f|H*^TL7ASVv)#K;R|K#jnSaD)plGs$B`AG_O}blZ!IB+?+BK6T{GP|!H|z~G*Dg!UEkyNGsQY?T_U7y!*qS=SkNAfKGI z6vHT~#l1`v0~{9DDFWV}ZEtyJ*q;6l|cqsmrw-jZhwam6BMGQ*lrD6rI{o3<2;*RbHg@ z?eVFSQk=13JMD1IRD={_W!d|`99@!Hk8ysrsNXH;)0QG8lvMDXnzYs}Diq3(X4UFJ zg5hd{bvM7@LjtS?t&eo`Nm$+2W)<`6FB*+k`wg3@Es?Ad)sua)7s%m8~lj>&!fONV<|5E^NqINwQqZZ`w#sUhBYN0#mV5m z;`B#cvnKO0ZLrOOPW?q6ym_tv2{y|y9ZP5OHS@sD+aL5uc0QTvUFQna3XzSZ=h6S< z>FUd@7@w)dt$5gVYSkj#b2T!U7`3EtC3Tbp^IWuvhBGSDWJiS>T4od`6;Yz>UR|iP_CN(csVk0~ zYDJlU)0XGN zy{|vRt-n|Qg@hNT^;nhXaH5o;Yie^~w}6#1CHxwgti4aqb%(6QLtP$($?c0`w`ID{ zX&6mqR>h8yOWn9)(Z+aNA9R<16V6t?<;971)Q*{`M%Rdl@rn{+RHGCt#r+07@ zJHMYkZuE%5VXSx0sF}q{^AzT63L-eGU)`iOF!Mwkl<0g>Gs#ay4`)MSz?2sg;YguF zKnGF5siV%YpeMQtX#|!p=dl5TJdnL|;W;P>C3FGUc*Yk>ZTyOK#9W#cIDeAA6qExU zXI%?4E`@mMdEnp>VV}Y1#k*y59B#Xle;G=OJOk?DL8GKR)sp#cVl zgsKCT1!&5YZB~U8Z-Rr^5~U1;XcE*k5Zw{)iFk&bEfz5lY3^C|L;|*ZzqnzN_&)Jf z>UKf&hM{b&Ji2M9P>!lfXJ=k2a^u6tFo!FfFwo+`aQzf&)9=w&5G?0XXIL(9WSK+#86uY}Hdh{?H zap2y_q*LhcF9vFjkkF8sSR~Qq@SuRq9VMwJ<`r*}y~B8b*-%V^$%Uh4_7Cmc5;);#LRJH@Glt(<5mTtEmB@H!1 z4aR$2EBqwBMy=zOY%H`i{9C5|3NAk99JjD_4>$Pkefs|CyslFj51Jg1`RzcFaERo!bYg!3YRVpc-?n)=3S; z;3EbAvFAmM3BFrdg;ZBS1=paQOUn*b0-R*=_0RCVovHJa5Ntxq7q23MU+$7*rnhh%`SYYgnjqe z75mTK4IN!wJUwZUm&5034mqZ7K$f_!gKRF%-BHwZz9BypzlieRuvGn&X#5P{W6dY4 z&Gb7xFPIX^rtYT1IiQzGxC97-hF=9;%^>GiNS*l}`uqEVl!K~Y!40?F$Io_CBKWF# zg>>MLpE?M#qJQ(0w{7d@SXfvzHmqX9qM*l&;`y^M$k3E=@b!~M;iHESM=H#i*9cg#dL2Ia;laN4W6A(tBeo_J=VkBfFxRSz1i$q(5^(iG3p#mdGvah#S>9W55tNvD2oPAQV zP<5C{9Ahd-)2gm%d7hRfNCL6iZ7?X5bhW}TBfSXIi6WgwvA=-kA+0#{puI6_*buB( zy%yO_RtbD60t5yt@?H=)G@Xg{g(P(0!B(`W~a2*H<3ItiF>3DtLCMa4c znW?A273wx)MjyrfEV_r~^^%X2C~i)44@WL>!EF>>BQ@x*s3E-8+M_{EGg)Q=eX7mJ z0o5G(_gJO1$wQVhdg`QcIP$=~v1Z){tXi`UYdSj7K-f$-kOl?MoPvpPGk^4NnRcG< zxF2$lf3308*;-gwY!g^w!=fSJUizZjlyN@NU-o~5L5PE0BEh`deva9MGr^^pn4e$D|Kt#Lum-CE_e$Obd~@2mz``qmU(N2B(PX9~@h| z6~cdACo@d)^tdaCYO1R~@U%zL8q!8bHB8BYqz++J(cq}H)Cj>fG#(Xh@R&1-n?|Kj z!XCTrj05(ZgF>MgxJM|;dsk`^*>^UU86NO28z%*XQbo2gs#Fj^vVnSo>5wM&%n%E} zKtY8E(v#%5)WEM9Skpk(nZ~lVO&sm4RD=5_+nmGNxs}_k7DrX5dO)9d z1G+ltB!phGAe_ZlrLy1&Z6r5nr(#W4@5$KSdiq+2f+KM*5@uWWIxHuj&0_D}X7Lo$ zjNJIoUH9?KK@Eb|ZQXz^&2nQb()XL;xF$H8-hk<@6;y7=K+Mq}PbrZg0(v24{EoOIzQaN(!FwOx=JVOYxGQ@n_W zE)*hqqHdTA7m46JRs9~b$wfIrnH|cqshb6NO(DMV+-7Gl==`NU%mN_YP592p<%GDS zgc1qTJ9GL$9ilZ-dQFucqWQ!~sZZrjv#10r7%54Y67O*3I^0BE_+?l5ch=va&s?;k z4Ky8p;Wi@OJfdzW>h)lym;HHQQ7x;b3Xn=;uYKlX)|3h8?CC)&&BzOJ3EA}sNTy&c zf{Ih=G!Gmaj1%2QrR`yRh`Uhmovh$X7lh=Xp^4W|TjAFYYB~hbHB?E1B=rIfh$_ZV zt6lp{*d2n|XO$|m7ks!Z_p4vQe{N%nE+18oWtj=DoGG)6yzu%(4QamczEV43ZkFg@ zthwPNq;pqEy-SUC?M9%j68E#*W+1R=8RENfv0rz8(dyP+J;ePW>(%Oti3@0dN&BcG zLlTvyIcre=n8rtzDV~AiIVOBIjxz>OfV{Og3%DqBS9iWv0v2o1bWEQuGTz_#se^y0Lq2QSj6(h=2U5>&SMI;sMoSkRj;C*+;h@tIhAT)352GmE_ zZAPKYBhg1Q+}syXTsP0ZY2rSwM@1GE7TYH*v0<^z(Npx6_&x?*ER92VXvWi@o#ub+ zh~YfbKqG*?r2X!ie`C&Z@4~JBdkBpVZYV?e?Wim5!4^$@Am#W%Jt0C~sPQx;FF@Xh z2}KZFC!{ajx+qb-DfV3Pg5=j z+ojupAYBN#!QyIU(rLI%y`dQ5hSlJNMO(ei264rvfy_s>9`S=*_d4P$7^nhL>Ofm$ zAqx27vs*=~+K+r|1~>h89yWCLz;Ruk6(U%jHW=-Og?LHc#jN%EV2rI`P#u{2RfB}m z_i4n#=Az#C8{!$E9F{OQ6heo_Y^l-KTKf_j9^Qk7@e&WtTAY+@d+k~&Ss~AhYflX4 zC`~mrFy8Eeh%|ZYN7u}hHHcq6FCsog+3l?@m@{h!?ATbbHk!6|1L{={ZGHtv!%6Lj zRPG-C<|()Nx6K^!`c`ORVX-~H5*rp4TNy1`7e9U57~FN~Wb809(=cy3i?lN@%Dr+`_1gk6o}WtV1dfqO;V@F$QBRwk63>3F1`Hm=^qK zY9NWBimXuczv?2h{Nef@m?pzvhIF-xz2@$UJ$Bv^8+v+>4ujRmdP5fnk!(g3#C+oE zm_=7LA~~ecK!fL1xgj4lR9%^`7c!GXg3)R+d`$*@t$k`3SDj&qbWe#kq2wmQpsT)z zbhU_F2IyR|1h@b1A@1H&tul9%xDm`t*Li|SN)R2#1LcO~Q&j1IR`@0b6qOB#x)wWH zmmz78&8hP`%P!QCOVNEb+{9`PMP02Ss*e~ANn(O1b&2$?Vq2)zpwCu%8w-V(RL?10 zOzMAhOu!e?e8nUPux#g+P)r1rV2!mUx-SF~tCQ(U7f9bSnKX8sG6_5FFc~SgVP2^l zKpy%Aj0Gb%eB|MzkA+uVZ*TY?g{7N&ifxl!$ZFChRO%I+AnrQ>HwK-M8vZoHd16VsW0z=VJ{4bzjILqEBXGG6_~fk%Px0i$q4B=B*$MA-#MNcY_fdE>JD>!)a~9-uukO z*b(jM?JtIeMc97+Uo*M_9Q;hBT)3{9=2gxWnH@*8*@=-Fl}bfR0ZGSNxco$t_<9r; z^%~cDYS`FS(5|f@FO+#Im$9W88%tez}1{6NY)qLE$FtEG z+&qK}f&A0v5O-%F42V9*Y2VF5W*{MI5fj?B=(bJ1ZencZafz)#JmU~`3DeIHud^T( z`E#R&sw5gi0|7xu66L8yebs`FW zsvI24Fl0j9Z6$i1DpyiADR3PWeF=zKLwm(DpVZCnfia_oW7hO3Jf(DlK#!FC#^a4w zue3O+{BQl6r$6l9I&-Htx^fE(i^hN@HY_Z*0@`vee*X3`xbs7kF@40gf2R<5ZOL*R zbYM3j7W-Fl?YNo7(Ib8743^wKMh&}gKvBov4m9MA?5h+3VoP4 zaRT<)ZD*7#W$r4$(zpusi|N`jcM0}0osNvSfd-TwQ7VSgU-?_jonRA)7Jm39?yFU1z31}d1|Z@hVoD<*>OqY;s$^K%3~V(e zM=g*Cm%=@21Y9ng#l*3r(3EQcp+v*H+rQyWU`$^cj!l z%{ri~;Zd`~nhdsF@`|i^J1tuVV5MU&f(S}mSfoiMou^X!P>mv08e}}oel+Tz!=z7M z@MD)k@#yLA=W#Q*;jJka^njqjinaMab-|@jl^Ehnu~bC&#%`3$C|hEy z{w8D56a-VhhKj<5tgsu-kJ_CS6DCqh1V$toRJkHF$qujd@ z*WEmi$1M?ahdpZsayyV?Bo!`d8x0g>Ft6x%7!(%f^rSVQy8&rsm;zhIu1jLkHB=9V zwrN6ypp1m`#5I+5Nv>CoxgnJuLXDBwT0bSm)VajFdHTuQt|=B47H>2xv0-7cnHZ9H@$0k4;_Yp-wobbXfLw5wntFbiVHOU%fHVDlui78AB=^$fm zGdIjcWSG8dH5v}>r-JW5-b1bJ?3IEbt!X6y^YkQ_?7uLunimz1Mxz|xBPsVs_R z5838c?77=43~g;epL9bpx105ssVul+Bf%h*CZimh=(xVVKCEBYf&M~&*nT(61d`O| zI-$qVAlqURdp$TL?E}d!8c|U3X{mXDD2UIqsEv$F8q(GZcrH9AgQ}Z_hZLNqCj8-= z8?kZYMm|<7)<&i3@k}X7*$Hk3(Y8%nZLizCM;Rl-&0x3>1TjGC3Z`{}!bVS3I#849 zJdRR3Tci9p;y*}cLP~3ZMs6iRiMR3wp|1988wJ^-ccrbZg)5mXu_2qVV%yct*|rpX zZ?xmO*Za3j`-^wS?jyHddn_z0-Y8gN!@^=SaNN#ZrGfVzIJ99@8qCP}!R1$Duahsp z{f|F`Mu++$8z}=gxD67n1PkRxj8G7PI886a5v^f-nGP&U)z&c>*i;dso-4j<@@(A~ zUR@6~zC)V~Z@MxX^&@^>yg5`xqHd$HAWRUd3`%-LflGyyCA?BR_+zBi(AyCa8o??o zHPNGZ3qqk2kQhRw5Un<|Ko1U_R0h>@71JkAz#co#fLEB9Re!lFH|$dJds} z6N=XQvxWa*i&8EdDkX}EL;^`kZ!L;31QR74lYO9_t<`B`t>l+>{pGVnK2M@!Ss-H; zNVHLq-SLoZ8t`M2V-3At;O;4@Go5PhbN($;-tXUcw6%LIECw;`4Q^qv321L}@w;=z z;pX>G!sKC%6H82#1e6mR@WK~z_8Acz zl-vkxQt^gN<@>_m*Gw2HY3CoJZmo4%?2fL_bKP-h&%gsIn{P>71q_!E6M zhTjNR@6{Jc-E1?rF6ydqEs0!x!~u$7M#oNxK@wRHI?*A{*ZH78!Y8W5>x8(9_p1gY>BeqH6T%E;UBEJed>4QVDCJqQ%K)|1{eyt)Pq_|e&6pTwmr28SQ*c-SR@Rnm z6E6GnzqqSTDxKn_4@;%1)F=L@5N+hzkO@JJN8>medSB~0DlRBGb7NB9k z$PM2g4JY#hbT7Ee`|mDuwsn&%EG%9hSYpFso8q3=`Wk16V*s4EODmrL{B-}q{o5Nx zrBSI?@$Fyy345P%0Umy8J{lg(0OFYzOhy4&*=EvqzZ(8|Rp--T51U|jLhF!{ z7bZF-NV`h+(^M$>@M+hD*nw`^g;%b?ZEC^X-FL>AQA1HGm3ZJveWl+V)*={l{=ou( z8q!d*-`zbuSiiml8rIV;3Rgvv{p z{{uHswuJ>{oicU=oHSePO!2XB7=ghYQkwF3Y5ohi{jLXjW)v1=l#LTFKIYP)NBYnj ztxw9t)%!ZN^@NCv>Z;M{8ujUrr3A93dRn8LJJ1r9#8Jr8ErKe+Zi4xCc9x z(!>7ElYim;??4+|)WTwr!4ex5+Z3xeRvYClAQ8iwQ~3S4<8b5qCt`eim|md)@XU*^ z;()hYj4%E0auiEtG&GoDluC2sh#O8KOGqFE(DcO2g@BM5V5?wNIDQols&IV|4!kIP zhFqB=E0qZ9iB?sEzsM}4hM{^%6%{vGG1LQ)N=87sGpH0uV9-TD8v|K&xEfn4xE*O< z5CTMRyNcSbB}!-L-k^nY<&>ILbk#qMsw#|m=~^PTle7G!YC_{7vcQSaYaoq6w-=`A zVFX6G*pI1WMqtiPGf-u5GkiY?UP(OU1=&EyQ3SOVh8&NPp^XX4uCi`@2f8}D;8n$H zYm`cpDG>sBdE7KLt>XI$YJh0O2QYGVw-tp}qcc^jS8`6o4AXBbvO=x&L9gCp_ZHCF zl!Lb+*8sSeE)aXqEht(QiF4m(U>brhgf@}l(hP)xN%%e^Gbu9L8jEG zv4bP!gG|v(RU0+xq4mX@NrO{vCgOoJ)wY@XpwFk6D@uY2g_0b~vn?bVqi0d{Jc?rI zXlYb8q0iJMF7`))hequ;t2GkG%;dU4g4N3TB%VvOZHo58L>{Ve$O6NvH617x%MC+p zI1YNZZ3`M>95jTnH~c)@?0cR5l@EG1Pd$EHH_F1oV(Y;Y8y4FX-9?c{u`zMVZmoFs zbJK9ae(h)!l*(m%{j%RC!Tu=4G%ZN<0SAEaF#|g2%R9w4VS;niWHwAX+g;g z--F8t4OEcwD@e&twoOF27X`YZDnxvzMzf;N4-KJcd(HHLLZ%<-59qGwi(D#AT4NRl zLVfb)sFBnpu6zAjtGbrrgqCd}L+CDPCjZ63AY7CgRbXrH1UR37bCzp@e|Dv zR4bS@b1HV(VG3UnW$q7wAru^^Up7_h4sLjJL4+4;8*5gtL4RMLIwB-1 zPCAo9mZxwGR1CUHfYyU(wYv@jSD8bKD}b^?>z;OfbcivPWPsTkGtc$VoMWVs>!2x@ zL8^&mMhWgE-${Ypi7XBYTHEl@Ll5EhyY3BIo{3~lo>OD{o<#j&Z`WAw7<4{UWP zu2=L?TI(sQ@G41~xTyS%-Mx5W!D9Y=Y7mr5uiU$B{bIrxM-7W0_Qq_-P5;ZkdCH^y zopYyd>n2%PSZq1A8#jPj@I9UjkpGu8{0|Uc3Z`p-+pcvMwXmj}1?X!$3~x^1vbT-F zxqA;mBY_ba7k&EMT&>aYV8j9=G-R?#W)`s*MUNX#f#zQrq8@ym38?E0(E2Q$%kFPkZ z-tS3qF~W7wkQ@j;)Oa1yvBKzAOXt;V^+!9jBW2J}k?K>FOJ3P|pM{EsR4kPnl9v#A z5=E*(dmWBua0-j1A+;+}JOQD$iB9B^LqocqW#(|Q%KC`*;z%efyKd-A`SF%!>0b3TXZQzv5KicYTB z2y`65Sn>EMnLIxKlRx6K`5SJ`2hX<`Jd( zqT{r=Xi&~5t*TWc{exxwrpC%rdH8(;8mocI+C-RTF;!FM(| zD`R0{F{p_R$8{B3V>8w=Ew&M!T~TZ}(&_X$ZTQt$WBf5~jpKI0Tur|Dvp?W_zrKpg zUKM@V(8Kt6uNsbdekt?*?9(6`cym**)+S}VVYjeIt`4) zkRb9sq}Y66X<^y96%YOWJ?QT4MWtG1&PsfiaD!YANEj+Ra=Fk2o6F?m2~uU&4`pD2 zJ{gTu(cD1d8U(U^f={tqcvOv?Nqo^#|hn$syyxr zOPRcK)fxi&3Rq z!Ga~rv3S|4M*45_#%K0K7$S$`7VO- zNdti{`ZIzlbTQI|05q{73V-GDIqWfOI=n&&PQEROb;Ir}uM)`2%T+k}7JT$O zzr*fxcE+&wHZ; z2NK#S&M_L!2`Y?8Xpl_880$2(Nsh4~E8^p;lN$Q&s3;GWN`(j9-MFy_U-;@}cx?V_ zIQ^8vF=66Z>^XZDCXXMBk;B^2l*{oLI-&yZb|%X)jEC%Mjo`TTLf#)yUzP5JL}sjh zL%l3Z(=#u;id7vO&_L+e)(r?Ee{WY{HvoHSnFX(KoByAQe?!}jpKuPpyK7rF%)-K= zepq6|!UDkC=C3-EiwTA^{ z^pG9aiYZejW52!T;_&@uWBQ~C*kjI2G-cA<4$$roP6eb3Iv>erU?cauz07OCcJV8kw+ziX zCnjg?Mb_k7E`B7fNQ+fntsq&3vdWwwvz7u2x`H2zVL{s`m+0jxBs25{iz0~_u`;O> z6d~av(;$j8Ct~GYo*%R!G6Gb}Cm3R(1Q1n(j2|-s&8^MoXZu-<5>I3* zVsoYfC!4{_#joL;KluYbe9_x6Wx`l))CvlYxuwLbiPueC6k_BC$%+x|k2)APe#!jS z6bWY1gbdd}#E0r95!+v@58 zwr&96ui0d_ura_Mm@P=>EavF_btUvTH#M5FYT__exT#3Qll_JqfxKi%2`4G@8 z>>PFst3R8~naKf75g-*mgb>Q?X~N4 z+&6z2e7_3cs|4gmz?8x9$~^t+)i>OV-FMyrM;^K_O67L6wYDIiV{T$F##X2oagxA5 zPP`s=Q-bP)wnq|bn}K=*dg@P*T3lAm`Ua>(5{mi>hUZ(YEKL0QoOZ!HGT8Uyk3IxD z?=c%ME?I_BrGjOv)?!^pCmwnBMeMoDOdNi|o;Yx?+1O$71hlp^ag~%wb!1lVYyl`) zV-(g9f+vY|@_kNbu-F>UyzpA1Ek0~}V#7fXe6Jr)s;QCQt?C48ZwgL|MUDTZf8(T6 z(KPZy&QVV;-qsDXu&}5NOKezJG!Xjw3;4nheuJO?>Ds{9Z)o_)_(g6AmNo`%0#j!1 zit|r99;Y00Fm{?T2`x=|Z8bc|oWTC%iYZCuXI?c8UNHH>C0k7oH!?r&6blZSPNTgo zjbo468%G|t2fq2qi*WNT58_vU`xhR3_(`C&5&01m6a_Erx@$yfTt>ZTBiD67(uBqO-qQG5*AzTgZt=b1ZG3Z+6n#*H0` zU3Z+yT_4yAZZM{zRMDo8T9B703bM%j-+6f=K&UWcs5Qf?E1L)d*C1Ksh;i$>N5h*>Y87{@HBchI7z9 zyWqa(UJHX~R5&R~B`N&g^s4yjRo7$q$YI!F$~f#Wc_Q-J_UJk*aTOxcB_at#FyFW; zKs6?1@71vl$4yr>=oF>>N%fIWA^v`OPJw5>Hz{TTjxK-D@)V`%PQ(h#W+px~zyo8P8#Sj>>; zURr>E|My<3Uf%$o9|W%BtlHLX08cmD{~n23fKOlaP8@gi z0T|Mf3xYQC*BGYE2(t|E4;Rd>S{i{gAfRdGkh?jA`{^n_lRRtjDj31S4A|kr+i~_; z$KWk*IRcM9@&bNz#b0piUH^kqE+WGe94u@kBUXYO4`D5VX&O~4a(ak1MOLAo*jA<4 z>Q`ljUkj8c4-`2p4DWx$xKRiKVceOFB=HZiMUihdKY4k`;8aoO-y+ycAO13xn5ZP>~5l;=2ATbZu zVkjhl$eJQfWy*u_nmU2ep2=pB&9Yb#8cPrxU_<@%nrUNZf-F73Z6ao$8p?=hiXg^` zDA-&t31q2F#Hh)^*fN#N{&Qv_RlT0w$y~Le2j}ERXVO@@cnPll=dF0(d8cFI*fD{- zkLU3+g7Tc2;};|IjIl?kVJ#UsWiOwH3Diw2xNfb$pC)KoOyav$BUpJp)hKpA)d_Jc zQE8nznoP=Yv@9i2d$p&MGH7*Hq%_yz&-qH#!^uY@s62SN$-BpE@?dysR0MW)qsN$2W;amvWc!k{J29 zreg!{d;A$Z`t*EGax@V9OeVcxTQ|Zz>d68Nqvqc{_I9LN{s<2za23S%2^M=}7##Pr z=y>{6?~dI*v5nE)+lSA6_gDD!U#_oPqP#8PIzFm>ef)3E z0SDjgc8!TWRgTXVfvjDKZSS$$2_1IGon zQ=tA}EBY+Mg?LfT=h8Ur(78D9z}@iZBhTWSzy2fc_}>HIVL)2jct#fC)*$|kk^vf) zzBLoY7*a#nTH>vk_Dks;jR`@1i}SS_0~;=MpCJqTGQF*=PuGG7$fyUTr%@&GX%~?w zG{!*;WQb&NRVe}yM!G1E_9&q+zskvtz--afDpAO) zeM}|5_tU_!f^mr$A!XjuQ$-(0dFgZ-EzNn%-E}G+dF~a{M2Y&IN#m{u9>HF-cf#&F z&*Wuu7azkp@;Rqlp~@|%yFi8aX81i&EfE5eUXxHoRPWQ2S6PEx(A6U<@szxecB!s* z=!y|tkz**!s5Yr(_zQAJ&K>PogHl63my%||oG}!`p7roCcGPg3a`d5iZP9YPws;wQ z889?xXQ5QWiZ$zbtcv-sF2o)8J%)q#o{aCu790cC<7%A;Yp^h3db=3@`L| zurYBQlEx-sC?wc_XV!Oa#6yoihX)^j7E4#IL8GCqrS&y52%HmFF}IjgaO>2Qe59^| zkD=H;!5%uza5H;4)xz!mEz|$$582`U?m>6h;Gz~54FpSUSXgXB+cse&)9|$qU4+w4IT|gw95T$!!}n1M-37!eue8#`)m3T7m?Yb=msO2a zATssfl+c(BZ5?Uigz6CX0_wURa@hCOl7)t~$p^PifJOj{e6Fu<== zq(L716TEl_ss&0hXqK|bJR%QKNf*60!t51?M%&ar2gaN3{vl)!M7bf3qsC#_9)*mc zZZ(QBLmlcUJBu)?!uTc%lSGb}@`z|#G+bJ6Po0el3!`-g1{_=A_u-XFyo`wx#-gny zk7XNskm0F)i8_R^3)C%N))ibprOIgdLIaJDb8u+6Qb9Jw+?v|L)Rlp%CwW?mG`5E2 zZ4#qF2n*>VIL6*)x&LWzgQzg65`tKs<_gIV9$WR1QeOf^soIj2Crq=sgoD zrvgJ=Yrm$(P~A}U_&Bwd8%ft8I=wb^p{Jo8h$KJm)QpO2+mFb_1rEcO#tC+nnx&yL)%+ z{1x|@7yhxW8)ji)5x^1~78ct8?B(s#|Mv@A@wfkM7b=ZZDvfGi85y^T3*UbMzVzXX zFltmQ&u9|F%&78}^uTBVQgmfax}hYmwkc4ok;nisy~v7HY%X?ZaTElB!qUu!FQd$0 zW$)bCOa`YPe+UjcU=Muv=YPa6ulzf_vIn09r1Z-?wJZx~YQ7o<3(M4A)8Wx#xv{f~ zIS?!zO9La~geZX^z7xaFShm_2S#(67XmktRCuA3-URokGA`KmrUIK|?d|Epp3cL!p zc?!i6T1O1Sj*}-yL!6m1BEpI`gt74!F-I8KIou^hs7lmMaD%&4Dq{H1p;)$T1s;C# zX`Fq=DSTe@E{5{?WJFZUz{gOvNR2Z#)lj0uAaPlW6-!zZ+hI7K#j#;HXpdbu$x&fZ zA#1=%Iq2-}#utC^Tm0xtmtwEkv(VPk%+u#GH4JkVGp9AJubb)#^{32@H7m{fOj-Hzf~9!)nfaLa;FGxf!N>W3#~X^&2*%ypaoVJY407q7 zw3MZec^ht;e6XL&{DTR4u)V_f({P*}ooeN`{w>qrg{HCp&)#>yNph6ezwYS?o73f_ z%L=EvQ_dM>iDVIsfxp3E<3KjpfB^wAIoJe~lffpKBm@c|f(Vj8Ip@=zF6XpqV)y^| zRj%ru-972JKwAKz3;8TIk!waLdnD9S%%#E-uJ%o?t{m(81KJ+ zJN+uSa=iO-tnWOLgvR5IcB38Knh72&6X`NAdrL(;i z=bnB7jye2r+<3=*7~Z}cQEMABM%1mjE1dF3`p3P!rT*6>aZ&-4GQ{Ifp3Icg6joT< z84Vg95vmU&fY*G7(Fa2_V-iDVffKnEKQ+Tx(7aEj{uupLtcqyGWM}m9uCt2nUOSx} z6ObQ6Qb#TdQQh5-MTgD7i(YUsrgn8RqJs@GFW3K0$?z?GZsuTm__~ZeA zdL46SOvCN>Jb({>`s-M+Y$c98<_J_O6-MZ}=OO2^^jb+lR+O=hX9vcb@XKO_?10_6 zuBOqhbKOU*kI9DBFdfZrD;Lo>g*~-R&R8J2+z6Kk%jb@U_j5)At`wr~hGl=6? z9*LGxk)@xJp(OuIr%BFcLA|NUNjcLvyJUGW$g3`=t$F>GIi;bV;p!2ah)~-*bH_cV zdtI8nZ(UR4d9!lDrdoq3KSt<-*g5eh_fJhplr?!5mI-2d6;l8h;_jDl5 zP3)vaDE|Rpt<{*K8B19)9j1& zijv(6a>H44*Nqv<36*mpgSB*k;ik-+Pu(?0%P!E^$J*W$Be)G&k4*wd^v~Q#?r&Yh zSlme`Pg;>C@+k)@Pcm!vG|Zkg4HZ_|ga@Z;fXztNWN(x-MpkMO8zem6U#0}m45d}$ zwo(zD?cMm+&whb-{@X`z+G!`@WiP%6!{suY|D(ek2zTJdwrA7#oq$w^k6D!=+>ew) zU#M5yRd&6dGnd4B#Zmzf8RX}jyaKm9`1lykD?w1m1C0OxyU%q3f!Y@1Suj6`a^EPW~K<85uz3*Lw{n0fNi7QSkqE zTsQl_lCb4kB*{^jlwe;_M9|DdT>m$0zw57(Umy2(!E^5V{do7x!{gZqPi%OMHzun~ z_CDd78~=cRdhaLL>d!<6{Rqs{s`+pQrD72u{@|5(-ODdVsSt5f(0W72f4MS(pi>ug z0Bh#Yr+gZWJb&}O^bdOd6MLjAcvDFr`{J&UD)mOKj!K$7RrsUvKSj>z-jDikQ>sFRAw1ugw*Os zyE0A;?0%|IO;vZ0`5Usc20fEOlo$nBByV63MTTjoBycjBDXK;yr1rebae|({ZgjS^ zph_efJ8AyDZda>X%yPHdOTw5jA{9j%vQ!#1^mVsm_nslVxw6_-U(koub zObTfQIAUde(lmiH+Ril)^Fv-QHGUa^WRu|4e9uKQKZcR`K^0~i;~`BWZNpd7h?Y;Z zQi+?W63#whIX?40*PvV-xzeHhgd|Y`U;ge-Fnjt`lrEq}&43w^QhG};+Br&dN~zSy z+SEPi5#!@D@ikW1pA(FU+!*QFx;<*P{Ydj7b30cp;tWUDcZTuJtFByPeUj|&0Mk=L zRZp_!xztoivR125=ri2e-iD)>F2Ji^{9N{Nzu|X8ZrJ;b1WFGZZd5Q_u3%3;uw}WOx8H}Zjy6VW%$+qI#~if;)2H?^qJpKdvD8SLdNC;&Sy`Q`cJ17~2T!b7kM*0k zV#C($*s^UWE5klv$qXCi1@mV88IuW@Zdell%aZFCz70X{y-1R_N$9^pUk3DBHV1@D zP}}>XaHg;IhB{SKM>g0|@gx<)LAc0+})f*Elbmu*g~qx58!T zRIPIqE1GJ>Sp@;9Q@7i&R)6(szxpXJrRA*3ign@abF=y{uZv#QwZsx;f3jAJEw@s! zB})%7%FBj1)GIZhtpiJ!E@px7wrx96i1I?2z|B%QpJ6a6;(F>ckpvHamWa7asVDgW zm_4-@w>7ujnlINYz(q35FgeXg#m13aJGxJscNDh)Qf6PG8 z4Nk}6L1PBfsk9Qo+vqElx&SHaBSqX9X*<+ddmCT%CXnLd%;S#2jeoiqqahZaH$;Md z&j1yO)rBp(ci*W64Z_(wL$0oS*hpQd2(xrcyuwxzGg6a0qa-OZnGoe8bhNjQE~}b=2rhZ@6Unu+ zzlL1v7m?r!F5R1qAW|mi1;W3=z!O&_zc}in=sosB!HNGX=ra!wj{}Lx#)^7)Od9<7 zhMTbbqE}1`QX@!W6aP+B%dlPfEO}Bc!ZTHA50}v&=KI_>?g%FgKUHs~!sv0eNnwg2c_CHyk)*>#3Do zy}4ZJwfmnsA@fpa1P)0qLakg!XIl&A&78_g^U~a7$CHthD9>s|RH|`9rNg2a>17eJ zcG_C<=$q1wZ(aLqy!!9oiAV3c8)(U)W#$ZAe9<}R>TE-$TD3i196yBSC+Mz#H@UD5 z>9eoFQ*%FxW(VBYaFO?s0=w3h7VG7U&NzO|t+CHrK0^PVJ^0WU{ud9gT8s6Ywz8@c zL?}w>znPw*99~txZW4M%uwnBeDruKH@*kw!biYqS06_=dDQo<ZRYXpkM z=!yp!0h2gkzA7tJ5!UWz$yJ_W!p_PbI_|iFheOz@k{W;8P zPkp}J_mH4aELg7{d*ouw?Cl;hvRw-OP8>rj+UJm5M@LFMFR6%`+u2B+uLAQQdLM z(8lraMOBkI)40@p-yI1x8jc5r`dqW`B2)cfr0tn=n!EUBDpPv9yKv?SEAXlpUxt@H z?*dGp(yP+ZJoXv2APiTpIBMw!um18!M$Q$R1PCu){nMl|I2$2;g2|5vVQ^|Nyy^GJ z^@~3h-~RH}@$Qg^$Ds^QY`6&>~adjhI0OwPrwX{+ESR{=4zRoMgsyBanKdy|oqB|K@hQ z_sWl>l8cb!@`!_+sz6|`mKsKwBc8gsN~-&;pPMI{jM4PZA$~?w3}o*m53UptnJvp; zL~W_Iv1R5>8&HZ6P3qW{QuuaaY=`4(MO zg$_We^xSMxUpG=}OzZ8y_DYO@df(^qU;p_{px6SmcVnm;AlKG`^Upg23+GN}tFphB}L$Uw-z9`*a#4??;r!V~?+5%8iFs zJtSRYV_i+VZfx7OZ5s`DY}8jK8r!yQ?cnV9IUm+9STpy`TnwMo^D-5n z<4H#|gH~l-(-wiif?nTzAQ*jL=wOZWY}7!}$!jdR*2}5q=XM?5BtjH-%OhpzPHz_} zSN3#6O3WUO(Z=51%n)*z6QN(^caUd^H)z33%kjcDyxX{UIY@rpvom#NUPvI=YZEj( zOBp_z%;dj4%r}$tULBHmo~>Z`@zxGIz$MD+!w^TA@%k_IJ@sBpG>EfBk zOIB+SiJu#pinSD|;}Fx~US+-Kv?1~;ru{W^W6wD|4iF>R=;5Wl=g@LVIF1L#fF%jd ziVD`6)wX(n2i^JTanU!n@8#EP2Q!3?jkI4W^1bI^ZPl?pKDJVqOtUGudlRaN8{@u zQ2*m2FMnYHTBnVo{x)7j!!l|!)snyN6b zp!H+>|0jny*>nIQHRj2%LwAI}J_eX4@_^r%cAd9y&wU1pNnf^2S)vaG)HB6<4d=Y< z!(Q9_;-&P85{~spPKc5iC+CD-=oOt%SPIr1mzwsVtzJ(aVgw<1pV04H3H?u%Wdry+ zvt?%e+qY+v@%Wsm7%EeL?=s`jW{X}AMW-SDGtr3m4RbVi>%#A9)!zp2tcu}XQpw-G zEx9m7w@K7#gQX)%O8zjjNIy|1k54oFN=s^}#AXBj2FpM;i%7U2AzMro!%ID+Y9OaK zUWz?Ev9^;6f6a*^{Z*Wg>l$QMY=q30AOXQ`B|ezGH6NX~;V$rbMzR&!dDVqejHixz z&{jY}ofjz%cWiJ+NZ9O+T?$^(nr8*2u-<<_UNSPK?{4!b+=a@>JDA?N?+CRh>0j zpsP#6|6Gq)rD#O6iLd=6sF{T>OuiY< zyp~Mjq(_}^SV>$_1~;h^mP!Bl1u*aCf)o0R|~iXTDt6xh$#g2 z&7t<=FG5=hq=(7u@Ptnw(D_{zft3>=&5Uk@ApI>@mu>2;+oqbD`)jigXOpj76~`P} zqq2+#f2iB1@Dd;Tbrw}VIRNNj_B+^u55eSzR?lUj_erpfiuLu*k%!|iYB{qmON6#V zAB)lUoRuX!=5yZ3Esy*8!+@jWnSJt9bj|s*ZcCzi$>;C+%jQVztf=>*w?FX=flmp>^Cb== z&rc0vg+4Nu%g_1{52VUiLvQ!xVZ7kK`$BZYI@iHpTC8 z6gdVP!>dT^`$FiMKFl|2lVEEC{A$3ilfA~=r5OHoikLc=0qL%Lb)$dNPVW>;_A9a7 z5E8LfMRpW$j6ydn*L5{NT{bTicz>f!dv6kA^RQ~k$0W^cV9;#xlnS{Z8N}24dyR!m z%dh9p%2E4G{if(RN^RZ%@ObFzZU)`(rbWIOOSZf}!>#txEfjR3wUwPXrK(kE1>NuJ z$3OQnYyI8#mXm`bV41gXB%N`DoRgWJ*ZN?AtdGOA-zKD2Ffj`=db4NKj8^XcAN$lW`zzGO@U2#Y)z8b#R1d$ zAFcj7jW3JSj*<(@ag;R#4BLf}*mpeqW-#W_@-_gQ6y?-Ou6Aly#wI*!EeTmV9w#9z zx=MjXiQ>Pg>^CWI;!eX?3o#z*#N>LPrn!DY=bmUvI?Qv39S4D37X^-qN0$Bm`t%Z9 zP0tU;eI{;62oTs8YOf{%@R>gn69|yV$zL|AevhZmZ3gf9#(D+4+r2D#)rt{HMXjx6 zfH7xaM3{v6R>Mn@P4%28G8(btgh~vIDTr4luqzoAiH-{DMJQ%>#^aKKHl|lp%d;~J@i0dv06}T}YzjC#xiqq_s7KbcKFsWAszT7Co0bWZ4v6nm5v-K7h-4-+I`0rRFDp_eX|RU>qy-yNAc zrWx_*yA0KU#V13tE-HB_-x1{(VFRYiuSI;=kh~1}WZKc?V?!NR@Io!kDArSf1dv3X z)24oit68?R4mGx!;-w|_V^@F+3`kmRcwYM__on6)*5X=%$jQv$|0wy*FWOxF)+7zy zTNP4_s>ksJ?YleBWul(T8=I`n{L$){os~aI`_q?H$UU`_jrP-UVtOY==x(zX%`!_5 z|Ea*|cK@%QUau~Jqg^(g-T%QH61etI`o2=AE~3xv;AfBc{gjct{;xl8Tb6rZ7j3cH zWz$*WZ`^npPVnJU)cWZuqKx+6VEiVCWnKl1h|=zYD63c`-BobNk9-o%Ky*4*xSASm zT=-g%LGoWUr<)2U%V?!xN)EZipRO9AvXD6bM5y zx%{2uJsjElWBdJTdZgKl)k4ic_8LMBOkHl(07ARoT-fmSeY}Y(JXNi@HktBmLX=Zs zk$g0`>MxTUC&!S;cu*l;(u#iE?gXsW0O0HJXvi~DQFdl6(_!x`Sz&reDDM?J7kLF% z3XMJ$Uiho}dOr!&8lF(=%f{y0_dR}aa3)jX?inmlb33jMF zVDsd}dSDmfX!ij_bFamZmhYk6Y$%(&?Y}9n~wfQ+; zyOCf;7BbH51mR3pBXH3`Iq`*(95%-FU3!KGz!0Jj7iENF|26OSiy$Gu&AD$=E5U16 zP>`BJuzq*f1-6pYeToaks*Rjnv^xm4G&_*|bHCe33=^tM6XQU<+w_tVoWZOoDTUyU zYg~4k6G&+xjoR=xyf_2g`z5-$?Knw8`(wd(!*f|eyVZij^I|1BeL^S{wRj0z@$z&a zNrurx#8sKs6-<*kD}h*^!MO6#98%Y;CKMBqijO!dQy!^NkoNHsrFARLtLu2bOcE@n z3FVQ2Q$7ZtCRd(I1*M2DzmuN|epL|%pX%IHSJ*0=0mA?n!qY4k332IAyBkA8QF<~!@U^aaJ*qDIOzpqEnoFHPd`$Xx9KdFbu?oqccf zMfH{D*1Tc{;Yw7gb z#?t+<)^`a0bp>2}eCeJ*_a9}%)UD*ANnP@6iJV8KVseRbH_If z$BtJe!M7LshwGKeG2!2p7M#{FnwKH5ye}wA!}x#moZ69Iajnm-rLHgdc=wAAy+7TQ z2EvbX!$825wBt;L5LDm7zcw+-7)6oU+3~l>tedme9ux=&zoctxt1*=1Ky4eUG4ok| zzSqAUS{V%Qc=xu6T>KLhcFJ&tLh4bw@n-a*tRq;Bc=uboGIKvC-nIdx^aLZ?m}&AT z&3WGTIjX;1XZLX%u_EeW&H-YWofb`$er9SF;PQpJHMqF`BUpPPs@<4>FTZ2 zuUKWaSgF$%z9snuNT0;Z2^KG)yMp85c$lZ^s?($bzU;-R)O(UvX*@8~y6_H1D4y5( z(mM!D@<`7Gq82~#_=lB>%!rEel3EWC6wjNrYC{~D){3VzVeT93u0)M)@xPp&8c>07sv zb+bjcFJ+cqbzE>cyxC-tdg_@Av;K5eV&qg;gHHEUTabm52 z$KjVT5MC?MOn)boGocVVofU=QYfYMmfVwbVHUy_6bPUM$*<=3=FIEIiG%OrV-IwDr zg@^d7)29)TYB;kAb3N~D;B=TU z8j9ns&iB4WwGpeq#Gl(0ETRI-9|$8doOk61TN`*=JR*a3_6R9taH+`EU$F^EcK3Qj z_c*ygbBc5+y1AxQF(*yh=uYwUDoDXA_Mr`H(QURQNKmX-n7b`}!Sh?)>27vzfVt(52(?TrQ0^be>whsgC%;e|g@ zGqjvXn{N{K@y@V{*B5T8N_v05Meqk3#ELGLy+Wz`9CL^U@+w3wG- z(5K=Y-Bc^tc)3v7^s!KRM?j!Y;>q&0I zXMU3Q!QJI9VgYLi1(e0d8i+W)oWd@S3fY1aN{GW_yvT5TmNc%EK4;H$K`mukBvvGg z#epP6PckuG7%rH}I%9woQ2s|#3 zY0Jt{1`mPQK{N?A{#(Do&sbOMRlFMet{L6Y=|+OWsVMXrlniS z1->h5`|W0eb+3z|qsz{^fMdtsB@I63nu0yP5Zq4dQ)+}T5zwm4S|jgrmD4M1Zi2yL z-fCC{u+`N}v`6J_rjD&@=10gA9hAP)FadrHge zWq%FMQch6Ul7%?g#LE>)6E@m16peY`%k;*Qd|mLY%=f~$kcszbOOlPSb%bMLH8Ht8 zSzW5EfhiZd{C-`H(@BKqP-0Ihs=zZ5jK>Me1#zW~8##x+M>1`b$<34Tm0`HGeqG(uGF;#J{i&|5 zJ|$D)^`(m!G9|n6HGkwdEf0p2!!&)~LH)d5+um6Mk_-@hy-9grg_kz&q-kFH`E|^w zF|vQ9M97L2+REvqF%}DtQ?fpxW$gc^Gx76vz)Me|&Y%c@%0$%&EUZ1p`Al@?^fwBf6`*X zpIlhWI4RQ^B zavs|J$1}h2^0xvsEo!*aJynQQQV;sDkMOqgtpHeocE@YVSJky`&Tx-%cw~OnxYSP zkTTwqoTdaGU@QhJ+^?Y+Q-AJT)O@tHqL_2t;BXlW71#x|ryd5DTJof>aZGE3OBkULl@2vys*WKQl z5{7@56wCpK-h1Tf~`fr;Q*(YWw3p z+mbmemn2QMS2iMbZxUHErQ)h8{lp?@AuNKrr^J)){%rJdJMOWVWD?o(c1HHK2H_@` zG|742U-9n@j37f85*63x>I!`2>sZU|-x#^y-!*P;hI)~e->>%fH`!1Uq9I8DzMxwm zYH&%t{WQ9&s}G>U;=!|Nw;e|Wruv^?L`0c1!$d6x3!S68_Sd6u!KCVp|E;15u|o^7FgY z6c>{#7c7}S^=6Wz86{7Tf5v9vJhm&HyO`h$I4-i&P4Z6Y>9iT)uS{SU(Z))?=*~&` zT|y2EJygDmvYB2<&<2WG@KJl=OaoKLgSay$TwtVpdGfsU=ys%zMfpuOBREsN&+AL$)??gKY&|C;f-j{0}qYOwTY&{Q7~54gl$)gZe!xn5)ABPd)``KS&&JlSC2 z4?Wq;k#{221!w}1myOI61DY2oC}dqT(TkKZ@EqbesXKI2UIbD$e=NSp22v!ykEr>o zJXUtd@ngL9r4T*f7u(&fntV-y0dv#W)+b{XBFC}pSGscW(!e2jf5@WpK=-pgFa1&$ zd^+d(dQk|p_{1aizIrnFW|*nZbNveY(?}vYk?;63`xOSk6)Wl>ng)9P>;31!GL1hP z$23sid0{wWsM8H}#RL1L@%wOhHwny8@>i`ULaP+@xD()v*Yy?6`%)oqS~KUw$u|cP zTYb!%sLT?bc*yfYg!;KutzJL)_TTj(7M!_i2E3*)|lm`#^w zh=jez>o;7(*P-V!#K@(v!j86qAaLruQwhX(X2T(sd)g-$%z} zA2~J{tw7JUryhzRl1|{-r7TwtKFU6Dgc>%tWNpX-CTIUVjR;)nss>Zhl(=KNQyj~( zy8_@}=>9#HARtXe&YAo#-D6m8BY@|2SrXrHnZUgelO~s*uJRx;__BT+d015eyH2eM z2A0z%GLdpSo5O5K^^wNuvx0`}i6lf}`IEC8WkNa~2uCeK4!Nrj@ajfT{y113&8y|L z>k>HFWkcJ+A|NJ?ga^pUB{eGc`7tNF&7iKqQ|bGD`kYuZ6)2{Xy7E01ZWwADqq#Nu zTY06%$|Bi@&IttigOZZdcnqTR(N%06Ov=G1VU$7$(RI0Kh&9mhJQ>PKR^B3+*)qjo;N}v*xRgNJ*B5t*FkE9$+%h%XJ}kGF-n7JCh8nxw&AaYa1qwW4W`Z>s{jMFQth>5>A?I)Nfz*m08Vp z7-zgpvK!m^Iv~M@+mA_a8taZ!B_U z8m725u1>SJOU7tek67#fDRsdE7?edqf3j+uv4RxRw7}dCXnBLteTBb?6nV${dWrDL z$Jdyhn~T6l;BQZpLceMHHk)}{$X?hGvq8-vzR-t_(sh7Q^FDJB`tqGHshzfXeS~z@ zK(0B6bDZR+#VSnc=PVtj;KnjjOb=`-7>$$ZrU5qG2qxNdS_j_LN>2SgeIQb1IAOL{ zJuYTZt+e&B(y69VUl%0}l1>)Z` zF3g`k*ip$+(#87!Hco|By1ZboUk7<^evxed_`@|l2E$Xq*$X}xp~|2hgxkm4Epzk- zB#9l1N?2o1t(D$h?mWiN#MZWGvNKNn>(e<0nV8=k)~a$?5+ zqp^1Agf&`de6Zn!K8^Et9m9BJc7gL{1J(YGS8%SRmJn7ar7fUH1J1TZ)_OC3lz^s>4 zj(A$C-Ke>>>C>?uT{T5OKjm^~W;;pyK( z3~6@g=5r8v&!xq27)?Oh2my%sr23C~_)%Buvp$V$ni^j$GKi$o7wOwR1%NNjbs;~R zx)3W?6C*RK)2y1P06Q~k$)uZ-c%R{+nsBA6(;nH~jU*m0>baG_SnYsUTlVmN~9HPeI3}3;{s1lBh}Vr9&6Z^^y%l zO+8T8g=plsk}tA$wZzQAbXZlObvWS;#TH?X(utsdt{YY*kD2LIHV$Q0uMGlVWQoGt z0@c(^Wo9Z!VB6#)5~zZd2{h+Knza9gVxMO!^db|*yFmTKN2dTJqvf$er@p6cBp7MU z>uVJKq4f}j<+L3+25-WYFVi|p-&&2)$nxAV5_zC{e_L)P-JXxu$1Lnaq>$h&!(>GG zRtz7dxouIQJi!r<54UGaG@R?s$SiOUxY%gc^Y%6u&F#dykOPr;W9(Y}G7KN|Sa|Yu zi5}kS`MZhQ6_VuwMsX|ZsV0ulF8Hqa^7AV5IH(mn&pP1 zN=d}R9X*GSS~Vn;=a(d!i6=-X>Ebua1aMOtS$zh`q804nS_09r*-X)$hU9E`DDD$N zU82~$6U%x%sIB(>r+LN|*So#G5wzO>3r(mVY9SJ+C#hpL3>csLm`xm>mvCP7&72{L zt<8Du)!<=y0I{~L(gObVL@BXI4F^MhfyQwB8Kp#Ga}r0y*|EJ@y^e}k4? z{(VdJE*9Jyx?w$<$ozSn6GXi(eTYZ^ZP#Mk_gUw9{OAxzUy=eMRJ?bfJHG5UBIAKW@?rYKkm3wKf zL{Vo3tapK}2moKO76K5R@i{D~EU!At2lu_=>6b$y@}3dWa3XDxTgj$Jm+%qCJ&3@! zUlh^9&a==d4^?Xqiq#huXO^MEk#rmhu%M5lwmtAKTf+CJO8$3OMJacGZYDi=WRtB>)V_f`cJB*gzVV|Tfpb<2T7>=Sr>SC$ zSIkQ8163hXb3wAD1~C2E>cvqneb@_6%l^4FrD?xf|1-n*Y?cU&`t%?>zdo2)3oE64 zz%pLLjVc1z(GGIkkgEWSbcTRS(ekbXe?~A@OG9fMZDu5*6_VzojFv0ql&efyf}62}J9*kI0(9n)A!M5AWwmF|Yj0I~&fx$?XWM}`4R9~4Xs5|pr7`F%wGhi6ZE?{l#q)a9K6o4%3 z1vP74v~ufEmMIhU<3o2Jm?$4#p-WVjbhv%{U@%$P0|1zBq_9A7^2W<+>n zKYoZ@CP)kB^7W4|34b~Pk-t_6LMSQJtW>k6|LnPYswOFaU+&bjj)I5gm#tIaDf1SG!tuQMvyOQQ3mvA$ zUZvqxpGjn%Q);~&JNa0{9R4>UiPIF(X$>3@5zNYtNilaWQ6yQXM>G;H>IKuUHha@G zbB46))EMb236e$FrHbqich16)YebEgU2TFPBHhBXW)GkamHT^SPGabmTaGV%<6lE< zYQe9-mJ2jzOwe`d#iPDw+nf8agt8hjlBm#+B`w6JAfeQ5Fot&Q{1SV=To;ZVCh>zW zrpsY8wn?O;R8PJ6t72eAw~lj`!Ae%Gk`pIZt=3mhLlpT_NdRx%>r!m0DuM$Y5i4}5 zc*KtUkoGOc!iH7fRnef-GIAnw8}HiABQhFW10%;2prN-@r_<;> zRzF_ne5E*%y%t$M?>juhi}JMDFe6wkr5KHPhB1+9>cM|QKh zltj<#>R)cKSW2Y^ppwh8a@n?Q`FsKxuz=QpE_(PVd{IlN|Y-tCOo6m zPk%YzPJ!9gtq>3e*_0@Q2C6+W6Ej=g+?s-QWOpnE)*$;D8ejj)Z@N3;1X!w#mcVk{ z3-El<$ZXn$XL-w^twPJu4C`tBY4DaZK$7xXO!x1sfuUTakvu^}MCJhh>XFgf4F-L) zG|$RixX{7R;_1e)EI#jzx6?=dEF-M}2vK>VlqI+}>~y?QPZ#|&YswJbyWUi2?4}x4SA9NqIQZ2Y9c#4Ze zuvRjF(Kic%UiZV*Coo@=jPndD9dro9hIeT!gSQGeA<5(+Vf`m7FJe^;>|u}4PnsYp zQ9{j7OSJ&nk=s%^eQ>lJnf<-=sudqKH63|g3_JzqLL{0k1O)H(d!x(H?J(;i^yr9N z135Hsb{Z&4spZiYONp~H+}j10Ak>4821PxAs`)gbQ|uW<_4C`WgJeR%P>lNF?Vcw4 z<_>(bqZHxyt9ZZL?d^9k;@&Sd%;O1EFpJPhHYMA(q{8%j+XHk``ad03r~&gOd+j)R z2{rMWAd~@Wlju_LQWV*2>Ah)cjVp;$vq0*`EH{O?-0DZH#_`0{gczc;s|k=kGW|E|J%>|*>pC1i zYz{`}kR~@!yj*}2+q_MoWFlqme+K?`j*{IlEE(e|#(E z1{^@(E0Dw>JU@T;Bgs_b{PL>jG=qDu{tHdm>x27S8s+k2_qX;rFSOx+bO34V)4|;V z4^1`VROjW^?Nwala9;!sx1b<0R^7KGm{hh82Zm*~{km0N;DY)}!hk*|IXp!R@3iKl z1?uldib)^2&o=YBV^ZVSp*-N*^K7pTj4t-1xLI9ZB`1K9kqhRi!m}}Sh2hVc5XW6J zz1<1Pp9`z{WT+7pKrMJOJDA`VuZjbij=o&bJT&bGxRc z-$yUSc?OunoNx<|!ejC}wvkGePlaMQZ@n5cLlGQ{s!V@PZmS^5`7|?x;pp+K4fzcL ztFD11qK+T0skdOK5w>~R=C-K20o-LV8E&~8nM@B!pK!|Pka#I7D|V6C{4*4*C7$1| zjK}4|G;~{<7yEOX7c^+ErQ?}?5d^Yd5h_ons2VU_oc{{sE%`R`&^Bu8da!Rr-K%|_ zB_y8}adw&6TiTlcH%oT1r9~4F8QFwW-}~)$rQM39=e_%UC~adT$~`)zs!^~Ww`>^< zUK`Yz<(60qRW6v1lJKDvG_e1!@5EE`^k}^FP|o}Hugbc{ z-qM_33@|Q=&aA%x#ztYRek1`M?3Tj3RHdPRD+HaysAi@wZfm{ytyBGiiHNLs7-rr8 z-e7>bl4V?g0Vun-i|0=;lyD!n z#2Jg(xTL=9;vUc2$%nK58-4r60~r4USYX@U^8l!rh<7QWuQH7PGKNXQUE5>)B`v03 ze*?iCgGi&|qQ2k*PG)It2lBR}=BjKaL_$ghqA+3V;Y;HSz~$E8RZ8-$W#K}|fL>U{ zXZjq7ycVCWY~&p_?euB*O-=NYB`RfUl2LgoN&?%|c?#)iq8xQGtW9tDcZ*Y|@l3C0 z3n*ra4`LQtacpBzjux4buh|RulJ;SW%QGsJlI7_#IU%p8BdR3X3d%)b-{IiJ;hW}S zpebaczS)dm5;vkWh=+9$*NBT{ky}o&^R-M+4DgEi5+2ssJu6wJ9<4OZ!0Sri!9fzLk%LaRYh{Q9|RbE+6lgT^UyF z_rv1jhU>)I&mG^)FtfE`v%SzLJ@`);?v#4Z0m}_6!C0fuT}=>xVmf<~!TXtBSm(UO z2(lTbfmVb^OMS0I_6UU5_>5P4A>zW$po3;AmO+s(WznK~_H+H$(1=+l$yfEd*B(mi z_RmgYe?$~PaT{7?c5?3^QlpXW&#Y_zO!i_yYS#W^2O5t2G-{E&c63#sR^s5WGP-N1 zaQ^XNYp<)N_g*VcoR>0aictT*P%ao;A0GDnkiI|-jwYuLJzP92xDQ`GJmk%!G#m?0 z1j(QWOke(c?P21;Rv(%Fm&|fugBc5GkVvp|{s_we+!1SwLmrow+Q`r_N@-hSl}o!Vp{ty2Qr6d^%0>L|beJolz+eFm`9zydJ9&PDpG8E@Q&CY`-F zp%{;qYZCWHD=qJ*KJ#E&PY}^__NvHj{;;>igcKh_1OpeM(`xW!R_p$gAF)_@an$Mj zTYBE;YAIplFHOSWr55a?c}W|gO@m9u>Torq6JWBn;wzWX947!$YAKtjus|GWf8%7x zmj)9tw$frpbLF!GxAC!K*7n>tj0>vOMGsD}il;1}%;LKr`j|-`787@VE%E!6rz`~3 zzQllbupLiH<~IAmMu1G3Y}0o$r0~r@pv!M>*M;@}<+G3t05S#G+U1$qwfJHLwf}u7Pl1Im%lKCbAq=s1jD`o=fz>}*$7u0D|J&TH zG1}uYAB4)P^2Vy_eqM^XqeXVbc&W@(c$CS7Z}_$5WxVEx7$JlToK5fxm6DSPDs<}O%%rVcf75py?Yr;aAZR+AIotU;kC}X{LF81Wbd`f;F^L+WAhk9W@@wgk7WqUs- zs7@`s8zNoU4xhR6M&G?9wa(}K8+^E+fM&{1ZBgd=*eXsWgLBO?jDzeQVX!;@%lj4) z2^|}%rA0!z6W6uY8A^@tgplegl>iO_ZVFO5#($I?I%GDYR!xN}y8IIjCR5cqv$FA|;n76we1U^B@BooT} zxiBOHIVge=^35H+{VTfZ2qXgM2WGX~EY&`QoRDcwg(C436Jn7*@@LMy zxtnQ!6Qq_7Mfwi~AU`y;xR7U)U~)7bi!Imod@;p-2`_Pb$JI2i{-F$6Z4R$hy?A@R zy&2Jc&RuBM$H*BWhehg&08Y=bKCv$D!L}d3(JCUr2P&2Ru8+&`Q6N>VKV#AxXicnP zpQlkX;4_(zoq{f&;l>+muKQw-O1cr9Kw!XHBlZiGu7+m+tw@g6XR3ie^3!9gBEvaT zym>NSIt=W*EYVxNBiY3sYa%O0RUC6y*~E>#V_-_fl3pnVR5vfd^xe>d=OxfQMa+z9 zNLy2=9@)`WylnFu9GZPxSd5JEAtiKxJ1OUiGM?apKKp0lp^A&8)sKuf03vULGw`UZqq$fm*a* z!(@5tEa{xs5GlyFrW=!;=oRg$%BhFuaR=Ne+c5|6_6JQn^C@`|=93V{%3q{Q4jZwN zdEW5PLJjaiFZV*;?vJ`9*6U88=))gwM-%N%uV@GXbWyKAZB zrdu^L3?$0$_Of))`U3Qv)8Ewbb6_W<4B%2hodNze9tu`@5kub%|&BD{EBW~#kOf`lo zJtw^mXy#VvUG^axdx&N<2wOm>#~T#FW`4gD(13QHXsbvn=ha=jhU-Lv>!W~To z%wKSI_X-kgnQE6Sh+_^4N||$E58~`K?1=oZUlw^`qsnne94Wzg~YI}RzZWyDd zVG$veA?|@?_|0S@AMc+Jv1lt*zDujLpQmv}@vmn;{Z0np%+{oE5Gzd{0t`+)MjXiffBfHE#%ulND~ZqY)Yq}7@LS%!G`+R#c@N+(vNTAqWijl6iEW(Z=iI>*Ota<03UNmGc;o`O? zzK;q+@*CX{jzR3_7&E#dY1(^743Uhilh$icFENS~Ytsfqk;6wB?=8y0n#2zoh2>tz z_;}*lhR=u%fg-0YYzXWTWjns_D<68waZz9_KpYfRgwaV}08mpsOcJTahQ(h!X4qcf zZq(T^iH5>K)q$a5vZXv3G;~uUzm{23XqjIrLr{hG$lLO*7`rKrKdEW5-4UlmUbQcX zaE-A9&BF;RrcfKEVR$q(D09_n2r4J;(jC8I-Eier6JI-A;*Ye8T7HXV`EL{vwK8>U zIw+f@O~_`jagVyNzanuxOrsdx>}$S$E<-w{roPHFOO9)? zzfHxENT66$h0h@p$<*o&ai2WaBcmO1tT3P={IX{k@C%8Ba9>|fw(y7g4#F@YV|7fz zHRn1zI#p@0eJqLs1}{znUdA+>hSo(sH3VN5pi6Amj5KHjsnM)!WJI~-RG89cdp(7g zs@8ve0bgrb1AIy14I5BpNP2K-h&mBzh}xpzKg@&+BAg`z|@JXt*LH-n|+@mfig*rH(&!rz??ok!c9 zfXlr>uPcOJLexnAk=UQF7X%^x>t3ilpWqnaQ+q!^a~bl#UkVUSerFZ$Hk7fekK$`A zX-u26C2+JJ%#|}!g3)bI@Ry6~+l9aC{xSi9bji*UP<&-Z#H4iRp`bb?>jD zC=?%mwBOi&1YlP_uoH1YPAjO6g$e22Md*ANmyCv5r0pl zU_Tju3dEF0$hS8kCXmN7Nr0eH2wN?CI6?(xiL~(%NrgqQ&;H4K&h9onl?aI%B~@e> zBT(tZMH>QI;AO=pV&}7j5`wzchX5`kHLm@#v!bS4n%pBb!Gs2(Gc$s5Vnvjaj?Nt8 zw#Bi9%G$vQ$^u{?1EGR#XkvmwC2IfpEZr;D7pMz{!ac&s?J;tBv zHmf(xiS(uCCgf(5d+rVQ_05<0O%&tHj_dIL`CqDbNnXW_wa`Lr zrh{5F#|HVynaVTWnst9H?dpwRuj@CgOO9LM2MAr5G#c`|B}5UJqL6AKymgd)*uRWj z8Pz16lJPZNe}LU~JQOHiM2!QmX{QmMvjJ)DGASfweMBSqc!sW2ka$0m^W>KJ z(l4_T$jR|>>%|R@bIPEUIzqS)-mxUd(7P!!Y1jI9w^^h@ zyr1@@=o0a|N{HfN_##FnXV$d+Bg7J#+Ub5JNuxC5emK3m+Ebm?w~0X6Tv@7Y*6C!+ z2Dr&C7ZRLMM6~6|C?a;GYR$@#)|TBa4yB=FK92gpd&`6Sq>8)bvKZQ@gI10^zPP?) z4QLx<&ryHJWa}Zb`;VkttN!29+5xz#4!aFMzEcwGrcxTstkcuav|pUH?Rk*@JmI?I z)OkFxb-3$D7?I)Y`fxHFoKqLRUA(@(##Glmy1jt>KIb1MoormT^mpdT;D06w$4(%a zCjIYs*Z-sG90TKQyRIEu6Wdl}+s0(#G)CjbwkEdG*hynFwvDD~Y&AxcH0d`_@Ao@@ z&!54$XJ31-b*$Sm>S80Xc~Nk}BVXFu9qDV{6fVz;pI)6drvPYFqcc>gEXJ-Z)UGV% zTh017fXFrV;$6neXEF5b0%6?VR>CdS>kr3msvM7SBR_K=XV{(11V z=*;EJAq2!>Pr91?Jy*%=A!i^m%MR_3nrQ`@=-9#qO^0GygP2GrPOr!sx`M)!k0Pl1Y&;H7AC5 zsFaPk$;dRac|ru z|NJa9ao@zLJH{LF$u@UGZh0(mjw)FPZ!C@@hu@jKz~7ZuYgpu?k^d309#;_G`(did zP<_6ifY$K}3AvgbQotq%rvc?7hgM!iOhtPU!}JN*DudvZKw0oKv9@fSPryXZ+Avwj(9&O` zn3R|C%3>TYdhGLk1emOtY;{N&P0GZ&C&a*;B9i1x`9bGtxiUtK<1OrkY<4OJ+aT5l z*Dw|-*5_a^Dc&qEX}644K_NnFpQ0F;nEn^uIWiSq#xhABuI_W!8TvPJA6%@W1X^90 zUJIlV>Zil>GBvtJs&wIQ>X`Z%sd9f!dzM>AK#CE$qRKaPH6SB)EEA=x!Wrio?`7YB zB0+uet`g3(9^9-eB#$pFu}jf7yRAq+26O3Jiz#U^i3plL0$WY|+!E4*P_;R1&n^?^ zsO&?aizI*cO@AkjTi7t+OtZ;x|8WNq*C$5sow$Qhzf-OXHP8z;iGB4H)H_!TfSR%P zVX{6>tx({yAUAw#9FxynO)I64!mi-Fv#P`LrFqU<#iEov+ik<^# zRcahSgK6n(W*jfg<;Y0QAKmf}5WhL{v~y^|;uZGGHwP$9O#r?=CqZm` z#GaH=zSk95|GZXBND`=ubH38m8M#wqHt$Uz4#WM4(!qAg7F#Y!yt{&8Ncs%{mKKTL zE_N|Y!-ftobyIjjqb3{KFWy73%p7jRJ++L<%-_hF#t~9 z4{?4qA-G$ckxUl7yRlY$Qk&oIrZUJ;Xowa|YREZkHsUF4nfRdxR>1s5#K`yZbc32! zEott!)3%Vo%NzA&PCoqkQv#{zl-W!dgG-VXzU4E6o5eZN1Vp<{4MZ?IBOVkROZrPTkD zV1K&$5V$pBLUc9jtpWuHttb`%F9DetK>ar{&^Ud-r8tIBt<}2;2HbZXYQ{fh_rYj= z{v4(;*uR6d=L-Kki@^gru*l&jbj&_VQW|&q4G{P_Nz=uqf9w6-tN-F#8C+6rWF6^_ z*CV(LeO`F2fh`V&y-|m5`-~0wTW<@plOW+R*`A>bqyE&(*FG2Lb5!p z`@)A{2khgvfP~5lPgnmvt0bsr!Sq0@E~7!@+K9a$y?rCW@#hnzE*RvH%MxCBxfP3} zCV+5U(18{&m;Z|(LNFXq7Q!Kf3h?o!MJ7&XP7n;OF)52KXDuD{onhY8xJ6-rM*1M| zm(@K|c9<0}5~!_Eg-w${a^b60?>DcYF@1ak+3}5@@!A7$@1%Srt+aX7ad;H>=c#(C zmOq;wd{?+ra=Mc|p4jA*qIP$M9+X4pBo_F!Y~ivVL(|w5gy~w_()epYiiy#^*C``H zSxCVnvY<&LE!?^EMwE+Q)+aU`gdp!kVR=i9f^qD_f~5 zbD*=4QRb8O&^O9E+iz}1D9s+gTTvn%n-6@3 zO+n9D<6rFXrtsN^)1ChcACw?@+-6`MfgR9ftBgF^72_DksZ8Qn#gkO*VUuAbwe1Rm zZrnbf!XMjiqhsq~QAwPNUEr4()D`vg;LZK*3L0rs1yNC(_vXoPH1fo}_x!Gznj72h z*>m;mH>ew^3BZyZc#&k18jPZWig^b6P+kXbDavA(sw8wX+(O?ic@M}Q?+v=mVk;q< z5wnEq+Bzr&TsSz7_=!Ej`@i4)MaM!+!HFbGtF}0l8$Y))7LSr}tN%KW`a@MARQh+E z0z7KGqQ{)^bpBTC-JB5oh*Y>;Yk~-TU+rRyfR4(Fu#7COc7lLmevzNWqU*~{<0pis zKfF0iLceW>vbXDxBRf@B-*pg|fT)@H$p%C>v|)-avlSCIlS*dLdS^=;Lr3N_srChp zcG96o>wUdPBi;+}75f(3zoE&4LPY|pGu|zA#BZ;&pgzGVcn-VV)?3c7ZIk}*Vv^EI zCLnzc%gep!BM~jg{|8N>f+bqhYQ@Im)xTP285W7-mR3!j<}kMU$)k4$_Z6jBsh~1= z#%Lm4TiHD)#0rKIMvDyhbbjIx*fu-;8`!{eg1sm7J%wuSO8+8?*LrY`-# z(K~BV%{$r<=m6&UqJ5MkhsTO1%P7_~`KfvdbyE=+w zI3q?j4TYM+i)F203cpaNrexVbrhS(p6-Nr%iO=4cYWR_`GbE{%E0-pJ!I zC7BKvn+jC78@`{pf3crwq5BtVF!=RTR&xQK?znvIM$LgR>lL!R)xemVmqrTPnGl3l z93dYGq&atKRuVmRA75c43P4i&i$c`Gj4cT-xTr3L0HJ_M%5X@_Wt=jOJmB%1SPAz9sCl zMFV4rNHHC=k0ixh5v|0;4?>gaLau;pbNX>2JPk|n==s9STx=p1JOy6~C7f}A-E<8& zA`7s?^1eZ9h`Dk|JOM)ooTgRFd3`BMtBRe!0jE{eaP}}3>i=$aQ!OsWv6X+5 z6^sEPM;6Ya*s_-q!3l*t5L_o-kpDiQPmPy=3$@v zHVPqReMQ>WqZ>iOLIBuFTd|5;(G|E&-qhkYS}J^+MVx+;ZjtRJb>CEQI4~lx_byc$ zR^cRu&K5DUGlF;vf^w&it*cn8m|HGAZ!NY!nk|@w&opI=8GDoi)7i}po}0d)No`Uj zTvQ)O%N{N8bYO8km3uYV9`tDFu-%v&#K$B%l1f63hzy-#2dsIkuj|=Qf5ZH1J>~s0 zW>IX87Nmgz_s6SftQqUyBbCIbqT3=wJT4CVLgSheov+Q5!HD5aDU#JGCEl0g*XYqb zwNQ(jKSCZKG1AVFL#Z);JH8BP+Kdm4_w&?rQL_yP3ewEaq#W6I!EeIa>sARg%CL+o zH5{0MhdyUJlBo`56Od=%k&EphU%;+ zcpBXfln`$;AQcwTfD?jdDzSQrvy*SA@5f|zcM7{=sTl8JQYvNrEw;MFd5fyVn(ajt z^5^<2wG3-{?Onx?iq&iDjxl7P?qy~x+Gi0p;a{|@1^gr=e9l(Zom}yw4$ni$yrq%zB87F!Cpb$d z1~wSVFfo^mnH4SW&)gWtlgG~pT&lw-?ZBi@R!FEg+Hul%(QW#A$m9X=VTiGza5~SnYMX+P+a6{-Rx>(>F)%Qhw~B$wz%|;89Ip5J_9x zpjXEwV%Da_XoS4v{9M}E5PP+Gjkw`DN26fJt10&! z?6sRTx{4;U|JpE*hi;oA2bc#_a&^@_TBv zY9-O61YQq=a1umY6OH#m5VPq~OO^NUqLpcPu1YOiPZ}g(G2A}9DJyWl={TNY&yw-GoCYmH7rlG}!X$4hn zFy%l~hoVxAYoCTzAceji`dwJEq-EBX{x9&;bjc(q-O8wA`)1($^8E5wt60n-?XTi! z!cX`68C?{^5Dy>fs>6DQndb$kj>zr#l`+q%dj- zt{@VO`4~e;aae$xCb=n1wBloyzu9gm{8ivyqBt?(Re_HRejLHjPBwa?4aL7Njq>bKSa{lTorDV|m7J=(AONqxU?bJ9CBRzcXh1LWD$zSc;VXJdEombr{>zgG$ej_|_ zEqvKh0wjl|YYqb8Lt~5@^P%ozTywn5m|<4BOCX~LgHjddW$r!4Y*(sE_>6}ItUP4( z35{%#$+%^G@%B&`8d}|Vl0WNN%MrPulI8g>(%rdk@hl_Wqv*78Dtr4XKN-vhzgE>g zQVCGpwQSxE(in^|)Js$NGN;cB-IjDYztnxq6Fi$1uWA?lh5Hth~Z@*Qr|mdZF(``5&8?|dBy@%+wnVnAb{Whl}KLUW?ftqc?Z=Ul^}05 z)Zoos29lY9#14o_K#s;zq#asFknGs#48(^Em_#X zQ#uGNRoTsr1>xbf#3ZMMF|5$6+KDijb&y4jtLlaam%9G;JBVKV^!W!8QO2zjrkWCz zGBjYcARgiDK}d@x30GWZ98CX>T%-_FR#49mE=THK{b=r-3VS#~NVH0Bw+TZV5ZN=W zyj#vh{yLaN4DXa@*JR{ss7R=AO9_1=CMW}rvKR0l@z3k_M}9$!%HD{qK(<{$Q@$fm zb)`sYF)&Mmo_IGqQ3{mv{TV?i^;=Ku?rRSkr|eOqDlb@YFfxn2>j4KNvDdcN1Pz%<7zseH}812jR zzGyQ1`aH_u-*Ixov(u*wQ83~EQ!s&pnW)FZC{Z{-62pT*n?cVZv1E+uX z#!Om;y(wGgFpxLHXzDU(RB^#T5N7Vg;50r)J87PwttFP%wsq- zJ(fL(!1HJ!lxE^f33glpG8bx=q;%xTQqCYf^L=)ukp?7x z>mSW(M4>mRD*Bpwqi5VsX=>#JkQ4Ad)YRkhLh!t}lxLSOh}ipvVxtRe`rHR_s;{0C zULiR)xJhk`5PKwDrqgiL>8E1>S{GPkF+i@FTDe4jk`~O2=RXC6>=Q zZ#BbhmtGo7w;V(sPEr}~TWJ&-$MH2O6Bj*f;^aN;6jMD8P0_^5^Y^ID=UAsA?-Rbc znE5k;m|Rqf1clar{@^k*{CQ$QGlBU%NUHjyCM)&xAoNIjkaG z6;7|tAFW@O-^*?wLAOVTFX@-cjgJyH|Dcy-vEyBs6s2U_GR=YAd=PBzC%iJGJuEkk z%jP7%=nBPxG+SGJHtBFe9BrQ-JHOV9DB9B_aFcZi9)wD_gM{PAYWv6=1}UtdF`2fU zo}L7LnK(J#bxP5EsHmd0NGlYydg=Ti>6t+h|7@IkpRr(8nZmWDbf@#uOwMn9@BU;u z*}}Be>w!p5T>ju;VX)0v>&|M{VKiW2&$koZN^<=>2OXj@5vqmAa%&c8e?G8kdD1Gn z&QJQGi~J!mSNU4;a|7p#xYby}#}BNS=zXO9H%M3^-TA*MS~k{VxcCuk?$LqP!yYCY#3kDWbkwc+EdUV=_0 z$4Y)Xs45uC$>7w#&P;Ib40TWLSavBSZC)V{GTQI+Z>nr}v2L90i`Cc#w*%{7SU62^ zJPH2}>h>S;mLy&2Gz%tjAD|Q%yQ;Lha3u&>iL1ocadN-Eux#v~6uF9dCI-^da| zG?bgMs$GvwX_=*f=1F?=T>~Xsy2ZJYtPiG3k?|5#Qltl9Vo7W)$0S$-1DCt}r`?+E zzeL5P4|JwaAmdcT$L={zTuL-l0gn+NWaT)PM!7C8n@B)`aHc${P~%4tNSau?IX)Wh zOTlmeWM0&)+(rfAxAaG|&9Q%?;Mvev{50ip9YXo^?4LQyvijS= zZA+zBlcYS+*&*|+ScregoA+0?nRk3S=58-#a}}#PsSBw+Qk)?7(ae1F zm-r^e4vTx&#or#svz*p*NRpt{xyzgM&@qjI+Hu_5l049vCxyeyP2J}ES6^Y_%WQu8 z(Jw)7L0^8f{hjR}vLM}^z)iGa`d7GvHJHi$O+anF1+l?KdJ!msSm>wW=ZWWv{brLt&N^zzFd+Q<$+S;aZHyBrL}8t{VPrGRvr_Pu zT$t1W3Ab;oqQOFKUHunW?9;eu(aPp<5lH6+>)UqDALv*MCC5(BM!p8Qp}yb$6t7OV(1D8}%gq#wcQuvK>DlNH3OF6| z9-uphFxO_oO{m1|7K7736f3^@>Gbaq(njdt*DDHY6ISZ-7EC?~K^HZnXlXI>vdZAi zG;APqBZx??`s7^EGMt#pg1wYSct!?Y?j?o?hR(H8az!JCj@0W93Mro>3S`+61Ctze zl;#0c2#=1|%mG#OT9WZ|H%gT!muABxH<>xvC3)&c7Sdo5pY~d|H=`!=(-m0D+ukHJ=ITspc2=Ns?>NYbaY7FVW z;U^5fCBnfB7R)mYpYwL~E4eS)7`x$M8LZq{cxTfE_+IN*Q=veqYYxbax_?)h!1NR{ zj(^#e$8z_esRFfhmcNdHZU}ux1QlhAJb$N~`Iy3W(fn!koSiUQTm6Fqz%$UN_Q?`Z zX^6#zI&d7|nS?g4E)Rhd97ak-Fqet*Q()$m<7u!Xx$Q=ej0BLlNDU(5;a-q$#IpIl zB-gBF!$@b4p=V`AB0-Tz7||7%jeivXf$6s3uVSPPHfRmap9Uw-hUvQQCX#^ngwEBI zSc;nb2w}s*{B=DyOPj(cE3RuKR!k~`?#0gcCB)ktHo@k8$pDKLTB(#+Z;FWG@b5^B zG_3eoDc@v4>5XeJ9?%i|5emsk_2@rBp;SfIs6=!_!!^NYJ5bis`C+Mv9~1n?RrM!^ zB(aq)7?tj6>g)21C$UzjRzWXO9(ayc$pUV7sYNyov}+qar=|6Ahq~qaeJQP(XM*v5 zI*tb1u_FLpD}EmHsKx0Lzc}mo@f+NLQXjs67k7vC7I?p>OI?admmxfYf)FAtznk5! ze<&g+@5#JBQ77O`#^-AMKGhZL2J~YN&kR|FTI3zz6l`33;}8Cy@8G*+LWORcK|@6l zR{QoH<@PteEeAA7XDtzrUfzh-zWchDGuR z|54(Ee_@7|EcIL)J0On(wiUd=N@&kD7`rhM0ajwXr&03IN09jc@JxQy_y(&@aQex@ zx=5u$@)!rmtOREK^w~TNsJVwEL(y=9H5zSAN6^{i$TKs)x)52}OcIenT4|&RRoy0q zjzk6zU^<8fL`~0yg8`y3A=E;*GK{!)c^X9^0G~UuYZ`SJv6PY_@=py^6!0xIp3I04^m^(atDYIIxyj2Lzaj97jaTS$lCFdxUt1~F`4VHsX0jyeBQV^r_ zOj?1ed6%m>t5<}&zG{<<-U*$hZP93BEeb3tY*vn?-KoFDJjDO{QYh7~1p8n|#;L6f zIHg%+O!u?9b}<{%a1q?Dm)oLPAxvlXvDG-!j1xL9>6qBpM3#}W6IcSbjzz-?7kU=h zU}Tt@&}t5PV!>&#&(%vVFGk`dh@v!CJegMlEtp_bW&q%pq9k(E5)DZ%+pmbvd6`q% zcG&HhYAcv2ZCXsmWoJpWark8d$J=n(XDBYTQ_jPACl*w^2O%?ja+fK-Y&F@OgKQ5u z553B+^Bn}rbkpJvwW z0J9i#5}emc)d{=RN`DaPzZn-8!4Qh3sEwz;&_-zll8ZKC@?2j;sH~Th$?VZiH~+K& z#zmp#83Z%oVSDbcD4w(;RB1{n1yJ5brb7Q|kh6==U8EdSuSuo-X=Zuactqk=^G9gd%d8 z%4kkge!DNNWYO*n&oFboo8i{PE7p+>#jvbu{*!7-yhC>+`7CoTqSxS z9xu5F7dd~7+SG<01;3MxfnWB}4s*+xF}B*$By{--iENn}K2#%-5F2VJQZj)oJB0Xn znjG7fZcg%^cRAp=N})SI$5D3tP5&%fi3VUPbyj`Xf|`L%J8N<;MXL44ZLeOFx>sek zc>3G;I_goxa%PIc+`&ZEgM%PNU~~b30BV$H9oFcK1N#JlgH1N3?okBOEwR|q70@Hu zR=i$)=?O_mhwDH3CGlV%?I*V;`qIW5EdJ3!5yG2tU{*|p(x;Q%{Ep^1M@zTajH}bA z!+qL3FC4G)JnT4k*8CGX^;&a7G4yaM(c1F7DAY~OASA-ghCF3jj>gG7(Tf5?0OuXu zx8I$AWxl;8#KFin!H3|h(9+&rJ{R3R4#}a90K?yRCI4Jvwl`vzbdoaA8C8CkVQb8t zTRK31r)@PLgYegPdj7tsk9)Y6aO%ke9juITHypGogvP_hKWl%b7lxB#rkq}Vom%k@ zLvMJTmn^C!Do(7}!Z|~f!&jhDfB1qk*ma9_J2;*MO&B4$+{3Dzn>to%$AU06ln7u+ zkEA7YziXl}8Yp5OM1qmSCczJ(6{1s>%srwJI7?a1>AhZ>-wEK*vRbo`c>o%OF5@jH zqKAePbt)}@lhum!zz8gwDl1;0=E!5@U1-e}n#4*(n#MWxXKrmT>ddH0s%A{g-Kxqz z%nN0s`TpevgF+M%Vg6%W&5*V?paN}*-ndQ``S=Bl=%S!VOlV<6lxLm34TA1!%wtO! z5d8qBSSSL4A`Twtlm?E_I!)tJ1k`uOMvhbunKp1XjH+T;$oaSfnk=(*>rM8l495-ut}s>oIY=Ix-*`xfbZ0 zH3{eQUd*hTXL_|2c=4k-lc?j!6+q3!sJ&MZT=qr5oK5*h(y1_28dnUcIgQx>y5u#5 z9(ge>S<67DFUxmEH9&K;43d%v#{8&ArM0n4p z4QUk5E_iQ=c5`j}Fp-%DiBTz#0%9U32aVEo$C94-I(Ll=TTJ`a8h7gnQv8!9F(oY* zW`RBZ{xord{cZe6*h|A&31terTw0%M)2J;dB*QF?6my|8X>&+ng%Hv^%Rtq%ax&>g zc6hAW{5Vk%_3&HmGTY1tnqYe1aEGU)8jJI%daaRM4*S_?<3*Lq@5F1JZUm1PTT}j& zzk8u;Sh@ESfdSO2TWXw8r}0L61HLR>W=|{0gP)#Kt!9`j%vJA5S?+el;C{I};NH>J zBm2A$b7ru9486RR9vj@{{m={7lmDxv{7!{_`TT)se-jZhw+fI!fc~|6;}8tF3Zo=Z z7rM^unKfk7`%KC3Bhd%#5~mCiFp*Qde;{~m^Y00pjQAJWh4rq|TWOrdfIGv;+)$oM z9s3i=d>L{3**DMY2A@2srD}-WM7t)mo6EdJ3Y{#YiC=|$>|4L0w1%JCvmBv@8G%2$wb%YdMOQvMucshSXP8a@92~6WN-x6 zHI}>S|J=7058Dp^R$2R~Gq5ufJeF^zklnxOD!W0%KtG zJXtLMjw=LI(9DRT>L)#D{bmX%F1>`0OEUGJ3xfqz&XD$nLOVutW{+c8-{=ebbsU_v z=aRM{h|<9TF|T$Xu9#5h@<`a_0rfs%_5WaC9fJ_ieNK`{6q`PEckS$QcJ5~X z&yn~)E>Rpm#t-K|lt_ze`G{D>dX?C(42t70H zR;1Ndcg&li>t!pxvJT5WLh>a8<9SN0qd?&JT}Y_eEjp!}j=_Tsv@16E*GRj%NY@_v z_3w7@#?IK=`6D*j?h|V_uLx(0R6pw?bCc$0zJ%Fj4T-`lX(*IA!`{3vso6ti|u9mT4DxdKK>q@0x7G?9vQvWGgi2F2b5+ zZ$b4k6ZI<4@MQWFX0{TOv)@eY{14a4&yNGkoebjNb>(jU_YjzMEx;2f>+P)~aWL56 ztAY8clDt%qUY!Uz%WnU%>uN|n&1hYhBP)lXNSSTIk|DK|>v^1U3()4baVu||Phk7*7?_`8N zs2Gai#u^_3KP-)h=C~E2%-)OS9Q5;ClL?b11~y#qEKEU}6Wi!Ol^bz1i&0D6qO5O0 z!qAXJX#|F33St=W7fbY)u}%ftK7-*$-0dBinqFO1>NC5^%F6%{G@V961o|J|uqc3J zBL*%nk*VyLS`ilI=4c!U(;{YsS{d)`;*+$WAAv9n{{$>oJiQIb5!LA#1^ z+F2TCoB2$L;+nOdKqy;7z8P=we z32TQ|!r4kqexv$BN-4q(o+qqN(?3J@vBUZy?IuwEH_JELO~3S>5g*mJw}!JF553-h z&{td*baGysT7Pj~z_poX%W&BsEMw7JW4@)bm!vM9Z*1-*&1e?qGcv{8wGMdguDkta z*~F}(6+bX#Y5mEM8dg|H==1%8JQ?{WVlFE`If8Ed{ZEy*2g@JMHFDOY{FQ~zTAbsNkb9};Sdzg|$pA#_>S$V`aMw^G z(HLM;(=?-x=fN#f3ARR+x#<#L%UnTxkpGPGb!1-fH8TI2GxH!5=_KSdZ~6jFGa!KHNsJInporVI<*#$$@b~n<^yEG*TJ8 zM&n2-h~PM-G8jIVptzz0_NUjS=_z+@T-`Z=85XljmCUdbAtl9Oa;$6RcRNC2c8gEm zSw424NlU4mhR3WJ#v?b)@**rvS<6J)EYKXAwtDJvq4fvBN%XtHN#%kF6`-_eLrra5 zc=@YkUesI4?ia3%5}v!2&p^XcgD_EyDqIiPo}M1PCQD9ozY7-?&pek6E~C$roE*vdI*wtFy^;)&z#qN zi{$^yPZ35GOOcxc-F9*bUhrqriqT!hsq3_Q^EsVeF-Sy zGLrSU!zf&fL4`C{L4ThWo2)a-jW{Bui> zM&@{p1$r!?(oR3P4$8cj`cH=uL^#yWIR=WAQAi?YK+>cjm$H{9$zaWkVnoZDtOPMu zSJT;*lUTw6n1{tRUu!Q_2~9tn2B*8ES4q1Y%Z$2gMS^if3-8H)N!y{H!IqusX**a} znR==nnF=Uj*k4-0zkKsmjVa@rabFQ2O2m6!&6`hxqoSa&k*j#*#*;Skp^9cRl%?a= z%}Z(3Nv9CS50z_%JU>RS$oE5i#G&T5p2>-J?Tl-+|8CZ_kN)>6qT_s5d+mp~Kn7ev zbTpJ%nXx!@fBRS$%m01|1*VYX4!wUvc4#SF*|hGU`TjvdYmWXeOa$SSW31G-b!>!d zn~8s)FB7i^W7Xunyco_}zg1IGX{pG)lH|#TUN2U6qrkfgf|#{4Qe192EB3NV5{Jt& z3&Q?BZ_VB0{=AflsBUg)ZzQRU(~3mMmK^oC)E;mYa2G;4*C~h-xU+;F0MW~+pE(LS#&i2#Ntnmp@>blb6A2JL9dZ1#ZdJ$Sfu zHD_8#!>%=-DZ3r)>+4k0RA)H9Vvuhnn-6`OVAL8)V`9Y{aoV;-<(epq)13*2qC+s< z+~1;dXVcp;W>qp!PX17Pz^XihMVq1`T<9{V2p{FWiEPUIQ2Mp=U00L0etuXWqtoug ziv<8iM(=tV>Cv|=bSPtbETyxV!-(Cj2oySd&&)>ofKjd-0nzL=$~E@Ddu=}9at#gfuO#uSPg+`^L%DMzH)bF}EBQOc&Ru{sD7}`HLb?;h34DF(N&4I8IpsDVU#WRj6Xne)Iu^jF|JRgiqsJ( z%t#rwTvCpjv79!KtkVTz@nuO34s<~K@iTS@w3C)nl;QHS6My2BYOE#jJ+WX^^d}Ts z(&xf#m=tr)M5)F~QEe6aiEIX-Pa!;Cq!LxwZ-pv`X?<%(-dab}L%C{@Lq_%FW+JZu z%#!KBpu+70ZGKDExB@H}Gjb<${WMMX2O$fZCE&DuyZfm*s7BjiyO%7LXpFoQ=>voL z)3;m>TL2tOM!{>!9>f&a7pAo8=S@nS)#A&BnW%{PPcH!+H@L$DX%OPwL74kD@mI$2 zZgXPr1bIF%p1+7BP?<3Gf zD;oPUya@@3dxb<5VTwP}dR}hEqUW}oe|lv%r~jR-XgvsXLwdWpDd2D;GMys^AB)f0 zAIn+5Hqb+S^hRy`vFZIyBOu&j0Qn97+4dJfj^8~ppF&KoIyRJYc#|dlJE4J>x1TKxANL9<6|JbZeHtXBxJ3S`XSAtA=tmm4` zAt80>e&;i7$U_mZNw(JM$KO>SgO!37sWCpTL~qnO{5mzOYgEJPjClEVU`!_gnyjR( z3{Nuwjculyhv++#_O3Rwv%pQM32(hg#yWLannFOYN_y0tx6H4|5-6AB#8bEAZ{(Lz z;?lc=?Wy}t4R|?G$`26QQVydBuHs$}ptI58hi1p$_t)%&bIZ}1_dThN*EKtlh8PiN zdaEQHbW993#Lr{iUc*B}7PZ$*9RJ_114WgIS~YqqiHJ#m!4WCNf^uISPZ%+a;jq<< zdi8hb`13Nqctr1(8jl_5ELAsYUY50XIHXoVO`?9o-VdqV39IILT8~GOuu}q(jDWLL zP8#N>CW_GI`DF2SK0O`EmU}omT&uqbH-+#g3TYK6u0+w+Yeow{V$ago5dT@l=1Xu#8 z71AU0%a<$PsV~#$j~aJ@7OtLWH| zWCE_IFli`0d=wB0C$-*TBCJx3#{5QZ-giN z_3IB?b`Eb0vNfO8PC`-D0imoQ=b|4*r`Zi#Nm)hQz^CQnIZ|ldl*T?{R6}1HgNf;t zg&3^nzKcag>d8NOv;r?DB`(_mPZm#?n_0D#t`DMrf9rbx@)L{iksh+(^UqfxBqpW* z@(;dq!6MSPv+FYde+Jd0j0sG75NU>v(Ri!omr#y)N`prv(%z8Rp<34 zsCA~Cme$pqvQk)WCjG(Vy;RrZFoBqO! z^B`b7NukrGtuqU-Wnq#UP+7nlXrdO;(jo-aL!&)y}hn0Ksrm7!&I>A5Wb?q9P*4IP+ zNIV4}H3JDUS<lKhSEnqvc z>pc-s!2)3xOSP^Fc>M44=VaoXP*A~4oIv~c=SAu6(L!-=(Nw4ajPC8CUoqTKW{r>+rndh^vQB|XGKh{mg1{EkUq~QSz ztWPr(e512xH1oeu%GE~*$Xv|IAYUyfo#;$9bAA@0gIrVs1_9$apxo$!fnx_Rk5YRH zB-rG|7r3MvkbuGmlBBiF(r_TRAUkB;(qbyX_MiT{zSp4CN8bDB5mu+q-bF!4 zDw^p0zk~QjF|-BOIS+QjWC$gUdhLw_H;&%_9*(g8V6V{{=Cl<1HV~GRca?@)PXF$9WSFTt%W3CIK|flEjE)voVg$g zZ{^q+RkQfcSjI5JfO#V&j=m8Atl*&oHp91%huaXZe%RGdP!6z~bh7E)X&(7wq3EOO zF!O@%>~p_2syb9ZB9R<7P?yjUM!tqqE_)=>wl@xF)j={$39ib3d|lWY-a%U>vR-d* zxi>t!%xIZOYm5U{()Sui%X8%}Bc>$VuEYjfV0prs$gkjpzUSGr-x)2=2;I=qi$W4S zC@JSh3-2gwqH9}hrdT+w;Lgi)lH(wD{~h?_^D`?XbU9y&sSQJu0C{-qDAAT>H8vA% z8bFr$*nVPFT4{dVN@7!z)XOWMk%@#@;lj9Z?Z69#lvifczbgh)zn=nZPo9fu%xd|_(%~>@kP#4% z>Kpxd{~DW_%49SeU)rQ1$C}TJ@aq3v4(x{*c3G8>!<4wT z`2z9Wz+AXx$SNEtVk-djGG@J`+iysMzlN3VHj$y~&Wsv9^WsC>4ClCPNuu~_1yD*C zc0zYBK5l>@)wBhvi9L~KYfsKozciCXiW;mF8&D4 z9|ZB4iPDeH*0S6Nr%aEnrQ#3=%p6)=%lE>&dkCgf2@=RElx}O;_`fG z+Dx-Voz8^qFh5OmKVx1H`R*49_aw%7{FqhW#ye0ESRTq7Fm|K^^KT=D>Z7r13cM>d zA5|Ha7{VrMu%k4!+&l}HjW~>DU8K5$aNu?1`X!3nC}MB#j)Xo)$8HqZMFkSy_1~C! z>9TJ{%jn{<@@zbMwuPp&Gq^fdEp(-vvSi@WiNhOu7uC^*J~N=IkBi|R++Hq%0zn+E z>nYIo1MwMxM744k>9Dt?7X^H_0OIf1G#6qy2Gg*_3-&eF)l@nZ^ zBj$%xB3_*FSgYMH~&%T17K`A@23a=nMlAzyUV);OREs|DSN$~HNL zjA=6x(<~2bxY$;8?*mT5rcW~pA7t~$B^dyD&pumJCAzp~{*DOc=%}ENxWBEPM(!qv z<9`2kln3-)Ff)w;r&dVQ@v|pMuurWg=60;&y`=stz_v6tx@H+Bv`UDAd9U)%Sr6Y@Xt^1(=QEp{YaeefSk3?_IoMNfv z-QD=KVK3=fPI%dT_&m%2N-I*A0gz~>=~7y*KG^UnYDt>{!Ak*JPM)kQ{#&y4<&D5y z56s?vBTVJ3Rt=+No;P5;Vsb>EOvxe3Bi}OYh)q9dCVh&;dJk2hok!}lPARRm$>?}} zM}W#~ie>^N8sFNEwm5yhljII^ZQx@^eU`EF?JsP!6l?K!0s5df3xN8GJdb-?z712?(Wj~&aIV|^(T z@Vpw%+-vUfhxzT@Y%w#LJl2jTVXL3jd?6&Y~ z1Qw?d5fHT$@o$8H0W}`du;t+H>c60h(#^LyaoiX?^XT;Xt{rYn7HLMmX@Uex#$o#D zyA^LJaQ{Nr`@2Xu{Ojc&IXMaadmB)JUPV@hv^8zlJ1?n=xA_^(VK(@ARI?O?uBiN$!C6r zx!bu?AXz=Os$xfu0=_ictkIXuD4G7o(<`hKlmhYONi4$`<#=&odv z59un;@c5bxsq35<#I!=?tGQt|$IN|$ZJ_|7SQxSYsPKg=limc^DwV3}ugBTn4LT=l zDvUbQ8FlmA!tD&R_HFc+toZMC87b#oCdU_TH0hnT1gh>#j}8I`H|8dg zUHML7{Cer2=GW{nPt^PLw`fRj_pgjVzbzvPo|D?y!*jY9RU09N4U^IMM)(rh6PX}MSg>_rA$Nv7sZ9yj`{KT?LQfrU zpAhvqM%OScnXGWm;(E+=&)Z!lav&-&t~D-jl|!!r2nj`_TjvBI`wV(VS(SWHC7rkJ z%qkXfQU+?HRc%R*g#Z4;DgJH?*wtwDacVb%e#iA!v$Tz3m)w~@UXHJ+r>~prF)s4w z%%{&48izTep2(4xl<8aCT#KCwS1k@!>OU=FDKAZ^PNQmYCFlL9ax-M}>-9T0IAF!2 zy2^?MK9T>sIHKkiayTp_N?6p&9N19XZsag3N#g2ui0Qt3$@GygSKLA213n`eFXLYZ z6aFZZeqqz5O9|3oPdq2(Y=7Di${7r%{FwpT^@>`V&cDZG?%L6Indx-o=5dP`MeO8Y zuN@dW-?D;6FL&K3g}z>$O7YfJt}azHwCl#@*Tn9j*!~XT8kD;?rAet$wL@IE7@3&0`!Lu z5&AW|FZ%c14!7#)eIIV5`+L#CqoA~Sn5TN9=bF8kB1y+{j-`_%bwS+YRMOrA!<8jq zb2yt)BeGmZoVjTZfu`oym_6ZX`2|x}As0M3j}vkN{pwJs9mr9kkjGc8s@A=^b!$ZI z&F-tKr~S4Xe&ve;01nRZS@re+&0y1nT*42`POT!Rbf|%I^=KrquFzd{vCS)u49Eym zov-9Nfc9aU)W#7vQDIsxV5cs)Y*S=KRYIL%KPjji-%H>-T*sa(zTv|K_R&km5A9a) zF!8!-8mTF8(p2TFN_B%t)l>fD@n}n*9{~|5>O+ExN7*@IOGj&}pb<2~7j1dB8N`NB zPJw^lH@;AUL}e`hrc5U*4aJ;**S)2;xU-n(n;;~VO0j&>q8~s$r>3aqD>fT0VM}4Zm(y3U z(d0nA`v~-BS==9>4r&$O3YovLX@d@vDGEG!6ZBgn**@{n<4*UP`aEG&7qWu81P~^; zIfx;}EdKD0OY+iu)%=;?jb?V+zMnO8JAq?DydJFHHZ)1%mdHs>rGKRK>SK|lKsgCh z7WuRbQ@L@4PxA0Cnh$=1%nU7;p_<8L5SrMaDA0C7%UVPwRPpG3_yDx!=jYB>h~P0> zUK#*utut)g`_BUKo_*YS{}}`3xfJr>n2~_vF%XR#z&4zSv#k~ZMmF?!aVlz8Hg3Ew zB5J%goCFHd%Nd)hmD-o#tp2EhzfoQX-v?v4?TI7odvVk8A~~)jmfKhizt@Q&ph(MzL*Kcpffp( zKxx8J?1E}CWu^)GD(9q_fR~N#yxi1m@TScMA?lB*=&eKvB;6XC_kc==V;;v24qTl7lOiQplAE%J~*W&Yj&#k&3X*6Q}sXeada@7P%9VM!~C-5NqWyqrQcH^dN$G?=KjHZ86Wedmea z+~xFgSUJh90K~~}kyZWp)QU%2rkP^p$yQs~CtF1B&pR&vo2Y@oKaGDep*kES?Toln z#Xz5QGctNCRyo4+gVRukM?~K;aLl7Jv=8#Zu_>iJ!XHObY9AH1Yma(X_{T3gXs1}X zRHos4y4Zh#yCP)3jZhE|jaA5pgf< zY)xU}d-z4{prOH>OUs+X2s%Re_fxW-C|(Z*)N=37Y6Tz#eI8v~7cIp>5tiwTHYkHl zTjV=mq4He5SX|*uPpY;xYQSiAOW5Yd)|#ctJ5Hw%xMfU>|B`v`?SN&@JNxkwpB>p6 zN+aK9L&Grhpz45co4`;GSa!#%B%YPVDsONWnkO&FDtyR2TPU}U`Ti|dGFK<9NJXeE zA+?EtlGgI!du0ziqZDC`$K{&Y!wF-&b-@|1@dRzudr|NG*H1X1V5DsO8?VrApXpJ7 zI0}Qe?e!Zx$cHl!LoXB(UAGd_ZPoUkw?4KShPg~ z+NLSq>)k3!`P3&G1B3Vjz2-+c@303(B(aO);^qV~EPgyr{p-x z+%ZumTfT`m_hQQ2i z@oz4!shY2=W6hDENxbJ;za>aB@(E*AQL&!LUym6iH4Nbk-{Niuy!Tn>aba?9*0aS| zp%e?SL?1A@A+xE{&fW*Wjt>DX2)F*D^!2R+@O9!!u^vT@OjN?KSD?$mO-@DO83~-jP*|=Az z6+IaC>6T~me{&1f+BgHbVVXqUhJsGH?<6G(giN>Pui6q1WVE0O!Lq|tBIU8)%A-{$ z`pmJV z#$x#?X%T0cn8!&OC$uyW7P_pByx7$sQD2-9(Lv6M_j`FP=G$^)ga)apyG_zwji zyr0}hKGV=<2*2V~b#WMTWf33JE;9Wx9IOxGl`C-~%Oo6sl@L@Y6-=w1ayf2C0~2Wl z?4*(CG?xZ~Nc1+5tY6-2mWl;pDLN-WgDp z=s!)GV3ftiGna@SRXbm5bBEex5^o&F8Tc2qXJol87zt;&evBumdqzrk_wS`#>O+lR z&d@!K`Rs@jjC!HR6HF7a19>Jo(Y{7#*q(z|h8n1sRvpza@M#@vXJ}I{ICKdl^n#Q- zo$OR>gn+2F+!yCR?WIYZIgXySGr;D3>#HXOcJn2)thnWrNT0^Fqbu&jr1a>8zCO7R{uceka_qZNQFM$C z3esDvHwfYiwwvf7|8M&P;=dja2W_Z}GSZAmU2PsMB7o*3ynkDlY=&hzp07KR-2F-k7JZnZZD23sFfl8 z1=nLE$}(A;F^wcANE+(?H(uEB!ff-A4!;J);iVdr5-^>#%Ca6!^A!I#(?enhd{K%# zhGa+Xo+oCZYC@zLQtiz*=im}#wpxEuNn_JZ8g#|17FO!>FY5cj8%@@p8RG4-h8e;eiKIQ+~;p7bkG6ZZh=EElIw-vn_n9 zA)AQ{o37#NWERs?f_4octkSyNXz}9a+UbSG!GpR-4QTYwigDp4Vu;t$OY@|g!>wI`%ynR#Nj)pg?}7<)Na`eR&scuTm&)O)j3BNR zMahyubfLlRjSonampGoSbAkCTzrlVGK!OJ0 zEvw8Feoatz|^|=U3PAM5B4;?G23f*5D~Kbs!*oFqaKn>WZ<}fn+Qc+?2~kN z_qcrOR2G-xm{?omM3{!4TbGXRVijFJ+U@q>Z+F~@wQY-yi3MU6 zkl;_}Mh5~e@Hg7MMEvIo^H;>C62(?g zt^)6fzd~tHhgwVn&jb=k*zJaX3aaISu}-Ko)@?QSwq7~q_RzVCv7`Az>eIy)=Xl-?#}wBfL9kR* zLI5{L2KOppmx({URK7Hw45&}^$M{xRiN5~v?5Y3SuVgkdj%b}H1bNJsf=4P{H}o#q z`Yi+olNil&a{7ctgd#|(@B zvXizZJPsJ%s%fRNO_~hdg|)TVdcBT+R{b131wfmvf6MzH@9l;D`C9+lZ;&b^XukRl zDi*AdalOHR%l1!%1Ri;amsMBbQqsov=`<7-Chmq~L)hxQd)mIP{*D+Bu)p60&-X?a)=N z+f}XBRxQlQe(|VChA?aN(n|22Ot{WAe#LtY+z2-ZjsV={Z^Zfp z22YF%q7gm&Ik55T5`4QtRhz@v=Y%j@X*A09Rr;xOFzo@7=&r*GY00A^nhS;Ho0InL zzXFawFU$n6gyU2?q-(v#!4!h0sJ^R?F7&P=OC z)y_t1=K=eRSPa_HI<1z)<>il$@BYDyi)*1teSh1$U&~KfOf3c8bZXa`h0j|Ch<=hx zjQ|lwF|U7tkA0ed@f6T7UHVMIOwd>wLfhX|AwkKEifBnj>H6si=xPG1QftO*s$N5J z`5!WDMeP^N$l!h@^hxQ#97ESSl7(sL8qAZzk}1eifa0lVvLh{in_!h01Apd&tor~A z)d~p>WU2J4doKv9_L26Hn}dvKSy2|7B;@%Q(G75*ZcnsQpJ>ah`!h)>d0{tv&KBM6{t1A~ zXA2YnTBb@X73(Ekyi(onPw%o~m#b z%T{GPniHMmH_}ZmoXn1WyuvSRt^c6Ake68S51a)~ zqG&k11X+_Nm4qodey=y^`OFUsQ2y5$B4htYn2jEf*k~$Ww;Ze~+dfJ1HIr1U@6FDBR&Z8zR!O*FO*Y8uK)>J>#!zsda!9E<4{4FgT5)bx(XmLWKl`9{lk={N zZ9tIpY9W0udqX|(L*3Tgi>Wp34JQ`cP1T4EKc(@`XYfOPgA+l8217aiJ~p28J}w<^ z%pXb3&+qnV?*sPh`SWYEf)RgFs1%x0_4h(PES&u(3tLP%B`1!b#Cs~kQMy{(Nx@1nNpyxv@rFDu_mGW-&JA5M4jDXZpzUq6+>lbKh4&Hh2BaJ`oQ>lb_R<8`5u4 zL7*;EuQ8A_|LuRRn=XBC<=T>s8c=cwOVC^yjp!zo`p%xAP<1FA_!6=EFa zIWWB%^?4Vh%5Y8EEYkCtty?R@o#R_#+0 zQi&RC%xT}JMIY7co@*N{pC(DtAD<&l!^^*-Mdp>~Z+E`mL{qp+gg3lrw%OXj2$bOs zB3VZiQjYu})ypw8RLh%MtjKn4GkQ)!$8i)wkW-QY`{jmuuQGX}lDH(T{gfK4G5>nk z_!<4D|0`{y*9LvF@|jgNj8loyKOv3WD1oJcHEhxMi%7m;DNb%q7`}@@L+zm4$j$X? zyEY9j+<`Lk1qbJ-NN~LLZ!#w)8Pj;32hNY_-cmHyMkmdpm2IyKw2Rg(m+1J!B<*3Y zuqK?^X_7c^f@)wDEcJ>ZXD|0jD|+Z%cBF_0T32rAa0Pi9cCUQ2qY z?`2y$XjpTj;aZjyN;Rw^XM(!|N-gwTVPkflQ}wDkCIh|%tXk0MIf)FGstY)7EhG6t zJ3!1Mckic59W6i^dBx?5M!InYpS9A*I}@zf{ei34ydkhw0^5VZMZg7NSLo}aohQQb zw1O#L(`loWQih=T!q||em%|Zju^7pM4i}@tR3DUjw&Znt<$-oOVzT+Z4spen6T^-7 zyO;Yu|N1A=jqf~llgAO@yTYYtm+I*>QB5HEmc>LIDwT&s73q2v=*S4qU1zDwF6~6A zVQuSD<4jZLX;+z>@Ja#Ih&`23E!zfd>-ev8zCh&h&T9HW1eI&y9@qGt^t9$G?3Q+? znsDn(#nf{3so-2jTev&k>OM(;>Z$ZFh4rqGoLcKdlB(F5r9H(`bSLj^G)3}@z}>vT zg~dvZ&y%aER zlc(5%ZI;)N&@alsEgOt~9hG)=c2JHWP~nd#K0@RCnuK0r^2t*Jk*vHf3kl=7_C95e zJ^PO*{_J|F2SfRxvkacLM5H}wJD6MpM2bpGo%LaQ&KpcX&s=2->l?Myda>6c_Y_8ot%$j zipLE2x+)rnnS zPnqvCcYb#5IN~gs9of>9owzLFLQ3asUZ{vZ-tZHyUq+FGuu|oNa;9BM4K7|r#@M?L zhZ!%y{JUMV(40OOUaP-9+pe(n!BF<}1RipF_>@XSRZ@zw( zchx0x_nEx^^UhZ|?KR^^&#?#*6SkxuRFq*L3G(58uNJ)j_;D!5uF%|=l4Tqy^2cGA zapvl@%C&z0V_FJuI5zktu_Bu@^9cL$VJZq`H21^jQM6^saYva~I|v9KzEM6OS1UMw zx)e(nF_3SfXwRB9J@91dv1kxSe`fVco*4j?&pu@sq3U^BS7395NZ-wI8g!F1_!7^` zOd@NpWXe$)&_m6irUFTpV}I53C>Eh^hp|CtWXaO-jW%EVkWVZqZmKxoY^6T#>C1Ds^0EihE4(z&~H= zmLKTvuiUM?bUNTu4cVhh?NVw`;Lh^f5R{UItd5Yn)&I^tCR7Nk48BOxoI}a~J9Psc>uz)S+9?uX76;P^)l2 z!|8XNQu*0klU7JQ=Auz%^r{KS+Yg_`+iSJ)Pgf)^;_9tBa*#3or)XFv$tN*GnxaXx zj6wZHC>6!KlTdz@g;c0lnInXjkYBa7G`8N=;P1t@tA%wSw{j4A#qArZQId1hf*aU@ z>Y#{{D$AK-xRP%2ZQku%21flcdeX7q6Dylu_@ z%hg7tP{{pmI5|RaN~&0w$@($^Lq6m)5KJFe9Wf?8|n5&X#zA?or_!FWoTJq~mhzypZ-kFF9LoZ-h4 z4ZSjypwgvIkEsr;ZAPhE|6x_jqYCY8oNI!4RfdlwvSz(Fp6Px2OGtT6wD;Tx;bRvkdF8fLgDDB5 zxL~p-s`<5D5@k|AM4?1qxd!P)QE=sUlUv1U0-rCxE{oGR$H&>*9=od9j|M^O0@e)t zh}2UsjYSh+z7%<18aeZu5|9kPct$*U_4;tT4L`3|6U#E)obkMfhSzB;636w@myZtM z-H1o4D}B{B_qB@W`F&Wm=Fzg1DNFaq`{ap(ey{TaQqLuWlZ47EOUTNj0e8E)`qSmT zXo@m>eDFj&US!&`(CtD$FS}?Ox$6TbXWcLdtLJElz}ClRqcbEk6HSvr7*DNIVN3$D zUfEh3v$W&l+XoC21j#aoZRPoRUv?c@rRU!lm)=CTsj+nx(t@2p)&B|4t*S<%m@&O zuDb_T9SGXLHyNJLA$7~BhQ*`j=+N!Fuv%arPgV2^$O8a{t*s@6m>^bT!M2lxM>71L zn4;CJ+vn+TPWb+$xu8OgYC)HDM>lQYG!L*ZJ&T)RYjIyFqPNob9! ziL1s}@evZES#-DR)|sUq1?dEy6ROhc>x7)E3#o0bHgDQE0+;!yc?@>Wi0NS>mr>Q3 z0)DSyO?gs=TyA)yzqea1LpBpCf_ewXtwAUd(Rc~t5pwh1ccKm4m;VNjA<@ixVBs1X zq>Dx1TcX3PNbL_gSU|=mhSDsF*rTuex$_c;%`7*dVeocQA8LPKv zqM_iLL*D$$`^Y^a})c29#SNh=ovFNRLl=Mv=6IAYCU&JZ|5}eHl^DVW(_El*v zfq3InUw|t_ITFPy7HaRZLT+sALatQI@Ulcw=kF@b>f1}&c~yvSVywnRwasdDfFCf= zJIyK@E?3U0C{8%ns&CTKfcnU9eUKfyUo3%Mrv)3|UD)s0D@)n*Ep4b_YXiyDi7qa1 z9o$OMBvhhhzd$zMeO=~$T9x$GhZh%{V92@~%=o141c}=kD}Z&d)bUGEq$D5Ly4)J) z2j^o$wRl&sIv8`=p&;MuU%R)|Ip*wtZK&D*WONs8K%4W_NENWOKi=63S5Gv`$%}*o!1+jJkOE`!( zR98#EtyNpFznD6+Ww5lR6TrDOr>!vOs>pIkU^mu#{yL>K^`YFtSS)$*Yf`*aQ&g2m zWQi4VYB&vr$On(bjWqSykO)Zlklv8B3BxxAKz?gOA6kJNG?@TzqiYI4O!YtpP1*}J zJ|Y+CBQ~+argxa83tXV9iM!l%Ac~(m`%0>V2jf$r0G-pY zl~&ITBHs-cxb@bzcK>?ylN~6Bd^bs+)RH=ej#*hJ0QU8_IcN;K{}222?K?k@&ijlA zI8KIL+W=4_H41npy#MVRM8*18v&Rg<&|jJTRI0wQF#c8dahXiRcs*yYHKys>7@wh! zt5mruLrfkJ>G`tk4)eWg$TkZ4H5H>JNnsk6=tp!qOuI@n0WFKIcyt=dUCA;^{tKA# zCmTlB&#Ua75|CkIGuK-aI#2_+PVp^P z5A!OmL}lWSsPhaxA(jbo!F@iIsT|cQGLBbSQ(WDL6*8$P&&V&Gvi;Pv!YW+{5CH*w zJSW+h-nT=hgptklF8CgD@lMvz;rcbMP@*F=#rkh!5Q9Z6_?QFP6q?Q>8 zZ3)My;#L&sI3>Dcw+3+IV!pYfGS5bIsEw8`7W^+*gkgI zeQc*Fr4TDUo%u|7Un87nqEk*B> zpk-0kzn0osh|TP_SPxO_N~z*!<~<&c%A$!c@uqLkqKGv1yZ~KhoZF!bvpiBKJ&V@g z>gy3F>hwc?|Bf*NeL9}^L(-KS`>V!EiBM|rH;j3c`0qUsh~wQh{q^4UW1m4xBTEUVk*>Lq^g^0+wS~JH!uclB!L|;^J9WMV59>wVgW$K!z z@gF41$e3u^UcA>!TL!TdF0wkzb6!eS2pFb?IYALf%mds$9Hla{>`OGI2xugO<|%2wBISgE)sctpMBDmhzC$k;l)XKRahhnb z3|Mh~glj&5(S@?ffZI`5N6p@h^{zF=tl-0wyO+mmeO`@W!Pn=h@Ug#CF#f3Nf+HWS z6WkiKQvwkbSi5ca*q}y#;1o#E@^KCgW*A-Ouj}jjJG-*cbC1=B6Yg&;3?M%nCePx$ z(ZxZ;rw`)IFf8EG5%b-4HTB(}Z6;d@W&v~?9Bj7HfbTrt)eIh55i4=rVk85N`AtHyLS&LDvkVBp4e3j+i|<*Iy^5$b(*vO|*i9#^Jz`$Mb&G3f$Lb6G(Fyxv&6Mes)Sg@KKY0 z(FA$#NHzxgwuVB&18d7o=(}J|s8`-wZazptvkuF1z9xpLE`#|cxu-9#gIp(eeDteV zG=Hv9=E6CXgooKZJAdnzJ7l!r8XAj<%Xe!%<;`E*YK4dv!_^r!U^kx8Ja69zWt2nU z#+JlHizLVy#id%G?l6hKx{ysD7O$siys5QDv?@e9Buj%xl>1_eK%YA)LuTHe+WF}Eg-%M3!X8-FyiEg z`3=#*!zoQn`@ikYLqSGX=6sEhbB6(^P{BuVDXnt}SqnMAVVn ziMl9w%iPpb-dsTAjR`Qk$4bW@06QK^wD;TAB#CezWGov6CGb#8X;06891Tz^^NjN5 z)Ex{l7+hi7zqpa)ZG*LFL{1lk(m%h;3NVd~`qOv50VyDVRP~0$UxU;@HNtb8r&t$R zcl)3*NDUW&R{oQLxtnIfX+5!5QIugQ0g(Fb>e=~ujnVrxBg-_<5sSUGl?<~|(2X;E z99I0Gm@xL7jU|0l=guoIc^`o}?XM*WBO#P}7+vly{3$nNf=`w@;)+#N9H{7_t0_n< zxEBKm7Xp|k9W}f7QDgvc#XDg5U9E2hwXRP02#3Dt!Lg%@2LBzXx`;LMB(DC5^bAu= zc=%mxhrQ-~F@M8%O$UdVc!YhGN#Fs)UcXf}e1Q=#z3C|IcUqzjyd?kFgsBBE4nFknAb}vd=eJ86 z5vcLp1?i~Pw~N(s(@!M69w=b|uU5$bM1TmN3C6SO&bvi6J# zy%>#STCP8}A6vLZDU8V+Mcw_ps6F0`8!6O+*T(3xwEY23ja}+E6e?KjfXY+}{N;p& z5p^`c4>>)Jb5s8QGMVbQ6gT)J3NJBDl3{bl`A$ly-fi%iry6qGN~_ zHFZmvB4?yc(gTD!swE&KYer!)FK>J^ZEZgSyC96=(loF=ZcQ0JXS%^OUmnS^iEHzr zr-3~p0KxU(Qnjasc%^G&y3dh=e!XmM;k}8gCno91QHX4GS{)D9HBs~OTyu11F4Pt& z#!dj!*vi@DgG+Tei_?QsI%L=Ih_&1 z+k@V*>owELoNgm-UJ!i5j6A#OhA1F1oyU_UTG?_C_Pi4?- zrL3IpU*XMavl`1KM&l5i_F{~sb&%xzMEZ_aD>HP9XYerTXXMR*y{qnIYmU1Z>2!($ z^#^(OJ3A@O6ZL6gMfemn|dIV){2kUm3*?I;BKjzl1luNq(uAc^`Z;tNmYsM@h) zv^Si4&57hQ_WI``l&;C#`Og|CYN6WiERwghB=>-3q>YkG8z#`>&vl}Hk*+A4499OX z5>nmiI$9ASljzHExZ0YWR<#fX+N4(?ezkSa15nCzr~5o|I;P&Mi4zQ1?N=uy4}or-|f-UAnr+=Wz)5it1V)=Zpwu z?t(M;ie^AHF-S({=@lghBLyXK%Y!FFQ^{rZqUQ$i&T5r0$xf5k$)B}IsY{3;Y$-&@ z0gt192m98^eXqDi*_FtqdqnII##X2#Mu#_iDW~r&20!vqC^Hn8QBW^~wQ)*jV9w;h z?dCu-3&7-WlI@XF0GdU@Tss$RklIpJE^J7hXnE<9>g;B>fe${A9%{p21Rixzp7{!_ zCLZvQy#9bNQwN^5QjEqOA>C55&XN?2r-C==Ta|;p9d_$ z<{`dc7pNW&3v=taiH9VijmSp};tWRfwf~0~_;{GOk@1#5OA`s9Y|p z^PKkrkQs=;EmN*?v-CrP-hX9rNc>H&XZQhO^L9TTDK~#E{PDDfRqYG(a+gFn|9b(lil#E;*ogY;NMH2u_ z@8dPmF*o&uut986C!uecGNQ14(|3`fTS%T2w*4`kLBE)Dv%fe&G{HWl$yhKNp{F## zee<%m*ur&oDE9y+46!eT)ffrG%8h**aeH zdgiQgk5N_T#*kw}2xARlb(}zv)gWZa`K~wjJ&o65JndTkgqZ{ReJ}@mu0Y1TJB7>& zO(_yXq1Q8*jOtx~+XgN<{?ZusyC{F!%dla6F~MB|nELwQwOiSovsC+azg!b^DTiyD|6A3I zhdOqM@y)?Oy~zrSzU{*e1KZN5YhIW}@vx1OQwLWmYfUAAMg1L47{>v@LAWG_b^G^I z%DV-*k6#Vqru)8GJqkATLVuwXTb7vDUoF~xv6Cd~4O;30kHexhMU;np>rDFoV4Wz` zB)V~x>sI%$eH9IvcX1L$I#DK*G)IU^K4o?Vez!=DMnIm0mfI*+f=pZw_M~dh`c`Fg ztT*~@nlI5120H?&m;@8#vUBMi4*_%Z&m}9CT8nwH$S7PcSgVRfq4v)Q5%z4iF)pjM z8Zgq@q3F4Mj_l0qN4;hWRA{sd4fElsYDC*8eezt;Y3hF^W_CCxqB7H=tEE2iHQ-;e z{P#Z~&fCx|5{}B@`FBq4hiiF-HE4H)b0~+K%Y^sKcS0P?ff&{ICx< z-Lz{CS+wo8R4@2#dqVf1c0>@^mAQ1p1Dmp06?n(v(g2y^@}_r#_i#ob;T*c#cG-NB zK#(9fMovWah2S0{kUD`*juG8|q9t)*D8o$Svcz%{>dc73KQh$x2SvCq&qA+VwL^si=- zrdJ!@NwGsWhXH=i265(rilkK;-vjX-uD-oF86O@#@SHrYY{6YGtkWjp#40qf&=xm12C`{ORLH zM%T7wlhtj+k-J!lPFpj0W5n&4 z*>2AP*TS1(P+~3y%#Y3;V=8olKz>$iDkLD~xbW)DJ+Dq?8boXdx#ENLV=;8;&T|t@6F0 zxBvq|fjGlZfGqlW)xlK;8+U7o(<^De^S!wP1NIbpJj9Hbp0$ z-*a;r{ojoZ>S3xV`lGG9(R`!+i7XE&vP_|0!_Q8LBk%-q)4=Np1q^BWhtX915&g;t zTcvZ;se&x^N}>8#5$a^{xP}NNq2@k!2&1>TE@G3%HrWMUy_D~$xNzRF?O6b*?0Hdv z6cbl6=6)f~p$wqid^j8WdCXuZ3_Sr$s0xQ=i5-c~&D`7p338g@5!0Lg`6IE94hN66-87=N;%IJuN zg%HfJhWf4=exbd!zpHwpS8*IVf#w}BO}Rjwe;uzdYZ-of(xu|eX92ypfj&*4pP+)? z$pobxv;s96=Y$o`IaZMj$qRRp1shdnSsG5NN8YLi&hDZJ|Aj!l#E=$KYCzVnOwupM zCAK)ZC1Zo^q=0RJ^T^vq5^h6?#rMOnyhgvE^Whe61=~^hdb#;@WFs_l{C%D~G&)LC zm_NRU@p~yV(Chtax5qC)eWdH&y1EzsW2Abo|J#nX%Iz}3_E9CpVE!#(=Gj9rn!j3^)M^__x8=BzXTk*qblB*#7%bmaa*-90`E;hNz$}B?zbM z-=|2T%o6ChK!PnCW{3tNJHyCJFkZ;4J2ZAZcWTdCpBIaTQzj)y&C8<^9BMK9xD)+B zC`9rAezjpS;rCtIVVX8sSgb6#IYYe;^LFE$=qtubBSxC<^zrL2E@8e8$f55EFs~J# z3Gfu>CyLDU1R+~$49uyPke0wMwd^x+ZJG5jL>B@p&^?9x429IlOmP3v-ChUTa-((Qh!1598mEMowQo3E zjaIx3JB<^~ieVlp7A)9I`!TW&_w!3=CWd&D-kh&l`dtt29i5kTmv8E==eX(Jc!#M0 z)Re4{h^ck|uv-Jf@_G#CM^yA3e7-kK8M$dm>zslbisHANdb~w`&?v(wwPR_1I=Kd%-N4xt3OuV&4VV^31Bgq+6KQxQL7R0>SNJMxg4tPMU zg=-{=d}Mwh#j^vN1pWa~F>++9 z)KJ^OU_$L01DJC)7=z1r6zpMa42!v(D2z|-u=w)X!YYzlMMgaoIAV{XV3t>hp1{fMUOk|&}_jl!lIWDIgI+61JmlX_IT^A-i?-K(W<9=c5k6Zi?rmkx)2SN(>t|d*)W2Em0{tw>+LFN_u^>nwX*mpZT8TzWkhL zegDUDvHlUNed~hZcA%ePm~|fq_UO#AJ4&@>*;@2#g*`Tr31`ryiiq}&67HAZ7tUTD zBg!~isgQjuFH<5+yCNOWM->R~Krt6f-(xmd+YH%8zgE(bBDUvZyDGVV5Ie)CFQ`0h zNj6!cAnu=CRFCo9A~-MZuJmofx=w1WZK~*Pb-;^i6{4 zg6Cc6+s9gxE?`gNOpNs=L0$!?T$3_B9#RnjRBy3pFQg=)Ql*G+nP(`ThVB@Trf1t_ zWZaF16`ZTJ&0%0}Xnk3eX$ukaVp6eSKjXv|cF=GNaSe%HS;QB?V@ioiRsY{6Gz-ki z2#7|gp^5#8s1jky6hpz8?BUkNsmyKN(ju@wtu`1IiW;DXaq*-=OEC=m3A@p}q*pE7 z-~-GO#6gKM0uu`^PN%}d#S7ek#oLJ$z6{WA8=+OC$frfkF_8#o>V zl7ao{wgU1Pd8^cMKzh1bjR*iohm+K3^n~=jEa?X4|L*?A>K`wWH^$7~=tP&(es}Wx z!frOWA8K@+l&01Pui^PA^~oha|K}TxccKJJvuo+aD09(vqF;jIepf~LOE2?fu6w^ zJF+ZO9=PFQNs0x+7nb(FHE(?TtBqqLb+cVclh->Do@}?c>NYwsH zIaKsBM1fEwN1*~h8oDd3`ly$O#9Jbm+gsYkkAx_Bn+D*sbUE!g*HAW84|D`z?gWCz zKiSXL;r0CN;`Ds#<=l8O$3KbR$0f88GiT4BNFgV>tfAnXjzi_NmpU(7qkI=mE(6su zc{d@2>gOuijv z-s-}~SdBT(neF>Mjnf~59b9W73S1$s2xuHeGv10Kf=WYd8G@#?Xc6KjRFK|<-;%{B z@PcET#)o!Nlbee@XY8T!Ct-Njz#@IGVP1BSwt2%{LSg3k$4T{mSl5f-|33Z}S+F9q zl%kE)P2O$tz#F)H=J!ufqZ@>#=Y?0(bLW%QWD6RIlIAMR>ae8LbsxOHnU(uY8F_F_ zY$BbGr~vT1HHZGuV*kp2&;Ri9@W_IpUCnVRznrB9l z5&8VQC#8x5r#uRk;*RD`cq7aeTX6W=HROq2l{uII^N%Sv>2Dok_%|BMqMWtlfsuEw z!hT|oYyZFMoraW6*Bb#1aZEP?Aw;%jb7+&D*5|fr>E^hK)2pmg7Y0FbEi`!RGGgmH zpVt5;Of`Xu*<oi1{k-e%FLPe?FB8BHEk+*b9P$K~A{dxJd!yg7DJ)aD9634#Huo?Pa(o?jKjBtq%FO1nNAOyDElH7 z_7FGX5GPUuq?W_x(WMi#8h+IqtG#MhBXKl2!{@Scc9UuSu3MwDnOzNp15&3Pmdt)w ztz%Wl8SnPI60GtMNdcyO=i!s!;5a`v832oZaw@chHy$?j;3zphmo(HW_K!cqbMpe{ z`7^)avh{@bmq=KQ&3e$Y=ZnM3g__Mk8TQg%BYhXl@t-Y<2T8bP-RHUE{rlz3z|&PX z%qm9~wEbb=e=Kg^zz+lUVxazih;BuRG5kJ<{Ec@X7tuuTMfspmRG7o%)T?$+hNbEa(70e+{;9nVCV?wi)n<{2xhUqBRu8gEUgnlb zeYkDRgk9x@>FGSPh#(^;nlTv-fDFe9#QpD2m^xH2aEDObZ#e*>fZbqmsYIIST2-pp$Vzfi7vcc6 zcJvTvnT6VL2v-B6aY{-lev#)eOlf$5*>{Wuxg?@_b`KrBWc6knT<4!g+tpPKoH6e^ zkL|!=m#d*mcs||FxCR9TgY_z}l_rlIeS2-^EwqlipcMW2^rVoYe!IJH%1pO4chCLg zcDn_%!Q}yLI^H7JmveX=FN2>#xT4Y50)$(T_J39J4{MT(gqmw5_!Hq#9eFH!y{WO| z|J!$#l)|PPZT2bz;qe#|6*Ipu%HEu_<_U*n+5`9~WQYVq$d1zeRXq?gwuaa%rD9EC z2^g;nP|^nuG^hw-r=NMa+BtdulI_sW-etkzG3psD1ynD4?9E%O5@?+kx@9W7ZWnr@wQ638(CMoT=;5(h8gJi3ezFXbMbjvICZ)7+JEj7~XJPvsePt!t{_8gFpRs99b0ocgK4PwW z;lI~%^w+m6=MWXT6cu?Hm?Px8PTjR#?sYiaCha}fQPFpcQnF%9jcTIV7Z3jd+Kj6< ze6Q@_w=xZXj@N4d@4f#n;}syZp8v4Se^}Q1Fs9mlZJM*=(DH#>2jzex!9f5%AEjBA z`tyJ!f~An^9pJlPd@9CujNB|#wIIpi65;7DGz}>TW|sEwNk);`aD=J!hv%a5lAojq z7Wf2LNxANxYqPBE?V9LvA*2Hwtxc(-9NGf9J*oN!UH1I=+M@FruwWcI9D6Mdl88xg zXkA>#Q;K}{Cr}cs(qA@c9Wfn#6=)*liD|;M52Bre2n9ZG_MdV5(8Xd4SYjeVdDL=3 z-Rpoylz)i;QS;}*gR0kMa+viUQC6Js$E;ldIsJ&SR%ZI5ysK-j*dT>C)Owhw_I9wd zgpI$o@~11c(e8}KE0z`W`~k#&2|b8rE5I40S6?Bj$5Z% z^5D>{5dKi9stvtLz11>yA5c9JYS9Fz)2UIC6A96!OOk*vUaxKTJHQM4&W~{I zRDy}x-Y`!-%qUZ!CKpsmx>d&$m5S=-$}Kx)5?c&o;8_@MTlw*~Im7upMOl2j6cN8v zjThT)r5mt>QWgQgO-h8x5UqmBXd~l2TIj1wvKES-M}+39Z>PSOAXAh6rGS$8^-KS7 z40WZ;3mnd5zu;#UM$QM`$4&JYBo}m%N0Em|`d-fk58TE(!jO67`zSKoJ)*>f&Ocz=OhMV3Iao&`BIiA0O4TIgi5l`^%Afpeqcne?Wv#oXqH z{T`2ruy7MhFzDlJK_3)-6!SATBbY3RxE_Ue8X_ca0Nj=YS2MtlicNIF0EuvLHFDyI zA^Qm9cgx(b9@`IheS06RpPVBD9u;2{fW37Rr7|sg2tDfZx|y{T_eP zdtsj^fj5Cp%>&KTP*yLHX6A`!$b}Hm(IDiP3oXh@yF0~jc@o&Yb9(}v`hIb4@!4lC zz}VL9o;WY%N?pLvMf895^rDB++OkErRSD+GhSlZdoglub7`nPewm^UEHiF;gxIEwV ztI_sWG)3=3lETX0h^91%p?pYK#d2i|Ifu<5-HP&4FGqW*v?%pGVNOR|t^Ga1l%uU| zH{Tz6TxoZ~cQ0&pfY9nTCH39~-wywNJibPEFpQP#RVoK4nxM7aw|?GBZpZIQN0qi4 zbu-(et!{xDFJAEar9u4bCV!m1hu`bLLA}t*d>e9S8m-_T8(w95(S-Sd_IsNDz>7zf zMt4LVJ3U|gJ@7ZF5IK#O5dDobfS?y608l}V2VE5ilH(9qqaoc?FWwhxR0duhtolxd zc5U3|TsF!C7X5)0J#Qn7OiF^7W}Szc33U z;D!(N7?t3g_cP7cQN_FM_)i531QT^g5dv-O7TF>0K~D7WvwDJL zMc{%6R;oOllR~wz`L)@7KNswG^}XM57iH)DeLfg$2Rj)Y#Msz;kAGZZEv_6;Y2f-< zB%P>gzFazvWMEv8k*lL#WX@NQ;Kw*^{mU!*ubs2`V1`v$5ko>lkI(_(JvI{!32peH z4LRkP#EjVkTQZpWUPcEQW{!@nVz(*`^Z`1H#x?ckFXIiL?OjPlZMRuY*YjrHaCpxn zYCH#1zqLlJJ?$8q?I3afS5VJeZ1(+d=%&5F6ur<%Y&oPnI%E3QgN;ETHuF^PQoFfV zpIM)~qk?E_$R81rMcYw>K(pd7`=1l+FhQX#RJjK|EMH~5)~I#FF%J6^-Li{psmbcD z8~av|7dOq4*U0t*P){2ZWH^30z*sD2-M2iMWQ~S5N_!zh9%^4QTmIIX-dyNc9pW;S zdF*wn{Yy+CaVtV#E;jP#l^`G1EO)~dZe~;kryab@oLuX$q3|eR!GTB7J4Uc2)NMYtM#J z^{6*tGNZ!rhCM@QR1n6wq8Nb!zbN9=N(W#`rB=}DYqV_(PZ50B=qLrK7)IUkQYF%7 zk#4syAXgF$1zUg^FSQ_N7uEu@^-NVa+%i)mLcjPcbv(<^mvWt77Bmk z=4~suxa^+U*}n_VNg^)ow-sG!@@m~m84O(+`&_2ldbZE>RIZs(5Z^fc<*m<`5%O>7oR=~7T9f@z z*Wuq)Ll)<9eLCZ9PxxBvB*h~*PF`< z$dg|NC-H<4!&iVeKy^#AqMoo1TNkK9=w;XjQ>Hujb|$MrLY7|urm4@Dj@c7KB+^yn zxtz^Jx08UATR7U(`T(wvu#g4O)g^e<=kI~)chCcAIpDW?I=KEv5Sc@#WaK5iVtugi zP&oQRF{q-j$#;tGr|O>Qc0`3zf89$pqhtCht{s2jQaArxvQL+-NSf&x!u~uB~s3PH6Zx1E}S>}qzTE;WC`;E0Fk^|lf z83}N_#W1o_2V1Ki;_&u_zvtzp;^8>wkiex?MtUaOnLZX_nO`#g;DIBmMuGSS+W1yh41UseGsO^FT52R_U> z0aMmmGD;ny>`M|V|0Ee=jQ4jI=`YetJ3UTM$yL@MeTOmSmh%)xaj;PwSIU4IGO=cP zMNbbLm_05Q!wx{veCgZoQOc=k+$?fQC9Aatz~)WoVFN(a0h_l&{BOFCp6?Ifi|j0P zt4Pq{t>y;5(piD(oZY4W7z1x)ZJa@`jRp;MK8Ila<2c|71lk40ioq8y4BADjsd!o} z&ks?UW?p)w(nZ^}BCYxG2W8n*@u?@rw5cltQ@|uC^JU zW{m{%hJt6Ar61qY53$%)?98!st%RQ5rv4mI)8K5$?X#6AlpkJ_juYI(TQxi2AQe^$ zed2@ge5*#l-2iI{DMHFKHNZ#|I4EnN|J&EvN)t4P1?z04D|0&8aRY5$wCF{Q$8`FjkJ8W*}?*bj(q4@9Fm z!bCcG=Rt`r{L-yTbtQo9);8%3f2Bh*`>kR2K+H9T1a7ypd(ER zClMf;Bs>tftwd=D-zKA<`^$~|>xdbAHZx{6TV|Gcm|bZ{F_9<+Yh*?Mt!U3600{NM zrk_pNCZ2mI+z^@H( zvvNv*&B{VgPDWN^czD-O7cd~=^GMz`eujykCs{*}MgwOE7PoS;;fWq$<_K7*Hgrzk z>52#qi$;SH4Kw`jn2rBCbxzYDyWGE3!Du6n*;7gCSrdWzf^e_MoSbjR@^DT zR24alu8SIw#HsH_1(W|M4B~vbFUWiycxh+RcF+N=34*1dDqv(ARM?qTJQ48~FLUhK zzOu?M??gRseB0?pq7RT-D8X6W(-W!}Q)MCTWBHN*h730?-rV757B>YT@td)^o%kuK z{Nvy^x-E@PnjpOtBc0#0r4)7Z5A7=rHl(t8K1`-&$BzEkqboay_ilZvD~}&+Kg-~W zp}fwY-&3=)L`&uo0YcWno4y0jt3#qW_uGE8Wp()GlhDGB4de+109p5u674ME@_uX# zKkvM6wPVNd%nZ+{`+MW$Ef*OP9T1en3(1|xvHIx*rXwN?a$RAiG70bbTF`OrAT+V$ zY7}S`1c?ncnhVW3=G>eQ>!S1F5f1A;W_DSpu7~-s);lcv0Zmn1&|I_?b79{kPl6)u zk=cq;=4zwF&1|2@J|i?r!8H4Je@^InetTs1m|bTerkT%CfbB|>DOwApCW8i5n0hSE zV=GjaO3hO(OH5vJMc3MFy8rk;I91l1Cb-@wwACCy%115g67N_SoYuok=c+%&p!WZl z5*W+^Ri++Xn82qyr9Ll9Q!&m58}+r%XV+KZ4BV74Jv#v5rurCUVvnBA8T)y=Px)Q;p``&JUE#k*4wzw_syDgo1FZp*-{mF|@#V-&i-Z-vKAt=YDnI4eJ64 z4J^thlG$OlNUua&+>oJj$JraI@=F0SOnjQOZdFV}A-Ay#2S6ewRwtWEVnd^V{3*8_y`+ed$2y&afEj6^{ z0P_mYP|ivf75BeirS`6GwFCQn9|mCj0k{AS$I#Ey zEW5LP;e<-Gb*V}=D)!yqflsw4&c85&t9-q$s#*d*#{O9f0tibo$3tjKDOTK@FN?p~ z5t2vBSk>+TcKz~Be!I**K@?-ufStIe0Zivaj>9nHv<+^J?>cFJBeoqwoah#v=L}sT z$5n^2SM9TUZ+G}=hF3gtYFESIZRF5@_&dgh^~duzMf*vA^k%dD?=)Epk_?iT5(7Z7 zc1roo))&n?h`GQiV+ zg%<6qLa|6H=Ybd^VNDuSsR1!zg#pmbC~C;UT>;p<(&Wp1BJPo23$UBB>H zQfDPV@cc7*$ae@)1T|f1pF|+sNqyy8gAZsw_yfQ2rPM$#)X4Enb-mj(Nw+$;yKY3g+My{qC2{Xr+qdxMs3E~xK9=9oXA*U?#tIX<)ZZfloQDOXL$k(txPTqB>>a zqsqXZE=O*OuUDhDU&r_+o~0YGYe6P(G$6g<6}3<#$ga}d`EFwIP-RIiy8$35ZU%-nac#685KMY>C4)F~@q=ApY|K#I1ksb?1&>M{&?i z=0|~9p(Xav>@b=l2YFarGCekGQ~A0rOAp>TK+{qweUaHlgHt*h`lbdXO+zX>B2X}E zgU0(JvHMivlpSm@9X7i3cx*k~qB;G?!5Gg2`(-Ghdq5^zpy|Y+TBP~h{E;X9-BBm= z*Mg46PN7dca1Uk&t(3Ht^3gJh`Nq+u+)?J+Y$EX6P#ZUlBKY6kq20Ie68D$`!uyK! zq9SeQKHl#vD+kekE2Q&VAjT+JC~bdh7WM)fjV;gRXTN2}uPW*UIl~<$dZ9rXx-{g8 zH^Cf|l*>?Plmr|iBNI(lK%5gaMM#BM0CMv_#W%^B7=LyFvLGeZ)h(az?mRT;TTD<3 z1gLX9cu2uu50}dTT~?;^-P4ijgmlmZ4FP)*%2X_d+TBm#bv+k=-*D%o4!J52U{TFv zU5|T}(GtM~2gRRCkfKWU9#aV8O-cjH(2eX&+JQzc{v}HP#bp(tbN$G@lmTs9S10qR z!!Q+Gbz@_eD|_koxmr~VfLTB?)L7$!vMQ+gE*AVcqax}RH~owdbBHm1e0zTF2Gade zi@Uf1V;*#pV@8LemH46E=;E)bbcs}9!e_*aJ5z%MGgC%b zhx6QwKA(FWo`m4qr`?>(2}SNPJ>zkT4ML%CS64`RNBNCs1(9cOn8Oz=eM4M^CYYZ{ zX)cFbxo{j{mW944dCfPYzB~I~enctjPgD|z^zYVj+9l;*x7<6q^sfWz9jxEB>GHS+ zuuIC~{h)aVRGh$)4|Vl+Ee~-Q^`{beIc!d4ZzJ ztESE2?yXYxhOOXph(11U=(jTm2}m(eLP(QEM$iVuC=*t(V$wfG2Tr_wXmrm2pMOh} zV&R_T?9OgxkyDn{2W1n}2=-1lVv@ZL!QYJ1d3X32Vy&n#umJw#n?!^j8WCZ;%93v$ zV@mG$y6@>;vzhVAyRvR8?r4BuOqQYTt-9UhaJ?abTb5m~0o#5LPT8&gzEr9MoTk}~ zBF)nM{spk*@$c+c9EP*)2uy4Ahd2Ind$shfXzJpaO(vyH%7YkM<9+Rg9u2TG3HGDw zH<@<)v^nF~&>OFkbgLGQn~vj3Qg$I9dguf&kq#E((;M!pPVO|%g(?T2O4OLpapVhz zVr(-BlbMz(Th^j&pQ`}b&*pvHcfm)B$V3(dcsjI;*Df!C@O(fxl$Ag_mAe->pS4Cp?ZvRfV{BFqY;QNw&Lo z>AIoXwC!uP>~4j?Vb{60(TKka|J>+L+6`?V+CNI1n+;7!;UB?E8&zfm`0K*vF$L?w zWeLjhI+eZG4Ne&P?PiL2uPBsBgi=7LDpXJzQ_3Au${kV49aG94Q!1P@N?mk`E0V=2 zq9N{&1>y>ED&tu^eFiUf_UoXqKONo~W#SI=55K|I+_6AAB zr~)ynVmZ(qUk4V3Ry++3x~5>FjUoaMR56svSQR@9YYB+iDMqS;^oiNU7T$U&W-pIy z3bUh=6Is9`9<_HY2=Btx8IJYOm{91>(7}~4FQc5SW-Cwvu{EGPgmbVn*z4=Bfw&I_ ze(o6H_veE7OQxov?^KQaS((}VpP=*whO*pO)a%Ys;*mp`Da?c;)5j9})1EkA4f?8# z|B!j0Zb+b^US4@#z=u>MDUX`2Ift|!g z=Z8o5pQ>ZMVe0zkTuUs(b$r%ihWTHRK;k46>G?5ecRXU<+x-XODEM7h1l~v6&O5ta zX(OpZr*5p+5~TZ>+K+<#w+Nm6y^$xGQrE!%JX~zihLJHbMwXU$2;gcNpZq=t>x`>5 zlDRKwCg{;w_J_s$I${5UiqKCNjXnQlH>at}kl#7W8g3`5zN%0Zv;i*P(e;R^`mgWi z3!pIcn)|v+B2(;Bu?KFViQGs(LrF5}=mqN9}*RdK6JyRCPcA;~c<&5n@pK4t1ODpP=QOn-~Z2!3P>@aRW#z0g`~AS{jrJKlt+3O})`2C$n|SPr6#6E2kYo%BRYTz$#EP z^OIIXKuW_VgoOO%Q$wp)rc95cvcA?r8L}ub%aUd(=8J6I`XCH;$;L&Cy15T?_I2~P z5o#lc6;pKv$!~&?Y(LXTC&6yk1PqVkpHpn~xCPJQ44utG*=!JC6sO8((Sw}1Ion&F z_OuVDI1XelVvl!hntBdfZt6 z`^%V@hrkAre%trF*j*mc2e<%p8BnYCGEJ2k-2*+xar)#CS&cCi(#06k(g^MIl#Hs* zn66%i?4Tu2JGcD^I;yViWV$LW)jm`sw^93Kz>!B7iF4_UU7=Qu2GrvC3#Te725$e@ z=fwIY==(OVmM6TLZEKt)9|&8rgH6XtT&x}Y=@iX zpq-}$@dQm2S{*G8hY(eEQSkKlI5;}wX=BTs?JWxgaq9;`wO^{Doy{(Hv_i_^?wB4A zLbx0OS+AgJ&CZMK_A?uw;`^lSDV_?WVj1slda7M@rl z$olIoUMb}Nom!at)TpO#r?AF=hE`)*!OD?__QUD;V?3-P!tbW?I9%Q!);Pc5newhx z;~ya3=uD9}4qBW#+EZA|q&B=urDe*`7^PCQs3Yoh=~x`ceNgF*bv}nRP*6au-sVW@e6bQnTpKaX2VCvFJNUAa z^2c>3G%S2TkkOmr$C}Un&35CA{Zn6w{vQElKHVAqar*Kth95B+7{OQj2)D|+hH+!p zFO~R~7iNIGmA_`JVH&V0xa<4EM4x@VP!KRhkPMw?&WEb0c4n+S)HX%%&^O;r z0*Q%@&FOie>uu_qqJXf8EzWJ`xH9QJM%;RDD798br@y7BGzbCF>vmd2WzjAn-UBl8 zYCViKWrcR_g?|>;PfM7NkK90rWeAVl1C>D-#G8aKK2n(Wh+5PY+4}q^jzgcWqsfnq)n8 zWaCaH=5-2rNtj_EeR-VV>N$mvX(gP~PJpl#ZlI?8$@l|Flc&E#&|mfctcmZwwU(Zd zqDYIA>g3r`rFx&%g@&V0U)Oz}(ri6j;gvi?O7d^4wO^FFSZ~%de&%aRdQzG??p=T5FqKKscV`0vK2^CqQ&V zZ6EpRW2yc?6xsfApqS>tHmQ!VpXSq#ZU7&#um-Zjme3Rsfin}Xv^OdyexiQ97U$%H z>v;GTy7fv;LGJ^6w+oq@L%3uGdx=+L6^1liqi0pxevYZ3Co|Y|?^xuyoAMW+Jz@cB zgwf!e$L=rXnyI$}{r(yDDnioT!Vo7#N$PL<(&Y5o+E^scn`p-%IUq!E>0{N%{I3aJ zQ@YX{G~B-YEH~h-QstVx!wKf3Pmv;V)M1P737mp1tQncCm=&%-i9$-*Emx*wxnKHK z^P(~aQkvfdL&j2?n=a+wgKigWT*&|eW8q9J3#O;$Zgt9VTxHqL4_X%`9R$TEe?s`0HiO%%{*#4jVF)Qx01;b>0O{@~TE5KP z9+UoCQwRT}d|??1o%P5!whR6CefDMs;b#|TI5F!Oz(WQ9mkts31heEuO^(?gzAdN! zY3pfO=*S=Ba*}4ZpW&!7|q!y1_3PJop*s3ND{*PZmDifj*n?Oke{$s+>atb;7{TqqdUAXsebCYEIKKvsZ_$9Ix5re#xaW zw(0z)RXx3is`n=;Yk5(tGerwd)X_y5(#kFvLw_c0t$59-bJ5EzXzg#D4_(ILfQiY9 z?)`KzO_;(1Gb6h!w>56ET$I6j^rktBgpVj^aCo^vzv2dCTrR+8^6Eo27K>0=vHtHH zhD0ofp*~y;Bz=-%FbkiDF^uo$Hr|(dxyzmx+pt$2A&r5NFax2GSa6yu$W6I3K7B=m zYw8_-pT+=Itm}NV(UEm5sKvassV^G=m{30NOVr`S0nCPlToP-$9@Kg zrm_3b+-2hFYZdmb5QkK7uGxJvLax;eD%NqRFroFO@PT z0!ad`6+zTC)%2xfW4|d|->0rIqe26^&ogu;sppSH8kCc12bsV84H9043NRQ*{)-@Q zNU=SU+>{iEYR}trG3^Pp4U2J17v7{9IMW3)e^fH@`^yFeemk;dGuZ+`RPXn+8g~y* zW=|h(G6)@Ri69*px=O?OCQWWpod8W5a^}@#9|g?MdT+MW08`ADT;D8%q~nya4wJ)r zph$Q#!rb!T{bNj`76?P|lQ)s#MYiB=#g6gxzYgM7cC^wBH?PEI?)Nxj*G8uA5^1dnW3rph+j{^8XrgGaQdZjx#)4h zFqhK1~Bp$>ZPd|B(d>5Mn%>g<;!8o0-eJ}CwO2y6G zzEF0&ZqHW{CG6ofw3eYw>uVx!-j%p)*Za2qF;#5_jXa*}+3>|9Z0{}}Cei?|c)Cy_ z(X8yO1ScL-b2xtzE#pEIQZ@P_q&Yd_Y|H_kHgFndx*cy>obTx+E}c1w@#VM48n8M6PNe9#k{)%>H;;n&iRUhuIILq zciw|fzB;^4`-4v)H?bW+J;^130Cz}!0dHqv=7$JIVi@bSOXBH6qlytUp56SsgH$q; zV2<-#bsxN&qa0zuQ%Lv=z&X>$jtwP5zBF;}<(~Mk1PR|lz(Pm{=8(ICdBn~~GllF1 zOwnfY%r2>RmcQF&-8^N%v==TV0p+;6*bZ1xwt1`GC&O5Ffq()VPPVO`#dJv_WN_v! zeCD6T(fN}Zl`tDVjsBrY_2ID4l~ebeLmT!vS^8RMK{ybD)5_Xv<0V6Ocj>5Ljsyh4 zz*dYw6)STpjkKG3A%J!;@-5~56iK9YDw+nc=Uwm?UvbsGP|3Y&vm3z0=`I1Q zvW&Qnd{SE876uYPiB5zKPn_NK z;0R3CG+4FNI{b(QZPK)+(Wnyva1yt)NZ&#Cx#0Hef-~UhLK9bD+P?1oa9#ds@E)ua zG)VYSj#~cLAKPMC!oC!zsURNVxZI9eU>j@@Ae z*(jiQQ-Cpd!!r>W7!|&-+tZ(yrQuG7UbJyJkHYX!_n9(o3JZX<)IldvI)%`EUM>$u zBbw=4@+r!0WnBzBtbo9=6D(vK#8$ET+E! zeLiz=B+o^W$d^;_B~Y3Z^Da%)PL357&t@f!k2Ac6M>>t`8=*juHs+UcL^h$}GSN}` zmYz5ovK8}T2r#5g{$N}jCwmW|z-*zTB80>bUZsnDUBJw)7q`|DB2OqOxcaKItBs=& zRD1lr0Th0wSi5iJ?JuwJmqLH!qU}2po5M@}geDZQ;Y{DkOuadAcUi>x6OcASrGX@* z0Zp5X^f%Q5nbnF{+B*Dbr8CsGqZ#l}DIjl1=rdZdYL~|1U}q|!xXExnDyC@P%K=Yt z6+hUi#{askWm!l zE4(OagevjZ)KUTp!^Jw3`uEya94AkU=yt5F|UH z354Cil;z@JMLkkzegRvf1BAf2DRc__nFKj`lCRQ&@3?4J_;|l2CJhM)+Ih(`XZAJ+ zAI82GOeb*$f3DaHgOARC;D(Dh?Jk!juW{EkGxijZ3yD}jJ6qh6a61WK<6A-kYvjcF z=z17SdZSwscUJUszh&7oii)8(%`#{mP;a+^U8+(KwyBO`ksN$AIy6sDY;`Zj(ho+h zB(a4;QD?;Fs6feI%T25X2LCD&pK5y+eOHUwAwagt_ zl$fFdcHk22+f+d|SV{pCIdBP9+H2B>0iu!UE)x1CI z1rGmRD6aq4w7|S5D!?sy@<=23;FD)+aGPh=Jr0~lC=K0hRvICS#vSwrH`<`({&N}W z{wr>bA9m!%ZG=u@R5dj?g6Y!*itT`R=_Ph@3ynCb`*o12dk~zxWKl^vg8Q7fKo+;S z+>QXV>xf>ys&}WAczCySPOVd8!DxHbm3>ikgN39&|MZBi?R!|N{c1%mJL z)qtnlaaNkdAp?&h%kokwTmK(T?-W>B({}B~wr$%T+v?c1ZQJhHwv&!++v(W0^{?ms z_CA{HeATR3b&oNw;YAFJQvpYhyhS_&8v;aaN`>?h{jG4a0u*3iI@D<6bnk`jaHiU4 zt1tf6_Y0}j;z_60bJA|t_3t#B4aVq!+sBHg?1{*vKo-|Qkop2Oo~*PgL(|tWSGV6^ znx2A#SJfO^JJpkVkeqt`WF_IFkd^T|ffB@RdXwdEj6G(t5cJ7Z!?Phyudg1Lx#6Rc z=Mya?h~_`~Y@lUS&a!fG2LNNQ-&gz(!jTVV?|qX1@m~&6oRy74_Tfc0RBY8`i;?7R z;(;AMTY>L`ELN;ct9*e?RGTE7>Aw?yP4N7lhLID?x$5jtn6SUtZDPRniwN zZ=ESc21FKsrPYic1$P-{-_ucbtz_ugb4Ao$r|HQ+&!XWT>A@Kif z@8*2EqOZo06SG?lG0j+_Lh<+}$x>t~X&ySWSyZ<^A-x9`yda z=aE=(V6uW={?HJ7p4Z`@2@$@(rWO9`IiJ`?hVWj|-Xc9*G}=3db_4mJ-;^{OW=IUu zl#5Oikpwj|>f;yUlh0oTu2n9sy~pM(8Z|J!HJZ7;dff9I3e6Ub9~Ephwl*TF{-}PU zJ_K>ZRTo3lFar^z;@P1AJus{SB|93*zEcbsL6O`Y=wB5d*5!kOY}!FC2oZltSmHGb zrdgzs3!>YcfriJ+fYrOf*Sgg~P)NGOSy@#|F%w%@B&06WAVLWa`*JCfuY6F_x8td7 zi*z~=<~N?GD9cPh!J&i&!`x6*)?k-MO=6{xyI*ao2)aR~=D`;t0=ZTa4P>p*x<>)!wQy*zMd3|ciKEg@|9*G}=dBE)uL3B-RL zrxh14fZ8{pA4Ls?1&3DL{-HSvzA8KYG4p%iPcdo(vjJc?;ZP4q*Lu8rQD91BiPVyn zS|xPAbySpe_`xSwMu*m*Bwdewec#^P<`I6;eeW^)VTyjAKC!T6#Y&e#YX-RDnXbjc z?Bvf%#l-KhNAJ9S0yl@FnluYvTT56RD*ndWayMdFMrZ`akD0U2j;K#s&Ebi8>3$%C zOUxZK+*oo5>h*fS={o!5>N?o^ZS=xxs-X6DE(_uQYK2CB)whHMqhkb`rvaOfVsp(| z*uDLo!o7@c&>?5;i=!gN-~Ty2&L;eMu_@4+u8gE5DI%lfbDIk4akql(^HU-tsy(z7 z`Azn!TGK4IrL0mO_icw*Rp6a|UGKW@+`4RKqFk^dvIMu4`#_x4e8oZafzf0t z7c?=?D2?%z22OYOr&X1URRZ_Dkte4IZqx)NF6#=LT)Fo)E86TQ0u&tWD|C*qupNS# zC1XJ8mmo#8>Ff9EBCVMDaw0>(c#b;F?AJ70E9jMuoySHcko6yFk_V~lTOpBAg-<5v z^NE|KA3?pU0xfn3c4_lOTwYF)@Ym$;BZLKtWF-^pg4O37&ggjo20s^JbK9nGan zW=jfx<;0ynPAYDD(B!1B<<1Q!V7=**6TlkiqO z_#>*4+q-thk(1<5;YT=eeh2484sef*_@DH2DX&MBX&eYl&cTCh$fjZxn2KP>M3&U5 zO``gOKs%s;%qy%SW1E7`2ZS3~C8y)z_xH;q_WoS^4|o1t5;{IDV;(fhFrvlj=l-d; zHv^NBTZ?pJnf9n?WlPo6xc-;3w<^$tc+?MfXA)ERrywFTRjLqjS!hthMFXG4Bmfb6 zhP<|pcl39U1zfNH^hDXEdC%dJq0gOvcFg3@OVMdWj4*ao_Pcy;83S#SfSOIdm7m!M zJsA~GJ|3mqWul%_2b^>{+u46<)A>Kk??2rBU;^B{`F)8yAl#+#90 zTnUH0N(?jaWXJG&6_8yl!GhT-*2Qw?GX*s0Py$WJ)8_@n8gWoSdEY)eD{F9>k-$Cv z>aKO`c69spencdO=eIrN3_>VcX z&fQhtKZY|V^CoFFl}I-yrCLWFG~-|_*rAmOCG?rXZTQZ>_P>X`kqR*3sQFP#mb&c` z6{Zdv#}Rz;tGzMH6RpU3%{K7s{G~Getf&t8BTN#U5k&+O@cD@obY#$>VfFX2-A1BP z(i`@Dw@ed9)&apyQS=3Clp&m_NNDi2&B~%Ygkal0>JhnZ*mt*Bg^A>2#uJx<2BOe5 zy|Ayd+JU3jVB62ZeRJt^i}l+LAi)uTiK{@^Hu}UECEj+^kfS?F!-)Ha=`WOYam@QY zapq~O%?{UIsly=LAMw5rI6S5_-3UFaGxMbgtruo!_>hhsJDU^Fn{WNosv%nB{^pU3 zFdx`Xp5SS1Vyx&yzcFidQ4JddN2pokARqZ;`qK0?6+d8h!;POZrPAdn3Bq+EB%?|j z0(Z%iQ*Z=EGV1MW5$k>8jGiXx(lcZX=8*Q=h|4G`r(VxdeF(cb#+!lfFLqw3*KVnn>ZNWST|Ca)3 zK6Ssm;D0AA)U*a-0*aKl;fZNM=PGc}Q^STaMd4CuHj>OwA&q)gl`w>(PuLvz?i|>0 zF_@6u6AW;gm5}kU0U-#Cpnb;3=#DC3qqF=8C$7H!PnbVB{k013Gi8Hx%(VCKKGiw< zx}X$RuaDOx)uHLw*mGy~cv+QvWGEAw7_ktAUz>q1KhX78dSuG9p(1G+uAjm{)bnFx zMp}lwUyBhET2$N5MIE5J_&|<CXb&|}$Buw#~h4}DN9$dy;lJEVY& ztbEX31X@r9J~sU$8U4Q}CBtwzJrB6MZlj`?dYBi)NuH1LA!fr!@;O6UlLn<@66<*X z^$Yl25iRV2ZDl`8kq-*>kcz`TxZI!MIv~G}RWr z8*oP4@3o^ohrEX_q*djwmVzlJ} zB(BQ?cWMz{FvGU1#r*=gUI#XF#?7ux9}^c_FmJ(2j;z?Y+I5QJa}T?^*F)*H(qATF z=9oEk@@7xZb8|$v?K@dUz_YOr(3#!Z+?Les0d8*Y|E;TkB*~U!$=CB&l_%CteX4CC zbD{N?YI3}7<7)+A60R)>M)np8gY_KgW98hGkQ}mc%3nxVA>9tKi14zf^(ZZf-b5U? zk)Uq3?wf93_4Z8)_0A3aEP@a|1|m&LZKT4NaIP^D?(#WHpj?gum|D%qjH^wD=Ty12!yxB`qxQKKNwjk2 zIc$z3CUUK5ES_WM&m;Bc{oQSl8gghad=kwBjoMBxz2LNVTpt^02R`W`)3`R9{6FZg zU%wc*NaSJ%8+zN#r~#F*gqDA}WrA8=s}pP7}pa;yCQD>LigF`e>-cV-b4PMA9iY0n+;pw5VX zP?{V7fqt{%AJ~Cs1;M!r9=SKdym5sQCK2Xn@B_??P70ru_ubRF8}4`XJ0y2Xu|onm z$;9R78`NKP0b4>{1+ZF#waL0ONm~M}!FYiG%O}g*M7wZ8h9;g;8aqvtuo6HQNW5vN&k-fw6|GP*Y8)%Jk&mR{( z4jsJkY(p@=Dgcx19eB?xBRZrtkB;F*1QszQ*a=@PfI2+*t5PL~8^2a}k6q415@s8Q zMTGWoqUzdj)rY2sem3&jU^k3*%hoCc6$9)Z z?mREQ5Pc5`v!B&R|={)^b!X3A3Dm~Y8r1virPr4iKPwr+L^0#$E z*aqtfrV6Z{v&Y}?P=%8=?Alnkgx(W<5bKr~tGnkiv%@F7o|}axlGGFTDYUb`>qk6x zUsJ1b9si4|!eUg9#5lMEez$?v^*U4PMHxp)sCLWg2+LXr)m_(?^HDq#*+0)QM0XR}$mXdorxrpn3h z9`g_m{O-E=*$uPbjv%+DPyjFx-?*}yJYj47zK?=xhzEYRDObxJ!#-+Nl0#%lg+RWT zzgAszT9`qzfCvp|Ki8uA<#5wc+2Qrf=W4ID%<d3>ixcOgz`aYn+Hctmo?AFN ziF50(|9fkl&t$wJ5nxZCOaMO@1@Q0E0KCz)NNX6$1W_Am2uff7$tfa3`HGB!C)VZ} z(aHh*Q`QF1LyqtjvQcnWwEUu3GlB&%LJ)w?hH60GuYjUp@=hl1 zU#!wW$y|6%SJ)k2wWH~Vt}O9aMJ6kYRH;oh`Qn8>+^YIJxS}kR6eWH>!^EY!dj-v~ z7E=4JF2g2HS$|#5z3i^SC4=Q7_9ko+T?gKT8c2p1v*kL<)1XnnAzl)M){Em*gx3ku zc}bH`ju}N`PPaz=P)xd_-`VaMzB#8ZnF}&yr0(6z zdo06zMajYIJO0bTsNWr;5uoxm9)F_qB71&9W%hLcsw{TY~jd;q6p`vj+|I->fjSY&Rc^;QTw*Q(av z&hwa=6tYMt32YpkfqATkyzS?Wv~sk533KuqxS@IU)@{F+ctY3zh!+3B5@Y{weVp&D z@H<8OR6YL!w@OX~XFt{TqoC~C3xfU3SY*P!r%5MnK?cg_r3AI}pnXLb7n_c9pocO9 zj%x(*yS1I5Wi^X_RJ+X=1`Wx>sZk_)#VIIQb-6EQlj=vPDE&pXj?*#Yg`SHS zCS8g}6aHSe`PT;dkB~TGuGi{z)Vyegbeb$tb^qRp{Pr<_@ZA5o3XH7O3b$^G z<-!@7A*=ytY+c9alc8$jNeZt~V5Fvhbj>xNJ*mgkGBTSs&0u_XR}hkT#Tny5{Xj|( zIX2j;;b%DC4fRTuRem$g&6@c{Hh3=2*EBPlT$JIEsK^H3u=iy)3F)5@9F^v776O(fUyo-co9ElNwn-ZSw zvARzg=_6WI@)l(h|z4swy0^1U0!LEtotkr>cNn@>GqhT}?T^13_p%)9kFTyKx1>xg-mJ zqyt!VVkT8`xw}RK7`{`SN8D9hJZk1V+f$u7jP6DZP;#r!KNuy$Q z8A#0T%CcPWje>1!;YWsL$$s=y0R#{K2-T|7P?IVfv+KON6HEBEF3`8-z8#zRM~$e6 z`8Ud=69_FX=SNVJEyel@-$Rq%Kn+qG#h3!yk{*g!PSAl^;IFCTH#JKcscPQ zKfd^%jv0*mpV#`OgaViiL2TWyxUIb7#L{j;tkQU3;AQ#0XI3V|Rp0bkQ!{)gNJ}6H zYaL?=!6Iwb(%;{dvH4fzvWN->&f}DQ(uGihq3U#=bw)!C=<8O0iE{j)CKYkx{ zc3aybRm6etdY#}x@GV7VjuuxrA_FjR$;B5X>sCe<9+D;!TsYPoIIn>1UMnaz8krEi z;A<^7{(EU26YYONf}r;Q0W}SBV$zkkY4GRCXcCAdg5a1-440~^jCK~T2~&uYy8iu= zPG#!d>-^Cjcz_L78oF0_5u{ak5f0{m~0WW9F-06tRVrdH*B# zbz1$eF`h+BG}N8yMDpF3TXe{bqM?9LHGDdE3%tmuK*;?4ou6!xwSf+rzP?gDUbCrP z#57xd2e^ruVnvnJ`nI&EWrdqH^j-;n8v44MJ0cFK9ju8&j(NcjKqJMMcS3VUVWfhU zm5y9xLfQE4=41&BDGd4O3y1vrXgXswzq?jQJ3FOVf5^w5;2iO-jjzDWD%(r z0kY}TqONyt=@Z!Djcr>L!d8Btn)2$3akl#BIAv}# z-!rR#8HD$42+z*fMJ>SCVn^?$@7GG2-^Yp=6#xQh5PZ4zO%084iERlmrn?De9qY_> z*j3{H1oeSD70SXMKAvzb1ZDpSM^u!AuL+EkmnULO%+F#0_33}rf`E0v>?({tvyyq* z1%gHGlwNsKT#%fTz9a$#*W^`%|Dd=Qx_8Uvh;)_vJ$Bnq>YtVfrFVt1-4rCwS4E5W zMnl+MC}*)|sERYwU(!@-O;0-+ESed7y-^+WL`W&999pDg!Vj5S>2-;3HD^iJlPE%mlu-_tWhS-% zzIdxGd^_gkdEe3$aHAbGM6yTrRrq@IjGfgkaPeR_zd{Qt0I^8Gp%QK^rJI+;B_XFZ zg)Tt1@S2T65mO_gah8>j*)`Vxr1&14{URXn-j`}tmoF6X1`=Dx%miW{PI5VR9Dg(4 zLuYIg4XHuG4PoOAiIbRqmGqGXQ38Wm2=8n$e;vxvWy@r%c}LMX*i*znu9W$6kP}eV z>ZBmiARrnc8`Ayo)UJjx=r%ik$-IA!&Tdkkvx@0N^ZIoF3_bJQ(S9u-8D-`@+ATa^J zJ<1Igpy_ug+*>#zY%#o!0d#9Gv4upH<$ ztP2~vaGZgiym{SZRAMEjXXS2DR?I>pYu_IVENcx3ZDqAsLAAV_kpPb0Ao>%ckdF|k zRG3%!TbBX!=j5VYEk~TkzqwUej0SF0cvd-HPSKI@%Z4un)7-M{Z$&K=YrUAyW3z4u zmSvzDYrjG#H9zJ%c|aqaOq%?zvVJ$Zzys_L=5PLX*cx=1xuQ400$s53;iQ^*0)l~P z{YM{JIXL57FiG&fVJov?3DJ)T%;F5(Se1Q~U%V@W{gxfqk#~u-w?TdA2p;$Yu&(c` z6Mr+2IHwC0Ken54(uqe$?EC>)t3(>}Y!6*6e+l-`k4bKrA!Im$B_%?#hYf%q;(PhH zDVxnXP<8BQKgD|D(Deu?6Jp-S)=WYsm^t{dtK2kw3CM_quO1`*niPKqz-K#g_-M&B zK#<}zV4@nVL-A+ydvbmblY^|nGW5%t0GT#JMLjqWmVLCidc({PeX>@g6ThY^_&Yy- zPs{(ae4+2W^#Aj-lm9yvLfhZJsV*!vbqNxNBEesxBX^j_FhE0)2niHoBF~eWPz|3u z_LlGTMx$*}iD5!}DT4B`&C#AMCo|4@i+rsp^_HBXLwA@~0&F@$X)Q;ONv7^mlHCFF z3y`<^Xk4dx#L!YzAt~_HrABuLIp!4hkF(f3nLAmk97jd{M zp#!Rt!2)9cRaNv*7T?m=lzJ_>q~Xp)fT5pru^wFBrg-U5X$w8c@714m0|IyBzBH60 zA&QJxAE75~i3o9g?eUga`J(K(>Vu&U|CCj64g3%1!l=_e@<7!u-WHlc*vL@Ac_-%z z6_N{5fl!Cm(76h;1!En3jytmMoO@qCJnkc}EEowT_l*A6|2Srv(d{k?F^a~m-L zQW**Q0@^5t-v8n#MnnYMoWL1&|?m2UQ;fq1eN$7Nt3BuYg=4kdE_e%Kpw6Hf^ zLb+)qoPh&p45a_@a_eYrU@&>G+vA2oY2&3tW>JkVb^FMg(YJ8>swl;mB`%pKh8 zZbKow;sv$V8x`guZfTU}4Y}qG2}W@lqFuz^CsitFPBt8N2}8$_txb2I9mL)XcPU9q zjrx;X{25y|z!xXlXy^-I&JJNOFw8)=cw#B0wqzkAAI@_DQ-CjAH4$2gyFUYLf*v-^lbZ+4&bfYNNq7nnm+>s-C(LeFbnQWPPNFG>jQvr z!?9UB0bS?m7mXwDD?imJtj+kQd%*#D}-dCMHlJ$XzgsuT#(y0F1Pq$SV^~)S@TXma8KUX-XkgJ#GzMP%Xbs`4W>#KvIKOMM^Rn)f-4P#}GsIq3* zIi{wypf;SSueGcJAJ>9!V>x}>x=wkf)?igdI!zcyfnS30KvdG2858J!imc$NWD~%| z-Eld;3%gEaBIb~2zm*V$dac>Qc|&D}btMUo=T}!-G0^aOZ!3cqSoe!#R${70hPmN8teE1S@e!hGWgLk-9zLg*2WZ)&$ z2$n${reer#nvv)-r4C;?h`Z+7PakbJxgYb~p5*s4+3&hxCeZk?W~l3dpq!S|FaY4C z`q+hHM56S(!nVe?-#eWs0X2^FmHW*z!7<_?7LI+s%x;u8Y0l$c9J1=?PB+afHbXl2 zrfBd`D2C!3!tF)C!absgBHM8OK~@OTh@tIa)UTo! zR#lnAnQ2TOCJ;M0c-EdrYZ@IgWI4ZTBHb8>GIdx|WlOIU0ZOwX&+maE`lnMM&WJVi zMFCDZQk-=}o#1LekdMzsGZO}%611wGd!y&RkJyd-t`o-j@PqRMQF3t8f`f^JKTNmN zZOjeXXoip@Auow=U&|<&p8YP?6YjM5TfdT z&Prj$^an>7NePjQyC;(>x9jDya7S_E0}g>Tx3u zsHQ?fyzTitsO^L4w`9OmiX72UoDVj;Qx8B|duT?k*i645FZuGp6qNw3KV--PML7kf z&@+Am&XS7_%O>DlhgL=cP(o> z5pz-PHt2>kC+zS@2A;>|05IHg<1|jEL%oz&=r*op5JrQ~1PE;^L%(kP_0a6k1mVH8 zw*BZ_hbqVtQX3XCK3H=K>@;>1=J*FTL))p2j_dr}Sfi{-gd}HEc-|I}G)^|1P-{?z zUF64$lGR=q{425-mCYEuSUI2y%8X+7oFvKbhkT5REOY}I^R+$xa)5RgXT-wVkwmYbd_@}&dZ^w(KYZ{9Z~5>OKdCC*#Tl~m#a1!@1Kz? z0>2OXN|<^;M-RLe6_yrrR2y{*yEx8RDJ*Y&P4s=UVQ~c*Il|8b9|*1jSPr9{5;2jq zrT)iP(W+hhz5V7yYDnk~;&t2^&sKz=vnw3u`nOqT3@0Q4ed z5P0l=r2CQhlksu)YnmvB`r+2zUK1FZwEzAoM^YiQ#fiEp4Wo5bnykotXCO`{SR_{C zcOWKbp!D3t0!nIN9GE2{Aw|azrM#yq$$$ci+c94%`*+U$V({eT%3rPPN-8j#P4_c; zE0$@cVw@#o#Xg#W8uZ81!yi-T$D&eqV#9VwWlNVp?cvmUo!s0tRZfNEW(SH#s&4S~ z{rycU(#pb-f8)YCC$!gpht4^e&vA!JYcz|~3i_?mZor8$F>aQ=bC0*@gX8^a4&;<( z=4TFrY`q$p{kkXI9l5BLxHwcD>(HIVQ#=b6gHlf-xo}B{iQNNN*9jUQS9Iy%6J-wd z1h^5@>6FLyZYRfL&`GDx2Li1>re}zjMWT2|J z$$K#=hK!myefB}R5$I9GieRPOi4d-bhr78Rk{i$hZ~~v05`yYQ0)Cyc0;lew-&FOZ zAqAYPxaN?{Kom1KpB|K?brm(VzI%?W9_2Y)+nwLTam#+Uno^}cgL(}Q*~f-L8^Xz~ zv{H)4&~_eeq%8AL*9fLh$Od@!CSQbv<3UYdH3GQ(+CA~ayG%klu5iA#95@!On;LMW z;@j@J4@$|t3mY0C0@=v*-XG8EU*|u*FNa1;|JS3L)AoN>kX+>I0y7+jP-DR4!`*k& zi4Aw5oo+wO>G&^55d>$le0UtFf-D@d5~YiLdTvsC16@ki(ch+0_LjKIl0b6gjAE88GUyQs%6i`uyU&52 zcM+d6XZ@ku%p)omE!Wa~cn}~ZRrQKqqQ=~*+5Jyejy=;f-Jl>KdWiPZ=Z)A8 z(d%`Tm#c|6JB4VA?^F4saOu%TJASE_;1oCAgaV=isANcaTqjX6M8Y;p0l)VhcWU~q zGBcHbv;Iby-$xy~N+HW<&Y@1XY|TFMt3E7HnTFD@ef7&Ij)M-L;`LG*Qhz4X;qaE2 znqpX+*)%lNG5IVJi$LG(QVRCZviM*iL9b2;By5;Lez9~&up8;L(hrhCS`tibX2GeO zykP5P7Jcmpp_N|2(4}}ugwIzexo~7+X@@k|ZPfN$^2Q(_(#sn>-!Jx zqhbATekHF0yRB1i03?nZ*yvboQ;5Bn$6!??W2sIxO=dbpCI=>+Bc*Qnuik&i}^^Jamd)%uH)ucKdCPRn!>35h4;3)6=59}D9asCai zR&%W}9$3d%8Lub~dHOvlg!UuR6J`HwlAE@mQ$b-?<7KAG0coDH_FZqsB_mde=iW5H zH^w%sx0+$D_@sbJEj~_DHX<2>lN1=}M@UDZsNQS%~AxNL8oy=9j6+)}$!8Nn#1!HE;BMEk@%57=tI-6-jZM3Z2FH`9ie|cf} z5(o!Buw=pSlw2NkR-R~34Sr?5Nv($M18epApvd*0vQ6l3iMajf{@Wj(99@b`_9G5tCk`p7^~sWaNDE+c~QF_WjwZ$vZWrCZV|dDpWV@`k~=R5-_hc8+YL z&vvtmnmSrg+IzN%E>HHp_`$}7iWIR@Z0S5&QVKjMJ-gNoDEMe+ul2jKMOZs z)Az&W`T$LFY{1#J9(5Mr`=i6+0_rZ(K!JCY^uBsW=KVQDAwR=F;RzFjU+B)CqI^UN z2=jW$LA#p5VhRSv;TC}l#1Wl&9SRS`6$otOGdx11VS`NzHe8sn5|R{<4P;$fZWy3n z*o8dc<%E6d@PS9iy8GP&rk&%y5@>dDG*ekq6S{+;E$#}}j2B=F8;#=t1>2GPZ@hBp zUvmG;gD?H!lO9x1!CY2{reNu18qoYPF8t?4-Z$U(j<-OJ$*_|N8cI?c37u#F8q88Y z`hC&@%&(2pscGZI4t9?>RYnCbh-6{aumUWNLC5cZ9EzK`Qe;dmqc3xxF-VG)@D${&98qq!(2hBa6gQdoSMP0?cTNP^NxqyTF+ zB{)R(c{{d54yEQyc*Q!iB5Jj{4Y@b@(YYQHo=BzI)f*IC&ZUELS2HRx3V+PJSaTb` zEa9Oz<;)E%OPrz%C4u=9G?6Qeh33GWZH@gJZ?lF@EHSlWDsMh$J2bZ=_E!7&znah<;>nex zhBD4bpsn$=qYbx+ql?Bs0hknhNtGVxbL-BRWAC+I)!Np*dQTML)Z&b@ z39@p#O{+UlkKB^sVPcuPypzj$bTt(uu8fk1B4E*TEqI(aU96774=qt&)J-s7^3E;X zS6r|8EpzOcpnNFkxwE40MvkgH&6pRN=c$z;BXO(Oj~$xX>V%&uuCq6*QYBm-kEgi^ z+iyLiceHs{d-NyqdlPb`YmJqW=tWA89w#nwovrS-C6EaF!N0DCzj(Z<5+)iSO&r${ zg7M5!E5Z*T<3@Fy@n$2=OBV6jd@BXg<5<4T?qK;>U%Q@b(VcY(8zF*ml|I%tcTB`q zUdX>XO`yop=){~^8675zoY(jSVverBw)Ch&Z4g1x6If*e!$3EQZeU?$o)>mzkFxLb3%3S$@Kq5XP zZHy{3t|W(*RlS)vbyS_Sqi}9}-X3`>;)sjl!&3$}=}~j#NMf!ZIpH%Z%AWT$jSx*8 z+`SKO8K{c7C>gmaJ%2Z(*%F0?Od!yT%mqJS=7IuR!8FUlkB3mYcEQt(ZHi;x-r|5> z*$hlFIN^qjsy=`n+TmCaXa=k(yzBodg?U#P3JkBCzcc+@T z%ZhiuJN?F+R7agF){qJHmM73sGPEpzv%bc$(w`_Ox{*PDsLbWS!6G2gV*_XknN^H~ zt9Cug=Xm%v!;T#{ef_R^uluWdw^#OF>*g3me*Hl?b1jm|fP_=7-2c?6AU@lb^(V61 z?HwVOc963adqKaMBPv{IY4kV}0FDGv+9j~Y=?|7gCQuF7S})>j9U}lC3N^%krHUN_WT{r%pMjPU1wJ;y}hY7ZGRWi;#7%+!_6s4A#v@ zHNF6(fD>{YeKSzbsDJ`=;tru^=bU~W(Lmo}JE(Weer6hfI;xlE?7`_(g*VYDyEruv z!>afeCD8y?YD2V+HCBeflR)oI7(0xlQ3@w~(p@k^c`_2!=vo2C0)(U74=iaOQR!0= z0`;~qozxarVTT%TS0h1A0n%ekr4S4l=UK8~kLOLGL?KT4JTVLyCj!`b(1`7PM7j9g z_rU)Jc%Xxu;QdS!USc|XUT`D;G>5IJ>o>$B4{7o3#gl{Vild;Zm8D$QRZPl3vSvOq zriX!jAU>-nx`uO5f;!Nmh;(atI&WoAK0Z_QA_gOfu;DBApxrNq+aT}UjuRRd`PH{U zbSwO&7Y^j8AkVHP$U9_(@(}K{aK2uuerGHWxb5TEKlRQyo{VD81@l}i8{4!f$Gh(3 z1xMGO(=HTx%=~!4>9p>&Ysf{vvQM>msqTJN;V6%8FrB$sZ;Wr4hT8Kblp%N^i4s;c zG5<<490jH(lHLpJApHCk@>4_elcywS`>83FreG6u_f>>Ru(xt@Y^qYusOx=)`5X)v zs9{mSPeDe>xZZ9$@g)4ZbS=yFUpaeWl&25=&(g6LX#`OE0MAAHLo)el(e^Lxz7Id} z`Aqu0%$h!_i@->7DoTpLy%rZ~bF)CcP9(lNqzU{NP@Zdvr5vGuIKW{eAuSRae`qS) zm^7p0{>nA6;r+_2$=s|*?N+qP9{aselPn+NVmIu*$4Lb_z(kWAKsdZor2vuGu^16S z>Np&ih{|QB2rT(l2EoYNF=7I+ z(f5p;h=qC@pg-4`;HL0Wqz0vZgFId&%F;NiKN&lEzlI|+>K#h9?ht+to%DWg^*#%1zsW?aS^VGGiunIt zL>|8t9E>_AGf0QgqCR@QeA$3wri5nl%DF7is5lRYcqxy+^AJ=Q(59pLDJ}=kR^fr> zyAG^L9%lk;kno%d!TEeR8FHk;VN^3AW;8Rv6bAlcm6fG8^UYr0`b#{n&nP*LMs4EOHLWqrz<%-Egy=r*t~5ck2V= ztVZ1gj(rhxG*-buu6UzLH-MbBCilPgY0Coq6YzP0ZJyvo7@qF@lN@w{`LU>To)oDO zl_l@hQAs5US2FnW{vjFsQixvub1JA>mx(?z-0`7!9@PYT~fyPix)qWVMM>t z3k*g(ivQ}ng7{AU8$bgRm*3)Jg3gjXW1^Dd*|++`QRcjrYbZ`vI;D{0-29*{hk^GR zSC>qEgSDPsv^7)M(IF3&gfmik0aX7U{exJd=Vv~!aOvwEZ3COsfzxW^8~kWSEvf{Q zPEOrE)?A4OeznH#J0qgE-Q0mJdXqdZLL`;Km{OJdxhS0lX5!qZ_~GjrRcal0EOYy{ z;GQEsq}EK6xU*Tr2b#B>n3jhUfHVcqoEsn_ttCD)(muv@r1xv8vQvDi%JI(e3iDv-HqY3%sv!& zPL>O?o(4!vA+{Z8EnV$oS4ed8CM4qF4S|}aYgNDvmWdcCieezhE;bkgsRo!%x1ZK# z;JnSD*A$K_t04uX)fKb2s8+%-M8AEjVt~&OKT~$#8wX}WEfwaR$i*}<3Z3j*oVZ>#?$AsHsn&P4lWQzT4_SfCHt$z z%m=aIA;=6H+iuH7>r`mevnc_X;<$6i^|`RJu8Ml;N>LfJW9a}2@&o)hy2Gz3OD5Xh z1wVn}Kkr$K{Z2!CsGRcMN*XHAqIjD5i6?|MZY!6l+;!GSQX)I0OsJ zNQ#gOvFRJUd4%bE>0N!3x-^LT4#ax< zrf9T4tCF|e?e#w)`1f6#^ZK@0s{cJz@)-$DRU-31nnadOc{-1jyA-& zlHAgk@a(y%$3dyO-mliRjf-e>;=os1y>_#P=2JM$Ji2jF(mmH_)ds%mV;_IlP}$8% z^En6@HmBh6d=}PeqolRr&hh;G@75!HRhAT}ZJldK)Bnq;fdb@NTV;+W1=t9TprEwMvLpgs(+1T4HofQmAK3mKMPCD zaLaguVRq}0isuiXG%fXxb5iwAi|uIE_tin(P6rh_pqd7-2~vgtOag%Cte)|VBw1sH zq(Shu*a%8ars`F#E_GgWe-1D4XCzqcHC!!kZ!oT(BaXleA8SNfOkrzX?4=-C7dlKh z(N_!^L&6Bx0+|Sox#T{cViIZLQqu!-qb@3y_~`A$P4=j(fbxE*_OasBv(N-?1eLPz zbpQtgh#~+=iZ9n=+CS@83i~2~XrXYXf{odCW`9y8F|6+DjM=42ImNlCC+CY-Ik3iL z04o-upf7PcJ!N3E?4;v)7P^08$Ed8!8c@>jqtE(mrbHzucGYM@Hgc>gXJQ(~VwE3( zaJUUkPV!<@LGdiTq zR2m|V{pMpnR)mAwdMfhrmiSXSJF7UG*|WdNxmM7Jk4r^j?wu)`o-ylKU)Co{sxn!R zH$8M`uNFM@AqNup{7c{fCUhzB@1|`kgy-F5in?y2hFIh!)q`fsWry~)&w&v-4qyV) z^gkNqSt9^e$g@r1%mPqERWKs!6Im!I+0(i*HC8aU_MtsV0yle2193Pa>d~A}&2A+eH5XB~^>s6V}_w zK7*FIG20bNwLc?1=iC=PzM>T}liV@|*2{(IpN z47{T^-6SJYXdSSxRGG}2I|OfI-U0(GrJ&ar*#k*9iuh|EXie)7pbt+u5=8>gD9{_=s}&8?GU+(!E_qXvdMnl=yC&!@vCt& ziA=0&dBA8>O#8RUf<~9X$&!6M-}@`<_Zw**!YovO?orF!*DcmM@ec)n75eW#h9m&% z?AJw$1ZZwJ?34}x)iB?Xqtn#Y8bWZ1QVfpkXqmgxamZBog~Z?(&0$F z`dyqWB)Tg?h-_-aJOx9{NW)5$04iJcfo4XsP%POU$62`cj3z+OBCg zcG9uUE4FRhwr$%^I<{@IV|8rXM#tW{pYI*}H>^3%HBog`YIn}YAw;x2+n9?Wqa^-#CJm(v$RER?Rl(Z9cg7{6+v&k3HkF`_XQc!t9K@ zL*L@5+<6v)SsZ=6a^|X8{rx~CwmEeAfoHi|D^8k@LmCa4gs3xgUA#E3hB;9DwHL{v zKtBo|WcQL)w)=1q{SA_dr4ZS2JMDmU2&p|a+Rw2DwO7^E_PNpUWgY%YgE4#DjTdEP z$+-C9hnprerRha~6^)1svq$AbDoU<|S!VhRnhD%ogs#0H$M@GPlk+gkNENsEl$DtQ z%Waz=Tq1Z(V9@qo4Dt>*mS13w)6;_God+uVv)z*M1$8Btb>^`P4uA-Q#A3ibF8#*ugYVB>;QqxiZrS8dD92ub%;$tuU94^7=fH>oe5O09gn|gB zWJiP{&T&9m98?KRY#>LXQSpNVin%YTVu(v=pG+qK!RHh5Np+4W8^K>2f&nbmr5bI> z6B{V=Gd;U{xkPkk7mRb-kNk8P7@qonK$D3|2u+cwhzLbaL$*xl7a5#Yr#>lJz>+<) zB-*lU)Q?}nghc@nPGDH%5S_l+9j`7rUe{}5|71*rBV%_f&O-%>qgk^a&RZea%IR>( zv0t{M$&px%NdI6mK1O0waH@etyv9CFJZs>uHqfjqpq(#wS}zD0neJIGj3kLrg^@09|mr#6z9i*>^*u)1agf$PzpXLoOKR zoF0(ppE;hj-XbKr*AqE{aM1h0h8B`;o4T^8oeeY>?ZTyc(=7v*s8N3e*%rtw71v5W z-dN0cF=H}thAs`Oi@5SfD#~or?7hM~j zjQ$(j^ipGO*3hk*zk(--#flcF8_u;ll)BmMY^J`rwXC{|jV)eor?i3L2eeOpzC=)s zSlC;~n_GvyE{9Tx>e_ofk^PQrJjalnzYBbPS?sP=#p<&Zv47*fB>H-)(>u zCtWQD7(3m0hF=*E?3g$M(=m~Vk`$##(PeI|6Qn#EdX{h`?79JfBvhf9nx7p%n|YQm zN(j{eFVl+X$G6>asT-{8GmkfN-$L+om^})t^zqLJJ1lI>M}Egt=;hG~s2-+=TvL_K zVlN0Eg609IM%Xn9>fM}qQkl;%?bc{mcu5R}*tI+k$3YYCJkZ|cbQK6Esc3Mu*)7Eg zmNsr`%3aO(<6fWFdG5d0FT~IWtW@IvE_6S5yY;aO@}L)R3ba)hFUZeV^hPFhFBr>t zVIdnhJX6B-cgFE-+)XgWs(2?8bW<;&i02|3JfN^+M0O_>efpi0JTap}#W*lumP}4A z$S#jZQD=|-VO}L{Q9ZU0a>yl#Q;snhCW4XBUpkPLV&(#6hc2xesbN649%3*f|uUhT;SEQja*Q zaF_0K#8R(BUh=7ojBC|*iyI*PKFCls{%{R`xWq8Yk{S?d;AlOl{<__(PFT7*kg#+h zk}B)@M}|qA4XvbJnqEINkU$T@0XC*MA01+6IX(<%L>~4D{AU0>DUNB)>hO3*`SpND zeYelN-sd8pJ7T=eVMMikRx(D05r5QxvGldu&yr&2R2pHVz9{JRp<*XqA2}q*4`1n7 zjbEUCF!VMSe_c}gRTUIPZX*35LRtRU3evO%!VAKND>HPG4p12Eb%10@sI);rU?^CQ zL;HCdOVn4bofv88Cmx=>3EGVme^Q_>&sq?j@O~D0CT9z-h5ubiyi2vZkoJoBeSl*5 z63|K;9n#%!}0N?&{9$1lMiP4#ahnHI~obz$`mV!gJPJ4?=M4RUMig zZC=aEB_7b9tJ*-)SCqmoWuii$GQKZ!dyo!L`Wa>7nW@WNVYpU;+9U_7M zoIW}{EiX9Ed1@yRXXPwSuB$Xh8k}VS72)=WZ68Z`P7*J+<%0#G=~|sc}%1Lkjvqq-B;N zKmL`JNL&3pv^XJ)LSRgkVje4yB zW3Qfe=a6Y=F#f?-a0HQ{qrTUYrxjSK|LX1(JxQ`Z>O|(WxuoD zKNJNr>=OdvmA{HQj!4u834#T+NLVmSEX=q$cdZ4ZdeX1d{^RcA8=8k3Mh zR72(Dty2zR!rABw(XYjLM88&3)jSc=uMq04$)O(dx#od8MfsAm6kDL&?y-_og9dz` zRA}wu2id~$?0I6ab_ggnH%lacD3b}R8Z#~iwQdx)pjd+i-aSs>mM|Ay3UtbUaN2b} zq5naOi-!<+-k?Z~qlz+%Wg`_)U=XU{|Mj6{BQ#7nxD3~62MSk~C1;wRLi$uo7Gz$LMwycBP(mZn_m!b@YFKLM^!NKH_zOFKuzakwN8PI%<)g- za-u40KX95G;O>*ERg2B}IZ>}m84nVtWfMa_#e%aUSh{w3(2#jgiCVB=kAkggbeMox zr_%Qqal1*+Ef%!^q|CXusCQnKdpRHC{col3qZLPg+8DX$@j}n;i=UJE|69q!^@V@k z@7%r79;9d8L^sF1VJX&*{VSWMdVEQcPXdj9ZiCm;e3E<`@KvnxS?5gVpNCAD34lIQ z%sL-BL}&6!pN7?XEJ4%L@@XCjGh*eYBdsDjr)J{xrB_79%ReQY=>kv&i%1%xKk%#z zswk-;IekNM0AlhBR?G5jNFH+H{wkn0&#wttD;KiJY50r&xyb|H1ahn!YNURB7}p2uccY=tJ|yA zM-)IK7KxUT9k(d05C!tSnWL?ftYT_LmTy45ffuTQ&MAPFWiv-&*kJanT4%Mt6r(T8 z{pJ5IG%-SPfi0fGWWFcc{>Tllf4_se(|~?pCPA3!C)zd>4%*f0dfmctNAEEzRot>o zj&wrUmdu+Be|q3Vf2qmEJ=iK((?NZ2msH1Vh1ikzp-@z#U6=YJV-u_MnAq9Fo0<|V z$~Dk=U0h>IWT^%TDl%+daxf52)++WhntpujAAqS`Vdy#GZ{OeK2`yT|S6?8ZraweE zEv(r0+=Bq=>sEXHs9&2kzX>Ii`9Dv1$EAOybLK_=b!iNph(qH_G=OF>kOW8l*!|)k zJJLlcE?y! z4f~Sx8G#|V;277@r>1Z#h&l8ddgRp-R0 zq9$Zx0cL?lCk)$f{YaOdJ_uKHai@JWR#0M)b6=DpB{5up4e0fI+mF-UM~Cgd(9_#W zgcofh>C#lIkQwRb#S00kjBk!X?6&LC?(wxFC`}a!tZ23slhADR-B*uHMOF#c5tyvr zs^q~rd5WJ>t-4C;)0uCR0?2q)M!MeaPdMk@9@IP$VsW6r+~9(Sqyy{p4$~dxyT9)x zUpSe=2KYj*soyl?tfw=FGWa(}W_({HeLmiTE!uDZ zfT}o^7*&~Hp0cNBwgLE$dFWBioIENc@G#2A(-r9as!EMyDfFz2U=oFMaO zE*d=9MjliHl-8s5u9iV*IgD{lYR!)GDzG(SCZ@BGGS4uT;fSWRM3^|qu@AO4-njZ5 zJ@F&|AZE{^>v1%>eR)?EU6?j$Sel0D@XO0aPb~2Cm9wz-L5-u}NG%o3zfX1ID}6jVO&#f5ZN~{g*ZI|atXtf zD>WM#sXpy$lf(PfY`b3`yU+)AGi0YkAebj?1bhw3l#-YbzAr(lVJ@UR*0I3;(y!d2 zU&9jZQgYaCjWIP832Pr+bEuT5>~w1+vY&&0K-n6E{{ZVMK@1G-G#`;eGz)n&1Z%FT z*aqz|i-lz&s%PQ=2mvZ9M**fBs+ADQggH{5eWleP#$|%fOs@x98ntTCsrHIau?M$- zvIRLR8wuN$wsltx89$2pAGGD`nsPRP++|%XjMDsBZ+K+it@X);V8SQ z-oyy(j~;=flu(^j3@e}_fYTRV4Ogg*WJ=8U&-`Ou5Z9#7++~fQ-fnXVt_uyLc!%rZMn;KN6hvOHE4rqQPM!M%2Sn z0U#=J&Z)y_Jz^Ic!m`#_%*)s$Q&~$h!3}6FLY7 z_ppR-eXy5iJ_04W*Wkw8c?Sf~mN%dfIk22UXT}b@o5e(bZ4pRF2UsuQPFD^QJ+-rJHO4=dEH z|9lrH)&t4TH+}<<;FuXbkt~O=Z5O2@cI5O3hvnyAf|!bXy7;Pv@ul)rsVeS?NI-Zpd&b&%NlYV#U_nB+?nf^65j>HtB1d)>2q?c+c?&iFC| z8hdaL7fY=YsUP1=Y~-*hvaO^DgK`@)Fe!`S97*wBg$0?p6A}0BUPoV_Ms_Y?dVc;$-5JKdg)H0Ngf>**0G6ico z^Ff;28$z;~xn4$fhRyMTSNqdMlt8jds!fNT4MCS5G{n;kltT zVWOt;k9y1aRRnY$7DWDjd^QW0pmjP;3TNKIQ4K_DV>zhNUmbtKy(b{qsmJ_0AQ%B6 ztB|%NvjV`F*ry2tp-!<`eGo8;k#O}yt{g*-LzRS9Y`{<|WI%si6`^{ ziTF~AV&nlx^KA|RaKJ(eQmWvMEC z(s#Sl5D5x7lF(Ecx07qZ%_6oa%P+8LXux!3!&yr{<9*6l zs;a6bgMq(od1kx5Y5k~HD)yh#F_}x{%lQwVhZ(lQ)!Kx^n6TRGz&bxcgkzz;ujZk< zKZ6{7KEF794!13R-ljQu8emUxClyS1QNc!_^Mli6eixY38Y6j?IC56}5s--lNY2`! zu69?%R<*_$-p`|!)nWe}bq3R|#4X5E4TXs0R4}NdO23<95z|A=^en@WO@$?uC!b(G z`qSL^-T7KEj%)%)I6@;#$0oouMfT|VFu4RIew-pK4pB=~7U#A$ZjQ^LMchixe&D5x zd3AQ*nRq%po`zNZ0x}y=YpC}cJL*Ti!%*{YZ$z#+UQhu8Na&BNDsqXeS!0<~##QDo z`_eN%(z$lyxpC^E(?wZ&kNd@7{jWCP|7k&fZl)A+RwP(&+Ng6IL@yct&H7yb8w;VE zr00Ez6s)Z^E(;ORW@Vza;D<@#kjL7(UgHc;0!~sLn!gmclW&F-pok}bcHpCZ}-4!*D+YL33%m2wGu6D|t6}*xAN+n>kLuISEPTu zV$3vXraeg?3nHe{90Neorc+UoaB5?_AO-a!OA8hc7@a+iT6@ z?vse};6T?@CKVuS;hkK?#GOT}D(}NM5{m1L)81pAH&&@P?g&fhSn?nt+o5UK$nd}9 zV!Cg-W>+@Z`kdq5?x?uOTXO4PR3I#CGbSPHi14yGb*N^_^IY*Ju6e^hoObp;Hioy! z?y$rw6l;{grusL((d$kReF2v9qj8XVU+lb3g!?9m`bSfO>63khr4afvgPQ&4ax<9T z$0?gvCm5@KSr;m~{k#H(<;uiXXQYmw*JNrjdTf}9v@u!Dvklx$y9u1{(Ac;HR08&-g2g@}mIPNKnuh<~vD_{}eB=9AST*Y0oXQ7*kU`Y~f9Sx=qqp5O{spr< zH)?7VYmDc6T0uB``2{nE1aG#As=w^_AYHC4Ed_)vRWw3`I9Z}1>T=&~3*}TX;3H0( zDr2brX~45AdKh*tkeTPwcCY^%?qG=^3c#T~tP}(S4}sbhz}WR%9D$tgMM~(u7(6|I zdqJ$3fU(iTF4(#@?BNaqByxeG&l+aXiKJqVTM0Q4mdv8z5zMIEHSgP>WTwj!C>F zD#q%& z-A?Bh+=Esh09EpPiVJ<&hLpp20ejPDrxI}m8p5^eoF9$%PJS*r=T7XR!qk%5Zr|2;w(epL6P>SEZgv}Ts{L_&XM3z zN`qXmqYx@)zW>6m)FG>sdGhQUeYUFZPQe@y)QwhdYyf9$<6y~P-F%l-BLAxnp(_qx z@bd2kvW}})+;=-AWM+kn4jK>fI3avu_JgRzML<9-MB^*&iJcTf2f1kOqaw0H(k#;9 z7tFJ6=^tCuKA*r(=kYZR1aa9XD6u2bvhdUtHOq z>s*QOC2JOo`r+ZPG$*G!3> zgLo6m*4ROr%dZ#&gHD+=-wEyjV;6Mqr`+xDmb>$}l$*=qIhdFQ!1&tV85%9Of}b>t z;lak&>;cyyF>K1rO7PIa{kkwksq3*dgHf5_P!T_!rNXL2jaH{K&WwxHb)DmEe|@qi zvMP=S&Yu&%nN*iBZ~WZSy=uH2EeoDZVl`Sw-k9y zjgM-^w$fpnv0R*4+LYcqlYD^5bV&$-D%4EXmdNhQ&0icyGNoL_-kfVJ#S?jCsC8u> zBS8)dvZBw?73Bdm9XTo2y6w}}7gXHn3=SZnF71S(gEMklget*!&5zTY@+Yn-y#ZX8 ze-vR#+!2WICmi>`9nXv&HK4B~`FhmuROHh9)FAc$NB(ZsJj29x=#X{%tL+=Fy67<}>`3DPg8Xf>t0AS2P<-LUp7U%}9N@ zN;B73pPtQ%Pe9%+ST7TX9|wqgmxtT)a)j`{hXte$Ns>SZsW9QKb|BL3(!ar8TvxI` zQ(GrOQFjrqLN&^>)SD{3|8rgn)WdE!+Kn>#kN{TAP$n0W)I|BMf*i_)_i%Eo$bww0 z*$Z9gdk1?Prmh$2Ji`T_gko$F(L6~uzxZ{A-Hxe42PdTF=03!Ko?V}r+*OTC{2B~n zIG>*!hCX#v))ac%V^;P3VU=cs=}cj?r`Sl~e3VQvAQ(HV@G?{j%Igy-B6E{DsD7P2 z{Gbzqpasx^Cp9M9Z=}W|ccf{$vtXEmEX_vauUoR|&fznTfsBjfP!32taw*^GLOVqd zw}rrbjDNmf!nLXSV+N78ys0lDKBTmp9-ey@~#mgjB{C z)U$g@+BSQd4*H{b+W@M!egwo)b$jhkV&pmP*3R(PYlQwNlZw@RjJ8ygTfcz8fo*5N zXk)dg5!+B9e^4PPqyKXUdb^?4()oX>Zou~``u5cEyh+c4aayZBjS$HWwM*xV`Y|yT zE{#{-M>r=sCrL~}mN~f|pSq;!twwl*T!f#dS}F46T&Dy?<$|+At>(i~aB)XLu!)GF zt;+?3kJk!eDZogbRVuY-x+A2Qv(!I~|Im1H-*j`-bF_O%8HlH?H3(Jw$snu-Y>Iy= z9fHjk1H6#-yTcm^(F1t&K+qfcLvqJ_!f0ot@6_v}Zg~l|R{LS~KKuqj0QUOMXOJqziHJ4^%+|Gal zh~+##nC+KFjNK!3-Jqt{>>$&m1LBy$5To*OC}1()ZLT@JEl9CnVw|%DW`j$NbQ#0G*0Pf$a_p!i;1TKQuj7-|U`0R~#hf~Z7z^C2R(Xd9P-SNAWMgStKs zwVpinRefJqz7JN40RJ9Bm3a8;;o@1qB8Ir~?=eIlxRY@r`^=^$th0CE4QU=Om43 zaAYfO_+WK25Q_xWpq;BXdJ&>4Eq}kDh_17K8lSvZYjH)YHAULhby=9Yac{v!#xz@i z|I`zhn&0{2O~n3k9x?rBUVlEs7OoRZ*m1l&>(8*%`M+U_vVqTV8N<1m*{90bX%@6p z&9?nJP~35l_Bnsf4yH^7nLU{;lrN(xQ^G$SK%R%IYh-I-KFC)h)yl=$^V`589h;ar zsJL{Z8o&l8{3crRjvNC|<^;&L&Vd#&1!X*G`L#KpkyL=)Dg&;C>Z|Gw!gMGNgfA75 zq;^aTDl&2pdm9*s0u*jy9r3Rmjw>ynxtxZQ+!}^+iD^;T%qkH}6--&g)TrR0Nu&Xz z8W!)1_)XL--a^E8$T5&(JW-4@tPn01Rig<8Q+r;VDG$h#CP*YiEHENe{W7^6*rdth z;36zg4V3>ik9oDl9_sH{f`S!lzF1wLlNo|&Q*@q}C`#|`7Bl9AGpa5~PUm>mEDaX2 zZmi-E*|^=W75+@4)*aF4zF&cE3Cva}YeMa-LYJ{o;e&xd5LT-h<-YvdLRV65SL9c> ze;d%N;f#nRq17@*l8?>lh5dH~=bln<*!P#{|6tquEiJXbWHU4Hv5`*a&AAe?z!DRg zQC8en$9|oE&uMrbzGt4oeg(S?!~#l?qZZpkASjwQsOijk)sO61j*+Vq`#?*=(TK=P zWG=k&)pwtM!oG$+@7o{eI4@Us1`oKIU7(&x=GC}`m`Iy4lSOG9dUbRTw0?e)-N4^TiQb@`{c~-;4($A&kM5Hz`v-OI6?wKxl!=?)O*% zVH&Bw5&=@7k;xbvaV5fWG}y=qI!sKFC)r$J2LE1>9(C+qZ^NWiK2i zbayII&^)D*p~@%-MYamelWn0*nNm->;Yk2 zQr0sitlSTrlWY_yL15gZ68m%yQOGl&fiht*$Ns`@^;@XHMO5E zV)pu8aqxNi|Iot|^B?2mi`!3Qpg$%j%pLF6oDoH4vj%g zGoV=WB!8uuTmX4F!pm32HcL`EPlZ*tan^cqX}!0|G+Ee5XOc@C$DU!tB=yo}5BYb= zpX4())a5iLBNYdq<$^+GRaTY+DGM4qsaw<>+G6gM7&$8Z5;TI-?!ZCEOfixHPQ+LV znO%+{-4U~ACLT`YgJ}f0gRb&s?Vk;B@mJLK_Ud%5RFuO`!q9JQ&fRVAu zR8fudXN)qieS$lwn2fwDs>Q3>K=gfY|xD2B@UxR!r7-bW|4Bed?DphF0%w-wr&dJP=lKSl-*HNUq->BMHf7-H>6^XTC- z#4t9yn8M?2CB8l|ST`Y!pLPxWz>mcHIJK&rOBm=qhj-%s;1$Fvl`Sz=wjet@VGj$W zPr(OLbS>2oTdPjU^bnB?jcutUNV+@Rf0{HgLm=e7R1bO403=K9s79jPk79I@r^wmZ zQkl$JH`>EXH1ln#ZZ!(hj?Tpb*ZSB=rIEOPM^jE3DYM^Omw?=dRk#811AdF2XCnMk z8k$&i2!TjVfZ*k2Mat)5poHUl(wC{G%2gc=MI=@L873I%o--k~~ zb((<(k8oqN!0%?nsYK-rXt#*vy@cD}T9WjXJ%9QzLS(^qpEvM4G&r7El%Xg{px$@oU zM4@#5>=4?CKNJATX|0be*83bPrC$VXfW)I%iBdEmGy=(d8NG^d{;BV2xp`ZH&XjCYk5h!UDo~k zSuAlr(R3>Oa3sT8ckjDzxM=GetMbZ{R7^Yw!^Jq*z!Upj`Jj$~U|Lv*77_;{8 zT*dAiSmpE?FTz%fA2%w-U0DA^ENJ&F-`pAHYs2<=$7*p?alJnE9Hf`SIQVgoiODQa zgs2B%WFR@<9_lra(?pZtph|7e)xI71rv2Eo{W6G|k&}-!uGoRPlWtr_X`eD0!4V;% zgJ_6DR4g%cl90y+f3et}Rhh~ts8HM}HIlRz7hkK8o|<<>wLC3A*m#XS%UWd@^Y-7|OMT;~?0*bor&5Ilqy~l8SlD>U1!6F_7bzM|#eo%y*eY3o zjeRnhb5pFE^K|S|AKw|~XEIpycQ4LsPXvMmZ5r(clX=5N+!oP@(r5ftxZSW}>YwUb z8JD-=0e8ITS2)fpk(h280@$%hxf>rv`%!6qu6xJ1yZ--u9AlOLu`_{BM?;JJGZgX( zXvt?xu^b5WSfX0Mi>g^D;;#a75@2mQRQW1?;+q12V46ro=S`Q4$xIrL>61PJRcByMmW&d6 ze#G%kGNed;ZwtyECgO3$vLN2gOnIwuRf8?(Jr`3^ss%ea*G33Mrd|?hWBoY~mMulc z42aEiu+@E(EvzMoQ4&_>muvCeykW3(U8p}=Bbf|zd0Z4b&WM)%dErcs@{~i|;N=uTb|!AgBG8cT zUYz)=TZ6Ik(+AtSx|=Oc;qT8bx2s8Ow2NsUL6I_qA7(mL}u1Z4uBlE z;nleqT<@!Q=fOYzt2E+nMK%*ayAAX4>-TK}X<+QsP}%QWlzv1r>y5qb8+x>zW(kws z4>j0tW{~q}QVse`t_AWl_EMR&WJbOcV;Z3#4Q~_HqL!btH=c21vqB zt7^Zj(rM)0cDWe$s0wZ8yWgJQMNF``TG?3nV)hEqk0rp&7-^z@cj(=G{I}jz`}y@X z=Lu}S_$GTA_n)Mi=N<{Pmf>iR zO2TVuiUbHgs!94u=}kUlpn}X$Jg(*Rn{-(6u;f2_$&Y2%O2~3VSSb zLz-a~cuC$}D@QFs&Ky6a>Yj~HI4F5UHR;qzu}c+F5P^~lDkLPD6$#OqtrguK9*V;H z*om*}H}6@7nD;DB7bl;$X99-?)oYM*>Q5b9 zLL7y2jF*@vtuI=+ola2+#u}QOHqAhD7}aj=}QSBs-I2qrvoSxO<5RGV;65~`6}`j-QE;OkgshQggaWPuUi z?O1E&FeD^{5XE;O#jJmZQ~I;Cm@2Po7IgOKa4#a z(p;jRw^-$Yk^LUQ<{XeJ3LH#(vVYt9E47_wD`#noz9*v^fV!tI+6<6r8_Yo6ze zhx(6+_rUbO-dv`xT6#qa7|u#Gwk&=&uX~uK$zcQ&cFatx=@9DQu!p z?gt?2JTVf_i`c;@NB8oC7n7R`Y7h1x${cO!-e2gOjJ4&6M4!b8dD0z7&xPy%gn2TR$JZ87mmfW)s6#&7 zdF$Xw6o+K}@g~aXTh1fzz1p10a=7{YMPk_^@}i06Q_D;My%Io5c}cF0_pSxWU2utl zqo+z<^ISwTa(BrL!unMklgW?W`>>qwNxBMc`@eC|TDUw0jEndyVc#lBx!QO7HvkB_ z-s+V!OiUZ#jYldwua}=(uB0wtPSH@d7)PTYy4Y}23kC4D!Eeu{eOjox+*B+F#q;R5 z8@v?m^cx}YxHZh=3=fBM{FR-v?WZyj;uD(6sj-x)&DD-Fc%!h?@?NNJ_E&L6u@!dn zxdyZDt%}iSH%FP{V3ITOZ?MvsQoBIzuSf^&H*+D%W{s+!L@06_;T&xyX9)q8zp#gQ zG-0W3cY$@bee!?oLAtsGnmW{(u*cx3$O3ksgO_FY?6MB3ohl+TEs44^La7!xlCUKh zcZ0b=C&EEUhbCzg<>iaiP5z5kxFfOm2i1F>$1r%Er!0245;?q={1Xd^OZ1TQu>E}K z$SBhCKfAlr^PsCN$5?{VGBmGGL%?A06;vx#^C^tRaeug5 zg7j{8*Cv2`HVvvEV#1z7B}RuzvilzuB4pQk9}ai&~8BRL*ZM5ern3inh-7 zX#D#|LGtZfJJsQSya7g>rF(q1cKIBulf~WSy#1d0?BHMwXv%;)CK)A2r;||ASl-x2 zEL-Dl-S;}QbHh1(s`wm~ijFR|lxqSh=3GQisS)R|(naj!wb`g%sS98C z@Gp@J(W-hy8X_;=S^d*b+8+ys`m(}+YNb z^gt6XuuI&m{sdoo_qHQX=uF+;ov7Kb{o+G3*ri8Rf(uB%g?LdZb_{r1*F4{1*`r2N z%oxPFf8^NL7Ck`m6*wYqFC+fU{l+r00>;SVI%B59-~kwBE~`z z15yc6%uXlF>2$kjrCSx9M#*AE-w1|9Jh$aCt%oZsK*Wt)dbtINN*wX|qa@B4 z5Kt{-;VDXznN<$!0A?~N6Swt~qd&IH0+ga!^K6=~jY*Ai-Jl7#Co#ankFhg>#${KX zw8vq_zbNlF6%q(4x|E=zAXr~m8#PFsmk6aH=v2^alT^C|r~ zSD6eyMA(m`nD3)taL}#!s%exbnX6I)b+=;PrUddZbf$uaAB2Xjkd!Lay(6k$jRm&P zn7aftG*(L5PRAi%{rUrc|HVLZ4_8s_S2cHc{f!QfvF?fkdujcE8ya;$>;TNd-93oZ zY=zQQnmmF-ge!m3z$784lbw$~hsVL_;%&pPKj1iB-VTE|b|dX;O^6Vr#48G#@(&lZ zf0|$a$y2>|S*2KSFK_JKwd$02L7lz5Ut?GUuj-CXzK&2c^vLS<$xMS7n+3AW&@*mq zUgB#Wu-I->l=AfPXnWJdSX-vUXw|zS>XpR6>l!=sw`F-u%xik=fM+o}wju7B5mqFL zOC#`{nF0!wP|iv4_=3Q@H4f_p7|ei{tWcy0a%0@LIiT6MREG_i1zVT^0(3Cy2?+IK z?qa8|v&KVk275E3jW+-Ipv6rFAd{+ zrUfv|uWARayAIlfqI#nqcL<0H73wKPiE`FL9HXsR3`mtmW2kN&8N`^YhUo;|9`~BO z#$ZQ;W|T5mghg+~p7Q(d)FXk;T#VL2Scd6IiKAZ27bmFZg&evqnHK!k&cUiqz`uFw zjYl=bOW^ncB&4(S`N?Zn@vKeh{MC~RRlfwGV*%O9KsceiSkiW@Tj3Z!z>>FKq>iRw zLD(WF9}(`?zvU6!g<+^|)SEz70o4qhNQ=@`6@<;Yh9*Gb+ukiM`##AM)GTogquE$5 z%NX<{?`ATi?me$0$)m0VH`PD|Ft+)_x>wmS`t;~=tIHe@Pjo-TU!1HXxpI2kxUp0- z_fFsQS)$A|{3KCoCf^AY3Rh`%`YWv7NlWXuH8(?#u+R^;Ob66N*ifHT#1g9ruc{dO zorGO)clp7rILK`$8TQgCZAcddE`0*UKcJ4m19HjK`=PfJvdI&{0vSW-Qp7nIC2f;K zb>EtUZ@}Yy$c0J;%?-&MR^aA&&TL;NkMskM;aAaGEyN-WSv9FZkU{yRekk0*h(eLJ&5j)4fU&pHIVjdg zW@l}$HV6%uc%;8B@F*nU9H)Gw3SxGpa3uIrQHhMRQVuicTz0yQ0UPRzjw}DT)a+vB zuq6=ZIsJa@jKro@5IiM%;ZT})#SjpbX2sRNykpQl4t3uBo~=@hTCRiA5hZq?#i z!L4qS?rVF~lGIW{7ffq0?XBY?iT@q%q4&XqLU0HOmDUKC(7U}2h42pJx7nKEpAEh+ zlfQ#C@-ry}n`Q4D>uf;?%KSqV@9~&eH`k*ui=ibV?VqFN=WrgdX)H2syN?V+?SwD( zZ(^=S^*AkGp4}kSMF!uui}vIjAkFB?$y8re{#%>Jj^XzxFY!B6{-I|LJO&?b0tr6% zBB$*}=ulK1`w2KTUC;a!(G<-a|d!wl5r3s|6QF9)Vd4 z%SZE%K9|p_zM5Td?erLbwb9e>C6bZZT+ z4zmzf(qqr4)5YLB@4G6>?EI zs*x(pJj~B9L^-3QJS@gLkWfzrQ7e>RR9*I|Eii($Y(z>t1p%F$=kvrqRKN;1ni1D5 z(JlG21xjS6uR01Z{pNTt>`J4JS2&lIkA;+gL`^EPuN~OVgygKH`h;fJSNp z&t(pE2rTX8k2A`9;Lvr@7ce<#htIXQ`Jh9ZPGB5LRqQJ5xZqazWy|zi!yqcxTW*6k z}4X4W*_I<{?dV%xTD+r}MpV%wg0V%xTD zb7Gvl&v$;q?p@ur>Z-N!Xjm|qkYr#Gdb_A1|KLd0@W*uGpxJ>*5a|KSky;axDgy zUbgcbGoCC5%L`*5sz!A6nf>Mio`*85s;*x=AH7%HXuz%`1ca^Y@YvphFob8!Am|EY z(dZ0@NefT>7L*XP>%5KgYIWeQZ$>}(!v9Mrc|>(D?kZ&E4=m+xG{{huW7Hu%2E8(w z)$u%)V@{Y<&sA@9->xEho+rc0h(}B!D2pG><^tPz5?L?dT_)xsG1b5hFIh_h61dP zN|4?w65h^YJr3^MM^GqQgB+zgeuQ$Cqxjct%SO9YtQTij(AA7E^To1)Q@FG%7f-bj zkQqu=P?@Eo78LBgMzDJn0TY(xc|q`d&e1qsBEJ_EX=4})RicyRyCMk31wFDHHc6vf zc|u8fH?>gpUeLE@VOs=~3ipZsOaK*wp#XCy(sjXqQ&nl0;leDxzx{lH{B_tPveK5E zP6{a?*u^Zxk&k+Xgx92h-ScW$o7MpLY18#BFg9fv=Hjg|UnrR)hd*J?MNF-ZtUXge zBIA1)(+4j#-G0S|Yb` z1zF9VK;?;y#E#}#N|ED(2Z;dwrPjY|2-|S%YS^xsCh@fW3LC1Va*TRTOZgKtyvA@+kV?~rq&k}DwhRhK;E*(L7e%;_vSLi zAMd&pO?|b-9=uLz)lmn%BcSB%;>qDUOb04hM1B{cSd%ODSHZp+8<|`T)~dta^~+rf z;Q-|owl#;TsMJ=d=7C{Z5!uXC5Wz6>S31g*>-OCg{}Q2= z14Rq94x0I8X7o#edjU33e#n*ORBk2>o~87{SvZt5@qS_)1ju8Hv>+)^yX&@S^)V8N zP!LgsA_*OW&>%Dt^cFY{$Qc!EYS#6+t3PJ%{6HM36TLURM@r%+8t!IMfJ zl*9ud!ftiO&4bXH> zVWAj*qHCTF9uK)4M~;%8Y7_LoY7?5^rLe1bBBs<`cjE2AY-dUWj(QLQMoQy(B$FF> z5XTAi?QypBa;M-QiBr!(AuA~PD^z$?wR<)Zh`bA0riEw8VAhJWy1C&NUIRvvZQZ@D z(SIK<2uNa4x&{}=5DyAMvsjC}ahHr{$NhV8tOB<`NAC!EH8$HB0UlQ{7|9lVBRQ2a zrcf05CzFrP&jlR;l_G(e-4$CHCvlKbt0RQBd<6u=*_~vfPwE|CW;uM6KOY4O)=BA% z`@+UtoCTF;AcyrNv#d)Bn8@aCJ?xOm2R4wU{X3U1UzHxT3xIWcGxQaXAB)&|q{jK& zSj6P?@w0uIkJh;FXlzmLRSPcUg}&U1d)prhwg5YLM+q3ChW~4y?P5XAIDiULLIM;b zD!_+`Zmdhxm$?mLug@H?BQW19fWa}lv?SP&n~&{DQYFL40wy9B6!WMRM1GH>q_dzi zFm-vry?mBV$%Tam>9h%W;^RqPs1cIj^WaiG2?CmKtg91+0hX)Hqf>Lnmsz$C}ppEhFt=Xp>B&I5m8_x3eX)}W zl%W9hL+4hrfl+7-yNn-8N3`?R$gMQyVc&5_tCFp?K;rRxS3{b=TPv=LrV;5FjA2n- z|7dP6?8@Q|3s>~CpbQA)sa&6Rce2=16&Wb)?i4VyYSLR`M0%;r*!O& zBrL=k-Vd#*7cRcpgc`$+UL(>W3F;Wl%5K&Ho>WW< znz=C4Dm?;b%G-;-fyjmHUYKhg0pg;Qhft3((8+@~1|9;%Y|{`U)^sQTN`ztpI?u8aI!El0}WGhRPN4Uk7zSU*h{%J)KS}_~qQ5rNv*>WN zDjusG>>u_0m#E4E%e6hod%+B;KCM7?IJk*2hcZhJ)m;8Ev|yidF~tMc%(@)gyE9Gm z_rZ#TuSR1R5K|N~AuUFHoGcEaGrXv8uZ>X84V&C3E|#*$ zTNLj+>n5^vL1movRw~QfvE%bP3Vz!&PN>-chQp4^5sDBQ>$-E1@~#{+u0oGmyWKRz zr@c7DrFu&-&|pwxh)>awbWb$%#Fu3<_? zWoD-e{OW02`M?m(Tc+$%4#p7GtD$YOl-pF5o9vP@VM@!>7b|X@K+&)-O+Ij@1lZAI zrc|KUhJU}XD8Os8e%6A4dcmiifk^%6a_VPD2;qv0=vWOz9^br#=8oI}N)y7B+{lapQJIk~0kj@oiT&Dim^( zBD5`Wa+|rK=M6|TnGU?xo*>AFTG{mzx}uF45G7Gf%U6FrD%+6 zE`*C2Ztin*xhu@!lgRbsOX^Z|TcY4X#ta;n*U{`Xx<2XvYTn!0j3er;ar=L< z@5R^7{d3!L3phkSkCV}GMsv%vX(p&Aw1k>n;qe@1>mbri7Toh6+LLRfbu?3fV^t%f zB!#7?yW*(mp=Wz*llN9|PjRt&Ur%ykqILkp2m{>MMT#R!UCQcdX_w_}^5agC<7O)JbcpRY4jnp0Fs^$tfKhC2fRthC4v8Ih|X94&M&E3@DO1VZc<&M+@h# zxeBY_hiF|rK_g}AS0$n(s-vFZu>!-Ec@RY#vq>z#>8^<**!|_EGrCZ9h&fOkReAHM zUMn}T(6+GinXs~(X-(}HZzo@w*?{*uY6# z2m?$ofBt&0B)n!uSpYBwtzAzg%gs;bp1p*`^adsPGi#GJA<>8!JHH;aEEgEHW}z00 zIT`I$M5mHWD6dx+PRN|1;QBQlvmg6~4Fa!GQzLwE-djzMaqJYMWmxt3hJ7UHTdPY% zFu}3t(dxyknHh&<4*N5@J|Vdlm$tG>*zgO;V!lH?&@ZaTih6TU>j(D%4S{#1&P(hv z!_9OAVggME)}d^4BT7{aU>q{A-s8k!YFW%poAP*&iII7dmUA$_p~rYABfT|MVq@(h zM%0;s&kLNFv0%WE6``~1GBv>1gmM*{<1Awew#LyW(i`2qWVk$Sn zspo33(%+_d>~CYXA!hLc!`ahZljaat(C#$#+hO70WHLZJqIp1FP z&tX}8Ab$S2=E2br4i)$^bHK~s71D#p!4pAoKQ_m$hJe+ASJyE>>B!b{j(-lM_33RU(^_)@%}3Y|?pB;h{Rw_l!ZG{v6fe@bWZ%rRi-% zI+}Rl;ZOS0u_lnEREuVLpciFj?dv$)7EaBKyy8;*Q#J}kZ-)P$7J?43tHNR53(Sxg zI>uIbC374m`OhkL^-Uo4ICS3Ywdn<@(3X&#I7bLUUpA7J(QO>8^M;OL1e=l|<{5Bb z`LaRRVlp{+7s11W%$E5_HvvoCm}%4BvO_a-c&n`Q_oZ{?cG*YjqfhKkJi%B2#Orb{ z_gA!$;he#}x>Li(B#$?JFo{o{*v;{Kt4Th9Z5!@5TgwX7r5B8D*y6W45D>roS-?@_ ziW^lmN?Xx>&FP;fY)H*iQgb^LXgc_H@d~S8XYjjHd)7`MhOXO`m)$J=Aiz^PsbC=0 z@1ZzT^U=rh)Zjz+A+3z#ocVPS@4F3Isp>4^Y{m>ZF{+e!S0OT?bmuV%^4r;+h`T6> zx7(@j$K59h+|wd8>#Lma}vPTjj9l6Z=kyyBZoURzhGm%w-*#jg8Hr@br+eM9KiEsF4c6dxigNnfk&FD# zk#7GMe!K1Gg}E6+T~#+{ygeOVPg}*f^3lTAh}X+|3BqnB8!(Z_0y@KI8tp4ZX3bx6 z!s1YXfK&(G^!7BY&zxc)N@8EHZC|6<34^?sX^ryCasxU*U9Y{Sh_p@22}6zz%^ zk5u;FWvB~6fR}kDmW%DxPsGpg#UD0oM0tTJ7@~LFzKOXR)NgBBDEp+gFww)V`e%k5CK#n6q}Nf2WLGc4+tn$$0KD!}(?2rSW&+nCWq z<#!(EQ?ey))2#Z7ttnDHW3iGrFFA}XEu|8YKPZF^#mCC-YH~x0Fk-=fkC5$lkBqgT zB!o84n^gHq;#P@Vwsde4oiJ=(#b5k>xT!1)cS1nS3=!IDDIfqBBVCh8v)259;I43Wp@fu05l&LNIzGc2%CePdi{Fj zK$A3bq{OVN6Cct0>j4qFMZE^i-e~6QEE7=xaG_;P0mYRsz!4oq^{od&4KGYO*tE$( zVmV%B(gA{CK3FX!-n6J+3iLY3ac(+Jc$=Bjd014kpi;*!aUoX}F=cZ~zZ54@xf;e5+lqQ} zOzCJ1q~sS3ESIQK1Q4AnDB_gdmZdj&=W7h+zMg|GG{$6JXb2V#6O+E$-cOET49tN| zM{J>fy*ofB9HSPWFX3)E8+O1qQ668hz?PgmzXkiC{ig-V@qga`N%lLv0EdXo*pu0t zAtLUE9`h6b4L~6(?93?6>H9lrN|7XAwhv_BJJ|q|N-rXqa~c~OddO9m4jxcv6kbB#KBW z5@PN!@G@(r-u|X zcL8mmBwX=t<-PVPNZs0r_K#BhusNBf@8veTlN$I1%=gQwNwmc@W^R~^=WqLBN3bS#Y`eAR6?e|Y6;F+5U zJFv=yAp{dJ&*L*TG_aM}4}%-?pITpe_4@~le#wlJVMvd~#AjD8&RDFCIn>5}R_+42 zc-|o%qIKzuL0csn7Js{@=MX36@)WGl6;`+b@^k()xJ{U-={pybVYGi!UECwJW!;OK z!~}DWXA#Q;jg8yuK5yR-8lPjF4zemcP&~u9zh6@J(%XwMF!xcLu_~ zx;^3XdEE_jB|7xG!oq^o&#~Y!i_Yj3iI%0K7ByM9di%r0LI#S$SS_s7gMdsNAT=X0 zR#VU>iGw@XdkaFiWR4oFhwHSeB(ZBk=t(2bcjxG;P6MSw2Rx!w3UOw@L>U!&+PWJ( zk;}wamjDt-srX9vC6y=Tn`Rh6g*@h@bmt2e7%ukn|9)=sf4p5os8`z>2Z+i9Yg+q6 zEVd>mkS{^xbJ1mU#X0B1zg~^3p}5(C+TJeKoPoL5Ml2lR^^!cl@EQ)R9V1u}lqTrZ zg5$%%+a9uc{ZQ1gS?iOxceqV@R8d1qfB!I^WIf>Bj=trPMkh_|kGADv8wPvO`oPb> zt2Occ4iMKfn|2a}$4nlp2p^{j3>bnEDv1t^LDv7A+;TdDJuy_C5}}bD{IF1T+K?I4 z!ae@ulFxB7W`%{jw%i2D{`U6n^bJ5XbaX`GSI$x+0mHjdGZ|rCNxvv)#R71-9rCn7WpZBJD>uA0?&sg<*qwhKZkpJ{<0U{T z(qf!|3H6#Gi`ozVny)uRr5g1;h3tUxZA6Arv0v7qi%?ARe`{2_&|D0K0-(HKm2(XB zpIp{++HZan8v8=JQR-%Mb{v<7LIR6!p+8NxMBV^(n-iFb?pIWeg95Wk59^%sK!vj* zVdFzpUWc&>j1 zy8c|_|6gp*e~L}|?>L2Q0eA);YdmZ6d6q_OT$F?O-7vQLiqip7vs&6;Fk+YNp4BJ%%mT1AK$!Dz~JpWCJlpS#{d zZf{(=#y`LxL;(^v#X)q!pYd%5Vz|7k7ZF<;hAM!|F-*tP#Jf=Ud-7#Jdh8z@!tLknxKHgWI7l4&Ps7<+O4IBn0iQHa%+7kr{1q%fS@f!ww`M4du?y@TO>IT@Mji73lt( z+teJi1z9Zj}INIGMQ zUWM!QebM!UP}%;cr2g2P{O~JbOX46uwPlcx`}XKWek4~#l18wz;&+R@!Im(H2NGfW zL<%WIJ6kUVVp={UIsSFuxXhy z6LnECm5sgd&4ricE~QIlp_(Tb^{gfX&3CGun~}}KRBr3KL{dCz7(gUXQWOZGcqveD z`CUsmvpa)^E;0USMjUtVqmh^PeSyitj(};q)_Z9^KrPS ze`!ff9vwV1O|cydCIGXtwkBlQ#?~sFj5TYo>||V6b?5>Oa7FLLWsuCLe8A~jOp#N% z(ezNhrejABba8kZT{iF-a1~mj*`*-aK$JNJkkCX&%HeH8Rx-naZrk&04>fd_r9KYo z1+_QKDA&r??9_vSE}Qnk*klIFq7$rzcieptLL`VQmogW*QoXuwpIoM{;i@a|8{yj5 z@?^zEC(%<1g&fRr5ITsn-DSOz+Cj(d#Nmr&5}szV3D@0UENB--+rT>R_WIB99O2&` zPotH}+vu9>J$pH$P(C$VFqj)O_Ho(c0bF4BiG3kFk5^~k%j^ozDW`)ur9^KKCLgA> zy6gLd~{iNk2x;}#9L zgr^8AoJtbQpMoA)!-HP18}(0SwB8pH4n53pA?M($i6S#qn^5ytDz|!f)>ipjrTpy4 zU$orisfjFoU{oP92+<_w(U22nLFlv|U(@Tp=ke3(FAEzCU{+{%!R$h!7$Os^Zw9h! zQxq5VsZ?u5q*%uuP@TlKI6JJpLYTD(EFu4RV&_9`6A&!-X?e6{!Tac2gsH0? zNO>o5hYA)XuUpHZ^eg2y+a)*bqT2^t|N9trZi6Rac$fez<@Y2N2FKn=!3(l{?#mqf zTsx9Tr%jpBS!o51J&}ZP2bh<$XU%*1=Orf#7v$QN@vh=^Jr@Fv01-zlpPtZ@Gep0& zh5R>~bG)9cHAnuY)AZWNr{tBYN@=|Eyx{H#NrU9gc@yVZvB}+tO;3o1t|Pbg(qBi_ zh)2PPKtHn91P}}760eCX<`s>f1X6|A{=lq={oW6QyU2%J?0Ro3gy_RR4{KdLa5O>4!n9HRmj1Z zKk39D{H)KU9qVmSOGuy={5%h<#&C5r$J(oyV!;Uzio4TouVaM4bWN--jNf~5V`fE% z89pGOB0uuXLycSyi<*|8qC5}t$Vt-#(>RPO;UqBIWsogB7$55V4D0hhfy@SAxwI{@%K2-;n1Z$`(>4-MBHNm79lsL-7Z7XARK~J@4eDs7{#Bd zUD(KP#pItW=SOj2HKsJ=y1gxEDBYc+loO@9RisF06+B01O^t*@I^mxxHm&*w#RuGWDQHaACKGq*|aXAUmHlUC}U`j`a){LSXQC9;b^dz8^(T`ZzA zI90JYWYh&24p|9G%zo%^l$bBMN*2C`i&`Clm_V!k6hdv70car#&5Ks-7omxto?#F2 z?0k54o)o4TEkBM42bXLUeex(w2BXOJMz^`h@!UL#>u9jJ-G}CeolHc-=m%L-3u2A5 z?h9ny#q@ZX+#L{84FzB-mcxaXiWchCOW5bdX)5mPdD2=amubWT2fR7L_m>pLuqX8z zX`Mc>Mx6W(7ljV<_weAnE7+jO23e?riLR}@|81= zn@e}5)tkNCOOFn4Xfs}5{fGpM$9qApqMqt z(?~ns{lmSD`hMEKV8{slv``J{r8M}5(fCPsUl`{2;;jF4#Gz0H=Itld*_qf5u&54t zkQX3;WY`2<@qJEvS;VXb_HI|o;Rcsl3|c8%tV>o1p0`lu!I$na3fORy%A=T3+HkG~ zs}u=F*9YW0;Y_T=n7AjBo0mpre&9hf|FsYwg36-24^hH$uz9v<)C{DIbj)G~!GMS? zuaGvq42(O{c|ADOzrBIk^-G3f{6ao}tB((~4Cbr49Nm)_INc-8Guqxy zLq8>ZpS(B#9MG69hk)hM_ZjEsqI>c^nNiMv`=v**`ZL!Ymy# zkz3HI&$oSZmklqt52IX*O{?mc?#N z6pfUS8XyT68+gT65b9*?Kmfe075{RJl_DqRO%6%~QsyQ16=bL)gKOE?#D2N`?!@5u ze8j=?m>;vOS_i5{FWgi@-cRxqt}x0VET6F8&sb7F5|#Xk9qu)Dq8=nJ&gj)7?8N1& z-8-W%f-v?~ZnKxBg`k`bsa>y3_ps^tClqyMmcu-;IPTXjcpk(=jYiZkV;bc+Lf%tk z)iIy`&trMNIUEN|Fu)p{>K!>Tgx(|+!lVKae;R_TdX@Qtu9VW%!TG;;JDCOOqE5_*oj^4 z7N~-)TBNNr`9z5Y80uR7@OX*Ozf9;fDec&hXt_Pig*3aqaF?MNSad3=bKd9Gaqqng zV%Pg!Sn+ck<;>&eV)O5ddkZi<%hkANdz{JT`Tu$PJ{{Z))rVgOlSLxW|E~KLiN#62 zKga`8Ai`p*dI4V=FL{Q(Jjjb-wSK;6!I|NFZ9K%?c^$W8%ODv~+d5>nCJiWrIP zcpbxj#3dd1>E_b2q^Jt_1g~I3g{MCI!-mvwI(likRqOR@EpV{PD5W3w6DDcbhpJT;Vof*PZ)EoSSO#@^+cqJO zk`9HRqcdhM&=e_?k;(I4i>5fwpbb#}oUD6iFfY!1AR*Owf2Hal^(6!a1@-*6Q`2m+ zMh_d%e=T4z8fvGHCL&eMKD997itOoj+`GQ%c@A^Hyo|Xr9Q)^T(p&q91C4%&js%|Dqf`JG|zQ9m-{OFoIS*cv@eR(M`q=tuN?eAsyIP3VVgUvu|w^CxrLkR+2! zYC|O|wu?_{xMr3A!k#x`I&qmeQ*5`-Lcg%(fIu)#u(`_ix-PV?jwi;HTHCLO;E6`e zIc##DSj~owb-WXhM!q0-c7AEZ(S4fRdp$L%hpY9Qin`08;indaH$I3z5JixFOA12O zVQ)D|JQ40zk`xdI;t*nypH(6r77tR`nh#Z}(GIn-*N$QX4W>%LEr$@LjEjhlgDP&> zo2Wy*GCfoCOb8u(wf&H6_P*gxGmnF~a89SnL@@Jv3|*L5%4X`l7{?9obaHMv96`ZS zTApO&sRC-r!Wd^4)mabnizUrwx15m4O+;xdDZ=}LUijsd;|N(i9u9Y9v)S$K23Iz7 zYD-iF$7D;-URvt`0LpJ%aaFkEidQmq1A_k~b~@kxj6Ktb{~>Oyi``f?M{wn{f>EEa zNP#4bY3KN;1?@U^*-erJ&==Zj{~9wP8^UY1(dHK90=Lr*KH^{zK}_7P_Kr>r3tK7y zAui26@7(s1%6zjv+*;&vZpPr;xgP{+^xb&ENjVRWg0fMmvwC7FeRqSGD`^^a;y(N#k%9|<#o`8YbS}DBm30ezH5HX+G4jsXjn$>gA|Xhb zH%oxab-s@^fjz0k#!izm&gPtccibW}wbGw>jAtXp5x1U~2mxn9a$q#-GG*TEY?JY8 zw9VU!Obh{22;+!*<&23N7Zx8y)#etLZV?jlcJ5g}#G~^%G@a9^svuTnH38V)nVHAb zO=NX8;4BCtd~w-QidcLY>0A#NEpxds$@#)lez+@s0y5?RG@D(h*) zZ-M62L9{&LQtcP&Z{R^zDx)~m@cc)(@j8~~@Qix7IzIqrD!tb4z&>EO@EnghO2$

kRwX4psLdE^t4E*~=eLp&_ex#QSl|zz-&YM&6A1uLRQtr&+ zYM!CLVrMu9NUPiZ&*2FX6|4o|E^*{8hVTO?P50FCNO)WO1#$Vbj2@FTgx(%cWpCW)#HP; zJ^VF0b@D&5n5(1d+y-&1Y8aGEALp9+9xOri2TrfwU4*s1PuS(^zKz-mXuq4Xtso9> z)ejYS7QnsF>R^9-`10~YBf!8TOOz1msyagBKD%&xJ~as0T~W6PZe_ZG{!krKD^8+- z^pm}Z#>9aTIg9Z>)Wq-zR$U#xQT*XeDoqLhRVTdhg?&JCA4__zJ~K1EVfCShOXgJC z0LZqYmvKwlc%}PSBq!4XvD&-rZd-2Wg)cj@0K}oNjD}@F+t4u$VC@1!?QQI3O6Jc%kaeMW?_ZlvTmQOS# zXJ9t8PfP_N( z?f?gx{Y}aBc~Wa(r)!^3pv$6lgO%iR_?42J>KjLj8Jcv83UBOTndI81-edZ_2f@Py zhNi0S=|2g)JXh!~U}A zBWGEt@{U?X7dGD7l^BCzXArMKeL^_r>xt)PWJKVb?c-?1e`K}dAR&3el1(SEEy3$< z#V-?1i?~3hUXCZj_D++T{c;#PExYHI#c%*`Xx{st$;i*Dn_v8#ZnT-pFrkto{`|LP zx+pC1xwNbIuxrzTX?v5pt6WjKOX|~-Q=rO@ETEbpaU7kTqq-N52C*+qKz zKSpr@@D?40)=uu@%gq}(#aegru?SifsnuRp{u8uKpcEtw_kqJR_D%?T8&_oqi|FSlRACke^Rt8q?hk^ zuL`j6GkTEs^`B=~R^rqu)e*D0#k=PE+Z?_)hop^*vm_9Q$s!XutzvJg0H;GUoCh)* ze&<_56Rxr39MkXt<4XbG(qvd(BFGgOPdlU(0|}b7_u9`a4HPWTtuC)_xfKk;9e9 z6Lj@nP|G(3-neQ zD~<3=v4m=6emDK#g8VwDhHfjzQx;CX5V3BM($JccSK6EnqHQsuvv*(Y&hH6xP~3Cf z84-;Yb6SIDkgN_46V9qK2c#Ll4az{n!`EgJ?V!2_)wjNS3H-NM0l%BXJ?u@r#Y9|8 zJTR~xiNwkNQI`4n)1Rj!_s!;rKkt83P=E8Zc)F#6QPG;;Q)b)A6$dP0)KRUH^-fz6 zZn&>ElfADYkWw<#N#Y~CsJpaI(R2WsRu}(Ox@9ks+j*loNi~-Bc>_istTeFn5en~) zNRILso5*A)6ZDPuLDcO#p?+jhEH_LS%}N2DlJ#n4zMc=%l_>RFi5?)K7tRLOBSUq3 z5|0|H1@3)oIcSPeW4~XqN1|rjl>XAO8qCg0KLAG+)HfK3o2@a8O#*|_a%xniW zf7s*Zy6>SzJ2-D^JuVQb9EvVR5=e&(W?fo30=23CPmTz@>JcJ}jp!F=yKtd#rgX6w zw4dmGxESam;-9a2C1$pZ>^s8oGjkV)qA|nT#|PKUb_}OftLfD2_v+P|tbxD3j@$)m zX+%=CngP12OTZv&yw*SS`KKd{_$>qQKi8R2Wwb9y=``4CE3))77?m(3?&zNc+2bvKJZpmH#0@xmQd!?fiIESv0(sVQ>H zAcejL+fJPbq~=xYMM5{~SOs&I`TCf@xdx2?9aG6MT@YhI4WxbjYvys_x~*ElW89ot zQ=@Hjs#O3g=@6$X7n<3AN48sU0Al-D!Zp?ETcxs1`>T`seDmEok@jJu*Rv9NK3Z0V zV8*~{i^uQZ2Nzx8U7o@V7KqPGH1Bur;bRfX)3D1IqEM3Z_!Ex_TO?_^lFtmvASigp z2t;7eM6mNQ=JqmZwvx}r7y!*uny8HP#_USL7Uo8_0nHSrJ z{U%HF1YyX}Q%^P&mI!D9(D}X~>b)G{16FmPk&>y4A|vy;`OA$O zPtlUGO?x5u9)ESomyq@%=wo)2yW6#mq_>B=wr@#|&2h88Zn|vDf93eKxivM=5e&Rv zF52~$u$LV!&p|laaKy2Ylqr-N(4!1^I1#b((RZz24+sNPUxyG*LGX5H&UOKJpziMTM&D{Ao;MQ#=I zI_^~MK$}_@{F}J~%-Y`TfLmR0)p33&y`>W&cOqyGm`r#LMCMnH98YxaPD#Bq4EA=U} z%@x;U@!uddl+Ov4zWZgL(n?epyVA8F`IzN5qf>(gK-TAJ5%sg{#_JlK^A=grE!m@0naSbYAplhv#G zsRV9Mibm$zG>0RD5I0$k&sOf#UUKOh-N<&*9%ZqBxDOy-8#ijZ>EiZx8b5B{`;Q`B z`|1+Xny2}NT0j~2l7lNV)6pMD*WF65Zp+5G2aSUEKltuI1MzupN?O>3gu8@bqEbDn6Ipetv~p_g8joLfR~VQb_U+?Fo>~M;Id> zRMze5Yo+*Rpm&(j+Qgrq5u=UAG=-aEXESBYbIJMgs)Y^&Kr5F9*^ zHfS>Ucj@Io;0rWfe-@!`i=xGG&I|XhvF-Ep(RWdY2?8g%KQniLXOB<#Sch<9Vc~t! z;bWHr00@rp&vCvi@cgB9{~pR9wEvAP#rvaap2j^EMRX`UPwPDd|EFo9KOI*6aDp30 zmvw9S9v3QeG}-x+qI(nh;{r zZe}YU=%)0s4+Xrls8jIlcGAW_b7MgwW^Ef4>C)=nJGDID#zu^`KiXcN`JRhxhJLxm z5<(_VNTNuWl;fig&pKkHsX&NkwyM~-e9}&M9bd6dTb|=V|tBRAtqX%T=I&ZbkImaEQoc(dt0jAQOK5PCr# zPiBXSH$-y&9&&@G;$x`y(2YTG=c`udITiO-JIt<-h?Et9-WBn`=pXTK0*LOmomm;VPbB{1Ujtx*4d-zC{}e-`19H8D93!NSbhjAL=P*^ls| z!)x&Yg{5ezt6v!i6Y_qms2@aNdppp+I>Y7sx_&1lSXf$mSSic%UA$E^buT36xKnu-#qE z4_GkA<1P(8gey{lb>#h96Ir{n9K1!LRyPy> zTjR=9&1|J)&5K&vNOGmLhr$IM6me3M(GKKFv&jgs`t=8%tbWhXZW@=zBD#9$Dp`%R zXMViaHIe&|+N14yG>85Rv+p^m!*C;Y@~_0+LeYp27{lNYD7l_6sK~lm>^6mEvJJs$ zlrwYLbK1QGIrz@6G@IMc9y?(J|A>4HqFdAR70!eT@{-X?SFG^PiFIhhdpL4DILAJ? z;CxtfdG#OtaSyE~1c4qp6e+sX@V{&8KxB4SU??7fzS+an~Mr7m|ZPha;B- z5RF9(Ob(pC)m|I6=924p01ufybs=b2$bdfn{1R-OdX}3udC3lOCv}N8+A&^cPaZhW zKkJ3ZHf`!6CVGhSoW>CJUTeaaTqPlW7-ngSYgMe}1$lZ}w=A&Mhp~8&bl8&f?wTxo z&1=MeVNot?es+V_8=8YxZaP!)?l8JNPELe=%#`|@EC0tEc2wbWmZ|%%WT^N7Rx-PFZhRWQ z2NN+B{v?OUe=hfw#}V0xL;JZn5MFC)k7T^Zh-nogCMRFI3Z-;?Gy~JUu+v|zP0!b% zy&5ZkBpup93I${=l=AogPE~n$^=IgzF0R2Az4ngt0&5p4KEK>&mp!%952t-QP z1c@J^QX7Z9^=t8_*S;K|c;6c_b<%ht;z~eL5R`=0iSJWV*bp;$dP(3D?}yjd%WN4Q_p4K;*Y6R}{O2az1zgyEgnPdig?}+E^Pi4NLKn)Dfkd zHsVYPjs$UU-2Kc)*l}oGhc|PPbG-ZvfYAo1$pN!0;YI^s4RL=Bq2rKqaGPf9vEM== zE>+0_y>jZXD${j1RGc9X&zX;IToeZu9iT;M!)WI+Y9ML$LF7;+aZqA~X$BgbD;7g6 z!`N`qiWK0>OarB}AmeAW2YDenqJZjQ*Xp!%gTCMQl zyYGKw+}ib<|8&mNj>TxR_<{GH>vcc)ZN%71(49t%(Q1QFD+;$G>OICgf9>7F-J-$@ zD{T8%0c;eu%iD{OoVa-AM11|$*`03sd-*3LIXx0xIwnGtA%TB-HaLI%Q$3C z4Xrql=>)}I=2RP>)(t(eU$|YWGyj|oTcOMua+OrOk{z`w4n1HYPCxNzG-3~T-1;wg z%{pr1#~}(yK#v>*TIrPHlDY*ci-X11vp#70w)La4nXr8f!AVy$jpL@AdNU{~2Sb$Q z#6}ojDg9uw$5_swX=--d)3O+wMt|fO5SpP|ow;OrX~?L7|7RxDaVOpU;VpRn`HV~Ibb?d(jhn5fI*;Lk$w>}sb);y$a`92fml z3_NKH-_d3M*{8+vrthENoQS8=p3#fBokUeCKP< zntVcEUwnVq8;P`(Tcy@N0&TFF;H^Jhi>F_)0&9C>5!06Z2k*&9>zP;-N=O6MNtIQ6b$oMCOSEG`sQ5>vQNgeYktsE$G3<7@Gp zXP<}9{qIGXHf1~-l=_fXeo0TGTsEt}Lz=ZBEsaK*GNV$aRH{9SV~i;WSknMr^7A#g z>iTu)=?_t^l70vAHGIS#@kofNK~x||kHPAZGmb;uYohb2ENTobd7Ew8;WY>5^expY z$z0extq-SZTQ;ZAEli#*!wPLG9`Z77W&DY|+Zq$sThGK*Y|&EKWJ~vB;v#i+bq;Yt z$_989a?|3fulx02_skDEr3C7u^1fixwQdU|#9tt#yOVpxK0(;j65_$_2TNfU@H5C%f1lVF_P?xp;_%`FTvV`&O;EGW2la1D@6amASR3- zi|0S{4E+4tpU0ctbRqgS_Mx?I18SWWc=XmENxca1qNslBS9%S^7!<}hY_>q2CHU#A z-s)r7aw#7L%h5A#o332a8#2`!S1272>qn1)Y+5Qnr`M8o7Ll|S#CV}Tw@~gFhx+Q( zc>0;A;gXNM9dl<+MZM9GtEUMRh$OS0$m@`%O_g=4@w4`+tS72%67WjU2xHXB0q$Ja z#7nPTjbGl{jUbNES)qI<VwJHWbu}=CG97l3-s?{L zT`^&XIMHXybTu$AfS#Tn47aUZw^2MAb`)EKOy~tgGTn4rFIz=GjjJU%a$x+I48!D(Xj2^}N((qj*&76AyysWzaQc^*WBH~K z%}9oSh*x^yOKOn9T@%zPSGAMGol~}2>nicPVx{7}ED%49MiXHeWB+~UA)yB#%id1Vb6MeZl;L*#bnc0r|(G(vS5396v*lQESNez~F_um1ky`1?J*piIf7vM01Q5^58znsc0i zcin&qj=##lm5gYkcK*fUdMtmmPnn?ofB~*qqAb}` zEf6gad}(Lx>N49k`j+QNnc8y(rG0hw-OmS(Z<9Z1fQMe;c7PrO{^%bU$kD7E4IWU4 zNeS@e9P{XTJh&0)Sq1d01_ryNenu?yGt$^9nll7qs2>eRXm*AO=VCyOZk!o^eQg;S z!W*ledp!KaKn9wyMfZkRLakauV^aeMES!q3zxhBsecy42=_V5?nB7o6I}p-i<@9@yk6dT++ZVch@b0&s(!QlhA^x0i_ z=AJz=8;(uL+fGKz4ZpLuZ{SB$4m*E8j2asu?_8eWmBc;gc;~EmkAK#Z;fu*Ctgym% zxD~)gVI$T)|Gatk{{EeN9KYZG9imiZB1I8_h_Hjfw?w z^^@3nTX=B{-86$4eU}eG2Yi%l&CaT$d%F9bKDvZ|~!qY*qkwpHY9d%;{27bszk;h;?J_BLX+*;1Vg zKGX!ujP+YQPjXlsX&W^#;xbNj zu40+_L~xfV%*6vLl`;mq2655}v+5gD-&a52*1@CKf10{#)LS1VI3d8U=!YQ z^d?NIHN@2Px4Yv`D+K=3P2Ih}-S>=FOx}(K7dDi6$t_9Jv%7!p+CL6=Qwl4rupMs& zuu<5EwlfbLA3gBFeJ}g)D`p*ps(*91VE`TRpj=5Fg+?p$ANk#SoczV*cyxUWVJuhg z^@^m72B4Qa9YUest2v`U)2LHnMniLUDgMkv))dNQC-gUbkvWAzD)w417ax50g}C9` zFXO~hj>q82RR~&jlsh^U5J9W1@<%%y46a~tA$sLJbtPpA()7ngFuXud~=8zz`#bJcO|fCxdb!@HX|^pTop*p0ifSN!~L!gRtcrwJi)=A_Qe{aM#d9T)bk< z4zEK3V2qiVcTCPU0$BL{dg^db3ZQ#rovEI}F=FGdM&xMaxgoAGTsuu+1S{(VxUF zDK#mn-IEYzIS8c+2lL;LE??jQ(bXT1^f@ z7zc>FlAzh=zL*KzIc?nHajca_!-F$u+JI9uf5`I_zY~?oWFyeO9;k1G*XV{9 zQiusL)Q7Y*SQ?|9e-AEYL z1IATj>@}r@{bx53wqiWCzJiEA;mqXRCZmV{?yFa=o%oS=y!u9rRBNnVi1mLyANBR) zu}hf{%=xbu^`GdSzxt-(ZcSl@6}EG&U~CjNYV5Rq$0awvYoBj?`E`37Fm7!5SJ9d} zDxqGf9DPQT2A{lXBM$k@Qrz`;L$EhUkAoav{nDQVBSo;?`^Nf?|yIvPC4ad46Gn(dcR1YNMMM(7HMim`Mzj7oM!Rf zyuhpN?xJ;O?OCtrI^u>p1uX|a7(*fTe^TVa1Z$Z6ti9=gq<$O`)k`ImTM3$_3a-5D zlX&_GhojkQN`{4(hM(j*B^F1-S;=MBl7I}N$X5d&y@M^h@+YhDwZHUWF!E6?`$!^6 z*GucsN+6W^)^_NOE}ZKp@Re^j)I&EwYhy4DfL@h$)~xLbn9f79%@KgjvOL`Imz+Ui z?UZr~PI)|22G5w`wF_r(CsrHgh-P5qwM|bCVzVefNUK^{WvVuE`5SwZ<%~t{u=7+(JxcI#MsawCPuJA zg_w|ThSJV3Lc*pF9de@B7>32ewmZU~Ho&2oJ}fN7`W+h}H4R~rEDidhmH2A4imW~bz^nTS(89Oz#9Nu(p7*NBx9(m7N188pH&y!52 zZKr~@=6Hs!5nic06-LKI;gQdXfMHWvTKL0gxOyw~lw7){8eD zzX@Z?O{oXHecvOzxD;Ge+w+u{Vno|6^=?HgFHI!0rfsFRk$F z%o);#dhO#c&O=GC@damE9V?egf{{Sl8}$416OP2UuJ|}E`Rs==e#!(4u3C)>ISG}D zXq*U;uwKd)e6WQ@_{>^0U!B!wDi7O@HfY-mFTY$%_aI%ew;dLJe2}2CT*AQS_4vX^ z-h~&OeWFOQNaJ292PlgbL?trHDed}v?whK*^oulILlOWbQhhU)^{gHU@v0xM!B79$ zD|7^WoS%rV8Go2`rPzLy<&ZJXzW~>bz!rmsD#?mad zotUoIt`Bxu22LHk-+5)5X5}zAoa!Ygz)#foNPh7ey3_Yi{++~+;H&yi#4$_9#nOSv z04~lXx4@DB2r59iDj7&3CF>d_n;ru;J_c-F2K29n*VrsXK_v*m1V~L;I@H7sCCLFP zGCw(>MlR6Cpv*W8F-gnjpovUD9VDbQ(?SVNk@C2s9H4h&KNjvW4nKMKLHNM=b1O>amV~30y+QADm;HGY_W#rp{Cjy_IJd;!5aN{9}ZjK0|NNuQAyy<0{ zmqAGU6c?4Qx^&4@Z495FvK?Ij z)_BpBjIWz!I?%`1i{re2)ay0AxlS!A7#r*`V%QUQ97y`rG~8)hiV)@vkYXILo*8Bg2YocS!RDXi*qd~|L)B&G ze9i?kY%IOs^@O=j1_5!z$fzJ8K*MVS^|e6vV@NhU2&{hyN%t~%0~_Fl{qPV^Wqp(qJFFJm5*W%CY_o`Q(IpyrZ{?^jCw~n%0dLEUwysL%NzO)?g z_~{z75@0Y?LZa%0oWfMnNB}K-DbLMS{%9VKW3!aVlzxm}Tmji@xq%<|;c$l%(p*1;a zMwuHfzH71swTnQi2^Amr^@Mosk5=Q4clE(56Q~r6@DwG@%2s&o6uNqion0{AX@{wy z*cq|2-3no`op!w~6OO_G4RJ9zK$Xr#cGYswD|U@t`o0znWLS(sr>yf_%p;QlMU<)g z-wt?b7aF4lE#-LG01pR=c!61Os=K z0_eIQ$@+VcY8#+$yV@!V9u(QtG|5O%idGMh_hkj zZFmG~c*_uWKifO+@$>z&m+Tr!xrG&0*sxk59$MH_!k&HT#6|nh>-fg~>l@?NE*&_k zT=l(#;xSru{iSJeR0ZhN1|4);BWu>FQWP&Iwnn0eoSOSG+e<~z71+* zdu6iO=yBF9ri>;LX=QjoBcix(9}o70c*74@;U9Moh{%B;2!(?ed$L6Wr#1yzUbOjT z>1Nuvip|i-w7CtVL8DUE-i+V|n>dydr*lloz+luJ_B$)enAf80R*i)4$;;Rtf*Hg? z70seaX&8pNGgQqcOGeIdyw1!;$O9Cei^QwvfB(Ky$+kFmfWrrv;jr3T$b2KvqTtqnMn9Q)r zviyYZgq8?(qk4jJ4QTin+!W!c!{^~k7aoGwoHP?-D?XZGBmlv{^955wiY@c;&R&~z zT6#mYT?2`E0GWNB^|W%HH*O!>35LpjBqSVR>C$EB?(W8}Vqp~FD?hv*1C$kN2Mc*m z9~`uNMPU;p)FAla`xxh&TAMzg)*sB!qhb7zPbcY290)50y$t@Q!Q(GV}rPTaP; z;wY%pmXn|FY-%H5gXqz8My*m6t6;r-1GwoQcjDuhU4wgW|1YZ3ro!v2A+8UKT&9#{ zPx?Lb?OtYMNx_Zuk)`9=)9+H;kF{Crh48Qmo#NJMD+d_NH|0c1mIkd9YhWWrPzm7I zLyVa^13&xL;bv#aFq_#kGIg>;*OoMu(*-~>Ati3R*>OWTCV7mM-13>Zj%hCNz%F{G>zvJCe-~${`7iMW zo@GSc$s%(}AhMWvl%<527lXY>n!N!1gp{49k(MQZFg14){vcq$mPrTg_LKMD&t^5! zAe#~5HM9ecq@`F7@2NyLQ&7nnTMZeAyPdIhWRyosF!KB|IkpuH^n@r+o`iRuvlm`@ z#yspdi_!)GVMxyzSuC+M0#knaWaGCGSFMq^NB(oK-TRc(Tya1k=Q(ACq&VfbD_3tP z0JUq=X54r0eS(FutJwNYy?FTtE=MyAcQ~-I?~Ep{c=mD}J%@k|`|rTEfE)5apb;im z)$QSihsNUSTPEPKO(i5EJZszgB_&mof$<%+XKcFnhCgEjSkis>`FP;NS7P%6b9ZXR zp5H*+{XXy9bzj`6^%hoGVI#r{#ztXJIXiRTaVw6T+wt`kn+LIEN$+u$auOtz0bn?3 zl{?VxUe&}mZtupyb2_lsw6X|CsVHBBm;qveULETusz8pj&tK5D1Z)UbM>zVvBm=|= zI%-wyzt?=6eB9xfFlhpA{ny=SY~F;<@#7#e>xC54&zwARZuCOByCSTQVf7LC;;w|+ zK|fy@B26%=!~Cc)HD zJ&K|PV=EpObw_x^57*!?_YWW_NALpzSfu{H{*lALk&T=cf7iHv1ozi7jw|2kcx3J& z`bam_!X=QnfP}cX?b@22khyX$II%?!h1IuJj#iGcY;Jj5S5)f3=M<{FbMr<0Rv{1~ z^djb(76fEaXh~V(2yqkM;3jx|j|1IHfu0p2g9%9VlRyq)ZHRfPpLP0r>hhQlM@GOV z+vN-y!+fAmB8^0wVd8?e9GP^0Gn2kmho_|>Z&bd>`{U95NFt#Wq?3qDE@j_Gv!{t> z-NPBjF2Lm%9)wq&ItQ~SmW9Bs#Ddg4P2ndNQzPfT$$7+WO<0IMHhJ?lEz5T>wC(d!{BDaC z5UYon8@o*iF=JwcN7n?{I9S?V%ni@;y;8l=JozWTzxCR;y!4s7MoOC@%oFKeZ{Aty zUK4nJ3`DDt^quCNyYAOJq0YhzD{KT>0c;fZl(dDjs?n=Yp897G$*=EttbSCZH=JFr zd5CxL3)pbkU^Bsw@9M$BYa2Lozi|kZh@Eg}RC+@mk|F>JmV#9{7G}}afdJaP)6qU( zt%iCk?1x_9!&WFL%Co0W!HLHlildJ>04rCm#UuAVf=Z=?5(PF3;XkTVChwE$zRs$= z5i(_|6Lotu1I4t({t?^wYOp@vR`#^AmyiQRB7~k7!7tS?xN$T7_k-`on_vDMm7zd} z%4kp{Ws9@}i(~}`Dj=a(dUYeQ&?rG?CBTZ_7GD3uwYd47I{X^tuh5|ezF=yY$PVp* z+e_=&`nYK=4Fw)$j$`7S9CUDDk1YGbaSknC-DN=NN#*Rz3+FJO^gnmEqXlA$V7eSS zb3&$UwsgyPbEc0uJdbU`oN`l~X?38>3MBnAr)qo^-wq%L3>`=M=2;FZpLW!#8|NR9DLvvhvB`? zT8Lxz953n}Y|5-BIx~^bzcFp&b4K4$JWaHI4$AvF4{X>fAX5p08gWnu+?4DzJ#JeG zyurijHEXbX)oSc&_M_|njz^cR-r;Nw`a5vMoF>lSzaJB7a;oPaw{_d{PbS8rht5iZ znd4fRG&aWlt1IZKQ+nj>(m&7h{R!*WZ#a1IiuKoCaOMfvrA+`Ef8Ghe1|ge4Ka&2h zq0(`lch>S%JGQ>U3M*_RTLEko_LR0$_aDFZ@VT{bJ+`jDclolOlgeekoRmf{d*i{i zOFg(?NpAB1roIde$rmZl)O>^l=M)_frqYv7GC$G zHMr^C27(GXWD?X6SzFYPcDAhMXFY!Ti31QK?1Ve(`oEOu-*zRvt#R-{0#aE|p4Vls ziW_Si?UD8~Ln_G}$9|sdifX%Xl&v-gN@MtmM5HumWL2uJkLK;rAp~SpBtQ|@f%-g>!2{sjRrIq04F~FiIc@G2!a zBP!%?*D-3}<=^`)23wTVX~!B}(_?mP;M6?_F}}u9NHg0m*n>$2n_$-X2opLJ{P*z+ z`kTS_!frgz^A$Z2fu4E-%2^P` zKr2enq6`~KL&I`6+?7rKlAgDEnQ_?UizU5?ldx!+(!-Vm)SFEV)*IM+{yco-ov*>K zzIiFmIQ3+7KfWIRU>&7eO{H0l?%3J!O+V~1RU$y6%EcE8KxQFW$YxqIBSW%?| zVQ&+A?6VI(^})Ad!uU=^QE2LD7s}JlWb-H-$P-Kr3KI%^pdQ6|!_U{^hW`w}FD3AU zP}1!^9}x)!8fH;?oif#`zRU5`SC0J)? z8Ikvzb*y5JSU*-CY1R*_Tcuj%>0L#36p;}`?=yqq2%=`MjnuUXQHq;F9F#OP%HrU$;=!mbB0GehTedvSb;WP09kZYgu<+9h>`hPDBM=#!4kd$xn7dWm1=A`j{AJ zEgZmWkM2chxuG;(w%roX^P0`p#rr+|RpYP=+m0V;FGkY1A@P%ay>r+6afeq|SYd^Y z3LAmxr3x!-$Jl3{zuRM{ESzxKr8lg(@aF&YUgA$EPw;9Lp`1S&?WP9@u=mFo<15df zj#nH#Q2-v*k|*+OWU__ug;Y(&<}_*s8ki7_v=FnM10$M^L1i_XP1GtCoOSXsg2C~_ zAO8UtfAQ-GSFXmGSyKgzA*S@PTDVW8pB0P;o-)=r)v4T{ZmVd6qM94c4r@eG#)mmq z_mGMRN|ZSg&2j~o{@(|&&tCJ;Xh^Y8JqRh;AWK)bS6x*ds_8N$h%g5aoh9JH->t>3 z{?#iq0|JkL6LNUTam@n`QZ9{NoaYzBd>=~q(z;f*9G=w=m&MRdh}8fk{9IMCH7$Sb z|71H<*%>kcl}%;Jwx(T7uSbUPU~UQ0+Mq8oONFUNm#as}IbV`Mcy5tMMd1V}djSS} z8=yoR({`VXS3PGQ&Odf84&P%E##X747_C+)z``JqWvAkxS@1QZZ(*KaDWha-*ce%1 zU=51$Ib$l%oM(=8r?m4N(a(#L?_m-KCD*V1YVZ1}ZQOG6%_vVDkJ6YLl13e+n4C8vru%BeRYci4{(!Um z>N;NryhxAMY8RoQ2B6*rk5aQnEy<=3BB3gEs|i7p>r@stY1OiuAiYCe1l5uaLN-CEZ_X$xwzQ z3d5n)$)zYky=MS%E5PjCXX52g+Y`?`ZWa!kKM@^ePp}v2%?K1pDAG-<>&A)*&lEHPkRt`JKKa>j00y8u(5N;qE=3@`}i1V z9ooe5O?|jw(FFJbG4Qsj9dxU8dV%-$2?w41weEXw+$A+7ZUHH-s8-Z4gQk9fMBP4zqkq%+-+5s4<)QperXlXbXi^Dsq%iG4-O1^NM zRXmVgyvxwg7>IV3`GjGt5;wMpvr;%Nk;914K%4ZhIp~u*_qL7#^UnFU8=;|xupKHdzD{Tev zKctbeZ7`v?r}sku-hxqY9?>`Z5cPZ>wV9vrPQQEf#sU{sSYbQVyj>OTFRZX##lCpc z>M5VNe%<98)`TxBPwhb1PI7&e*wOQA_`xgZVCMKhytom-p`8SozE$~}ER%r6kI11} zbG3iD!On!C!zJQ~G(IA+`bq`WsueVw5&m?`9r()C-^KMe+zh|up*D3Y8euGA`O}Om zS}4(W_OoWZ2kF{tzf7!#lVc{g3&B!q7=?p`5vt?I;LqRt5)MCLPc#}brd?z~@zpu1 zKbd5)P6!Hh*5sm~Fq|;K*lK`pKUl}xzr7Sa8ylbuJ)$%b8Bjt@LaG`nak`|S8U|!A zN{HQ%%y4jMCtbnkzIqOgz8&~sg>K}Gx^~t~yEssgSDw>^p+aef@Sk!+0Za&jg=}Oe za|YO8Sx-{bCWVBAmW6_>h-Fc#06|r1S&+DA|62JouofO^XHeKlLXJLR0wnlJ;V3E> zv#!%n4}8X=;Hg410HM7u-5w1fN=8%KAG62whl6fZ56_4s7#0rTXYsfg3oC687Y|iJ zydrHg6XjLFsQ^_(<|GOoNl+pdR}`Yz-$1Jw1CyrUjDu$3S;y~=qxYYR!{&|^;<_}( zS}rDzQxK6J2gKJ9080)ThX=Um@2OC5{zTduU?b1A_Bi(#nI78(OyqTEjz6zyaH-F6NWpCWE ztd0XdwHTi`ZyMfp+GO-MW6;8=U|5jTpd6MobYttx0yo$Sy>>`610qN$JViP!1A}!` z%4M8$+Hu%#@A>%UX{X?dZ+s7n9(V}l$&(Rub|7xjN;eRD1Q=*1PFCRn_+_aJ zF}PU(86tB@eKXML0TQW~;ducP!N{Q0tqNFDLb=k})?+K@6)`y^5yRHXkS0BRJsbx2 z5G&?##%bM`G;&7Op@|jB<=2`TD%q*&JJ+5`fs_;`VS{T8p9O-@nZ&FhfznVuP~vhG z9;L6PG{*xi3=Z@Gl`0nQF$3owHWz0cvpWu$KLNW>=@93K5v6XWd(%TH@MWrMfo)oM zvoihtN8J&=V0zc(s-a3IOWjOPUeVO-*YU>*kB(;Dpz}DlYXK&7BBX4mC{4CQW59f ze$NB={tteQD}L~EpkBw=DU%R+F`{N9GlqzCkc`P{+N<4q75*Z3rVf!#@k*<{9#P_B z1eE~M#!Wc&j8pK7Yc3J#X~USn4w>D>?Th?s)5_C&q=}Xh_MpmH)8E8%u3CjV9;&0% z*+MC4idF2ES3#_`UChe2Tc*pB;8`m8TfW_6?R;ilyxWi>Qbt1dV5=2d@#iEf+Q7ff{~$GK(`G&Hi?-*M-TQ#2nS;^ejPI>OEBYz z**I$NDcE<`IE<-DIpZjb1*oCfBlH{+Y%-QwIql34E@j>hZ!6gr27_bsVLLE8+BVe@ zU#743-FH8_y1KBd8tF?s|DBg$6J$MwiQ-bVNYEf-CkK?g>7fM@YHF&7oIxpqBnhK#gD%Am-Syy;%Fa#LUo7JZHLS5 zTh+k9pI(ChJ!>*Pc-B-j!x-g2Iu#;Qq|)Ee;VH@1fEs-9Jr4xX*#rd0;0tOwITH0o zQ*1l-ko~dm?(=cd$w%WWU->p}x&2?LO&X7&+JQu5PLV6y+?7uqI(zllFc>t;L#*yo zicDXuru+2>)yb3axr^V8&dwUbNUjtc;)!rVQ>Vuz0Hjr=h=KNvDXAun&D(yy7I!>0 zh{_nrv|^xE>(! z&$uaBKtUOaUy=+B?GlT$u9W2@2hcHW7M(&kWl@;s>1(E$3MTen!yQ^cR9{COs=>#S zPazp%mYNd%H0%sEFHD+Nq8z2*mgaa!&Qt=A#!3vOhnN^FK}9gXf+#|(m(n~B06`6t zW=_HJ$Iig1N6g2e3#VcKc@r^vJTYEmKbs*5i4wWPN2wer97ttlYJ;dgq35y7KH8GG zIGiR6?SMrh{oJI`Jo>zH<;qxCowij`xf;i+4oZ z3M;Izr>+&iMq!0L3A^%@^KLnPpRtF2?)U5d_l`&V-|LO9l~5X;+8ZS3dGQS!@!fy- z;%Bd(je}-aQE$a6g`!+BwVfpG>@X%oj>&KUC)Fb}0uZQh?Yjh?k5(9=x4$3b#@6t{ z^H0Yi`|OGD{OGs1^c&wtxNa@R&6tW-D@GWS_e0Bf+82{v5LXvSnGFslLD0X+k*K0? z(?)#mGatc0d(BV#q*jPcI!7x*Oxkx1+N7{a`W8$XP&z6;F1l_VesM=1%GDSFT0$f= z0>!2KId9J_EQ3isSWFGG%AN%@?1u1-`uy}2-HeYZK0`s#?sM+jt6^`lPkVc{MzeXWWYfkD8MEh& ztr4R4q~SZ;YI$W0m08LXA{8<}K6<))v2^J&?CN&I-|yO~VKNgtBTOBqvkeVx84P!u z*d2+~3(|~0wT-}yy{9(tk+V19#n%Sd&{q>-l20~tl?aPox$-Q3-f_ppD{i|DBiCpM zdJ*@20oBF0K0Fv(C3gU$)d<9R z$ElO>sb@|XqL(EvMoEijI*a{VdTDe;{LFSuBWKU16muXph74695k1o1s5e5~@Yh@M z>Caz-f8F*^bWE8HbabLwr*xULK&0vUJOo9RuRsi);#F!{B=E~6BwgJ&=A>hB{dd2J zabv40L`IE+%7((*%zyJnOSZDe2_ep*P?nBzfbZYigV%k1B^oWD8nlqau}E)8zy>Qo zn!7TsUZ&k~GX2UQaLX69Yhb`e#$Ro>nw<}7$mSXE&=xioI~8anR{L`b80t?FXPQ`T zYY%ML08ma{Rx@Ij(L{koK?NvRBm-k$J-i+QGFHiu6A}QW?Sk<@=~#)KAwx{mTDuu{ z19=J1$CvX`&5d;U`F*5-3q5buehIGeP>~eP&>pD;i`mPH!_A?zzy86*Ha&_rXPfw<)Z9pf-<_7ohq?<|~r=o}oh*L3VXXCii+ z)*%>Q0=^;eM_m&NXV8#~W=N$WCn_Tm8oRvN*muUi6pK zF%rc?JxYbtS`LKI_s&^)=T2!~VTBd;l(d4eQCMNyXrF!Voc|ubN9QsBcjLNu|NH*_ zj|Stb9dT*r&~SIQkuQ4LpEu#B|Lw(3Uo#s=?$&|+27w#W*(c}2ex1#s+A9;O867ad zVm>f-f%abr%+QWaoWw#%uU4(%f-_IXK6}i^_kQ>reD=!kBI((Pj_FebSdc`aaQFx` zU=ZLgy0rr_|2$*GQ2WYU;ODczqdxRM(3G=`$jZC%- z&P6!QuyFu}jYbA@$e9Z+W?~)d>@>=lsZ*XQgDue;r3#lFV(N0QT$BPDr2kOvlmJqr z3(2~BfbPXIqe)XSGpGgvJpwvTgJv;2!P=1HWB?9Hg|NmeiJ3d(452(kLZl{5Xqvt! zJyv~}$FTIZst^+@rBcF59Sl7f&b9LRILQ7;(N)=IDU=(DP%i>5M7?xBmP)dDjTRbx z4TOU&AP#_u6S2qM^KtSa^KjU{vvBZU)3E0ZVrEojnQ@FpGsIv_X<8{G3NbuXRud=x zYD%!MxhC!<<|AXcDb_r14_sJdA%gG# z-Q__IvWJ$gLSJ_`yz0&g&m?BhzBA}P3C-h9KLRp(#_Y3&d!^Dg&awH|gTWvv<43a( z*|DX_#|aA?Sa45_M>kUZ=C*d$KiQvu;*s&PTSgi%5F_dPA*z#K@16EQArM+vVTJ8b zD}ar{3fm6$tivXS&pK?}j4_8eg@Y=}}}WHibCuvgNq&xC!{e zb7u-sKgyO;DJgd{^}3Zq#2gW}s+vj)a)3dts)`S`zb+R6N#BDg@8kq|2*Xf-4g2pk z9~Zy(O*r_VeR26$zlS?+zZ2ET<4_wn77Y>>O(+SZwurI#(`*!?y?zai4eRiM54{J+ z9=eZMEu|IYK!wfN>Hb_I1*ezW&Jg*`Vd9~j#CY$o*W-~@EtD!EPJv}};i9K}0Jh*J zxDH?How4h{@EpS*unuA8*}k9+JhW>Ica+l=XWksiieZ~v&>^QN-X;#J823;k2Tola zOUT#IP?ZbTj?{yo^px-`Kx+WrrbmznFk`vY(1;omzz_nU63mcg6Br^xP&Aw(0VL@Z z+v!ks+EzM;%2ItW*UAQtWS4bFi0+UaFpH0pdj9DqvrCuHucz!kchoq@B2uNA`ADfp zk$Sv^#CmZwXqmGZ6=P{Ybp-ia|RAQXdV{Knuxt-jmL~}>8l1k zE9gh-MwG-VyUcE)Ta%_w&YB(}_cqiWOlP*Efina)Vco`h+>F3(HmeT5Io!~@w)Hff zV|yiC+pS}x)3NQOW7|$Twr$(CZQHhO+cr9O_PXEa*z3dk1GB2;g)z?2HieX9(7sKX z6S`cvsyCg%f&u+J$+y*_xXd*v5Fli-DzAcw3KGBRk7eYLoAVNZnpEjHn5%oGM!Ezc8WI_D`kU;2*bK1`oJ^m$>(>fpIUEZMhib-nhn?0&t-ZY>tfUS zf0a_j|L(W!>kiM(Zn>ShlYz~jO~-J)9!Eh1%%~#X*Kz8Ty`$i4jJ&0~g3g?I zwIcS9D&zz#*F0Jncg<0u>JaMEcoom!F+FPGa+Ew2Ft@$<;@5w*HUp_WKD4a1R|9G3 zO!GyL<<%hvvtr^0Pz>Sz5vXTqUF?D=Nq#*(H0*YKpNFTrLywe)#$*srAks~D%+|!W z{0ZlwM#XNG7 z4WvDJk$UuD&RtRkdXqaUTw6=^<(<}zl2>G#n-AX*$5R-g`tcpvR$H%>IM|8k0?mpX zSAts#p&3g@Jv+{QCu06Gv3=NT@N`%4s|kGwxeL(?MkeJ3?V;80wikx7i3rXO^JvZv zWFJtLrFSJLzY2xa)SmG?QJhwt)sk1H<~Ezj#5}6qFf|4PV*UZjP#VOGPE`fzVtQfA zI{!35_%a#8(Y?Ixll?m$)BUD-%d6QuB z@gB;&#ir>TB!s3oL)JR|ZP=AA390#ys1opA*((ewa&c1;dw=-UY7&I;5X*mXZ0&?r z7oGTEYhC6{H@!c=z&ke=E6aMKNpNGit*N+9+F!pL|3~Zu15`8C+uA^Gt z;Xg^#9Z5hQx!*NIQooK`HZarUzK6d@h70IDWrGt9aK4vBYuPW7rIR)#5ui<S$zB`|=#*42*#7vruGxuPz>aVlywsl=kL#U0?(yxaiM zcHlrLw5^5mD&q~P%&FUNAkF5S*+@EUFDu~(;#a02l$Ur6 zyl@=`AQugY?ocg2poY{YrbrEfHM}C0KL1)y!cfGYj>D0omY>_lJwvms0;RR z#h-(sw1GJUl-}~8bVKqV)5HFrvqRsWGNYfAhxG1R!yU4L(SI4+h10^jbdB4v7FqJ@ zkXb^LluFYDyq-2Yib@c^x#@q3rSV!~Lw%0|N+B8(4Nb8jLDbK0^f?xjFWrkmn^#Nb zw_yT+zCZ)r9_L3z!O8@lKQwO?Vr#M=Tw^TO4K1QD4T+W{BcjQLeYRPR{Iy&=bAY@c zic9+Wd12r1DQKm$4MyMRRj57Fcxr-Eadu*VYRdC!m1ofZ>_`@t10&)?+QGO$JIZs# z{yM}90>yux(R3U>^Ew1X2@JRx|L-akcut2ssrsA(cg%5KF^<%6THNN*z2}-(Oqe+9 zIzD?kDgC+W$H%`KY}MrionPi4~FCi4llI#yjh4pr`?yvK7Hbcd`uh6VD>X8hI=L?CtuPkrQ9qm_8TZ4u_LQIw9M%jw(k=dmqK^GuNL0DtM*$yO`sGfJr!}IK^L1Ai;6NMTIG#S8Pa~6FS4pJ z9r?efIjOn9ARb4R-(g_R3}re0!P4s^*jzVA`;CvXZhKM#t*R{_n`{!r##kxmsyTo2Sns5Qj zoHfJkXkUz;e<}@0uL$%$GSeoQd<~0h^vvUyA$Xt($~SS3V7nJ=LBbowJq{&E7BDZn z1&4f7IEjL{8;hfP?piQy7HEf$8KxhNA?UwZ#AdhoCb02nYM5?|RfUPEx$@qqpBA(2 z!5y@eP$6*IvvDg<1u!Mt&DJykfY@N3VrwT4bwHm7yMDXdd>YI9?^475x~&!v8C>|tEn4J9K(yE+jH>odJXRzqKZIoB1k@G zMucD(^eBkuUM8Cp5V`w}l?32bycZoCO$-tFBa40HFd$i>32m5ISYVK1D8k--^I7%z z;$!7mrBaY8JP5JR+F))~$;@2PnerA@6uQhv9fCEM)Ef>(vTGiruNF)PkUktK&!1~{ z6%)oINZOcI<>As&8puE?i`A$=&;A;Fj%ed_wFClt-)UX|+Av~$B1+pnIH^Oz5e|UK@eYB6FabxG44r7##|vAx`)SU;Kh|w` z*3-yfy1_n`vlN~YQANC0+be4bF7>HcB3Qy9yMc4=Jin-P7Gs7OcdlkMm5u-*Xiy$h zy*?beO|Q1}G>%MhLjB<-o4F|wtLyT|GzXOPstA{LOpQ3*<&63de>Bjv3UVy1M%2xX zO%zJ!L(gbkeWb7urllz)wmzn(boei*dd-d*D$tY;-Q#OAXMJ{z{#wvpbobY^uFrEO zEooIG>jj80+yM(?y;%vu9Q!?j8x1B1Y!9hT3X+TD0h2@T*`B+=*|+6Y-sVQTBJRaU zmVh%M;lN#i(=9enxz0ot*#C;Ifso<R+&*=kr3 zS(0(}P&V*yZ%1wL)q`reZ8>%9r-UC9pF`$0Hw7_7K-Apd>qGVTyVlvi&j#;;D}DW(rSOJ>%o7P}gpP=}<^m&k9#oJ< z1PLj)vrGDMl}bR*HIHF&E}WW|Qt3fPlt8!c#PNGxLZ1Xz^0gEY94~S>N4kk+Y|UD6 zuK%oZ(e0!g;+Fm`T{bc{8dO+wFX5RWUDe09`B%a+W$&CUte#EAJV-mJ4Z;#*rsM4u zWrU>*lYKXV%g+sE(5-4-q4xsFLy*mAw&}=tDmgfeA1Y0Z83WmYDQudObHr>cl&Ima(C81NL_u;zEz}hMF6S7QH*S_B zknfvBGz67|pa|-8IB~30W2-lS?S3y$^A~q5u3;G&THrlrXp4pSNg5bH2g#T$Apr%w z9_IIpvDhvRv$UpwcX7O+NX<|6%g}OP@P0le&IAvN8(OTiVaV;80wGqfM^)NphW&*K z=WQR3CXoQCRp9uMYyCQ92J#)CR~Jm^C!%v~M2F6~&2Bv@*|(4aR9Y%&ErQ(tMa(v6 z`+QB<>$cf%m!{fA?HkMUV;W5q!$*eCaVB&Gj9f#RezxPKAzzDhsCP3`kU4r?_PGjaG6S@ zRcC%1Tl}S-vnpl>SIAO!U(TbDor_4K5dv8nggVu($8qIe2+gZ#ke-VjtIJ=oXfUY4 z_>B4k0SOkOfp>!N@RtV0B_%lfh$h@|H0QcOWvF`D4w#7p!RjKfk>!zwz>puI*1O8z zvx^_gY|ofITon^k_)8FqS^Yy-@8P#wwas}Sm@*-jg@SNJ6n=*+C69h+(Acb&6fu0Z zI4(i@{@{Sx5JM4g-e^cTbS86sd6-#+x4@Fi@NdR*w=%0-@0G(Uf5C}bp+_(GKMaEu zL&8-?V$cU3pw&A9z(@M`C^+=KWHD==XwSQ*e-JioSzU4&pgm`D{3T>9f0LAH5A`OHl$1^5UMcx(WF~Eny_J6Pm7^X5gV&&rbW$a_Z4M ziR7A>uB;rRIAcHqR+uJ%4IXU~CVm5TTAe#^f4@#_DHV2B@H82b?tJ>H#?z0Z|1GsvMx-i~n!;P?V8$j^ zSyeWqn%S-JfZA#)3RwNIAj7*YxRTm@b?se#H|%=6l^dm3!$FvOxTu_wLDOw^U3jXM zgDCxWUbJfhWV^utaw9?hTlnfXm})a-kIzM$KDgqay0M%{XbfeUjvN z3)DKH1kxScJrS|Vdnb=htaw|Ch5F8te8<@)s;qz}o0CKk=L7!|VVF2(tVsT?QgbmN zSMxz!^Li3ne^vweFWFWrg01U_2G}nSn`%g|*{p1MZVw?vx!J9pv_VW9q@o(*kPAfX z^JDt;aaM(}{BJwqKwH!r`sKyIz7yqnx>Dc?`pZeWnO{v>tcYg3$f|jx5If`F@G$vR zO{J@jaES354UH?Lo&)PBo_Q#cK@7y+OrYWjm_xj)+#8>&A|&%Wrj(L1MdH*hW|llyjS z_OKSOZ>J5^CFRZn}nKjfp9Qm#Q5zb81m`htd*|BbREfSSt$7?mlFY zL-KaoX?=2zbIu3CW-zg`54?;(U=G6%N^Kh|X0E9TW~St~fYF~6r-4CPrvInBTs+eC z0^`exHNvSpV{%|4y{4;DsqVz>D_jlJs>`Xrf{wK=|0G!7tZb>~g1cMY1nr*)r~ZLi zoT2Kygdv$I_Zok*bM$xL59tjUB^mFH|flIQf$=lLN+&Pm4^KaG>#kR5We zF2{8SfJ=+A%;%_kRHU}tTZP#It|#=VP_DCBIelNqo=>jHJfgDkyRLrtnZsHi+mWH2 zwt3Y@OCADif)iQ)?H*OF-r&d|OA3%FZprs5l3!iVl=+>&^fCY&b2I$C&&OaZsa%)j zl}kkIrj`viO0@V=1(&PH55A}yk44_`_=vaK*A7HO)5Qvp*n}VM&^c5EWhT8Toyq$A z(RSl6+jaXcab_MDca~^s9Y(Hz;|pN))z*3}1fu}fy1~(m|LfVnuZmY`I3G8Bt=>yi zEAT=X3NyO}DDQZ^zm>Ya%oXR%;tSI`A2=;RzmV$}CXnX&2AX24bm`$2`=6Q@;kmg* zKFR#$fA;~s4Jwn!N0|t305kF!SFHAL=_FSRjy^b=Dw50+*oi_>P6%fn_YH#YszvW^ zsY3=}_%IVF*p28&RIgMfti#rWgESe^VK9eUGPErrDYIe|e>3KA=G*jWL!ON}?kObm zl+tdL&&dXW{F*YJGuP+n$giA_q1d$5mP4SLM>OUjyzT(gv98k7B*DgmP}xSX@&eNF zCg$!9XxpO1#EPeyS|SyaQwWb^Hm-#J(R0RP1;?h|G&iwc)QQKy>yg1F2178$2siDa z&KoTHStg_e=>HRrEEuc{+PrJEP}98<*!e5x`4_kdM6^d@M|B)a_T#!w57vZUhWbBd zhI1Op4Em@3nXjqh7~~;E{&I(bU(_elARlyTO*5;SU?dblS#;NvVS0$)&b)GXpp0)| zScuFHfvW@QCNE28K&b;Btv^;i0^pPxMA*ktUl5v&?}Qmy22{dGrBd~mE0l>5mIyxP zj&}#GU+6Z-5RbGC%i%mjb< z4*>**_D#gz`+xDI6AFN*1+^ux>K7tl9jD#rbN9~k<2k@<8Qb%Yh`r-IyO0wLjy$SF zsin`X(Ve`P-K!OA`&Eg(+U>(Om8~Y|8;C=5Y?W|FE&%klk%dVfPTOKibGTNPs=P0! zuqK$4$2%8CNLVDM$DgF+=pPx>q5O}<u?hzZk@cPN?C4Rqw0&4sum8&yO*W1bDs~RZ6HWAw5O$RdgceZ%kV| zYeq03cB$rGj2}eDg0T+xShx%$c+s@5d|#9fM-1v1p<(@A~cj143M(^&n( z;jif)JZ1_zzUO_JtRTY}jj+O(P25oYhEgEb|7O!1iX&+j5`+1u$h|_(L14~+&iPL` zNkkJc+mzCz{QwL5JRo79D48|XDx8u*32H2n;krm;D6g7Ar)Xk319WpqVqH$78V!wB zBY?$nMUP~o0M*idvpaU;S53AFv@Ew9D}ok1reY@ad!=~^RSarlr{V~^Mc!Y(unS(( zSrLw7WvC3Fpm2BnQg>FMnu@XRR`|?Gcdg5c2~xe2wO_N`dHB9+N+Q10ZVhcARi#=L z-*36Dy=Z->Wr)u(=_&s+)V>erQY_KH|i>(x)CB1YbpS|z_7a2dd8&&95@jL`8qg#noy37 z^z1n1uS_2OEZ{$^6WMcCnao`M?~D3C@R26^Y~iihpr~kSg=dF2b_@pT(A>Z2|51x` zavGZ?Cb#x$Izx8#pe~qGM*E+Fhc#2>l_X=|gV*CHRfLptFE@&>qdH{j*`|3@Ne`U~ zn&nB?y9k#gu9-Zi)w8_hc|lF_c@Y0K zhbjjxT- z-znl~P~f)E?_th}a82A>|J=awKfaW5OuKj)l-u82}pnvuadBX6w0w_-KO4RSqU4RB;5TF4`t<4MuHCdVC z!HBe?6l9J+4n?`mop}?rHG~=+aIsnV_4_(yt{%9sUkV=Ly=sRY(J9y$5N)yi(d(1a7g%J>iOAFrtM4?E2!T7ZTohExImm%EPA9=A zUi0d*UPv$^Z6Xr%Ob03fJ<~kGjvy1t(rFB+qqwa=Vz>>EG3;S!n2YthH;U=DNqIX~ zgedm%%*S5fW@Glr8#o?0rLw$`%T{&fP?l`HZf>!#C`bOr9D9(EaQ6zGEygx&{864Z z9mK8I5bV=??wGNieV zdw$#G2j^lls8)p|dq4pV3(fnpWZ*NHhWt7(5S8w7juKZVRP5&up^vssRK_)+#J5P$ zE@8yNtf-xC;kxf=;({u)zyO1*{m#zU>&pZ}QybJhGAl#*bI0}W3m){I?xupmf4%XV zS>6AwAv){4@5rf7bskcqRc);fv}zS2v}05X$KqCVnUQooW~7~`kR@6DJ)ai{Zq)>gr*(Ms%Q4NR0QWl5$!{TX`{5~~d-7_}81eX+ zO= z3CO?~M#|{+4uiNLikxiXaI|{__S_pIe(B%;m>C6zH8FTuPaNvzCu{gJv&L9vgZGj6 zj*~2}60B}|u(3y>p`pt5b(j74dkdG66O!)2k>z!{(P$-vP}V=`)Q^w~5b9;{5H>af zAM4Y#JLnB#exCRD)Vdj3B)Zs0Kg_hcCrZAaSDGB2foGQi&QLiUi{AJoL#%Vf-(#fY6Ev!L-9i!ky-ufQ6m_0jFKT5WO5k}uZ9s89)vj7&t zMA)vv5KAVwCFt-ctD(EptzU4>1w7iabZ8AP6;csF4fe zHV(WKAZGYEE1$WxLQAfuCxV77WfUbB=r$C-4n1UvfbcBP*nQs$> z29)RcKCR)w6(;+B9*^D8KCJ!K{>OpewyFM3lzoXZAfmSjxg;Sx9x3hyqCSxE3Ub@! zJWpK@!qR}#=ITAeJkKBR^qu`d@l2LJBpDkm!7^nw9h_d8$7G`T@#{*S=zCeSY1(*PL0@E=nk>xRUVa$mGvQ!|bQ)&* zL>@`YC9|N`Mp!5Y4-W0yM~_@X*#DjdLlFu| zKqmt9ap;Mz^U=Rs^FFxx)A22J+1ZWmj$4@8U2mWmhWtl2Miq|T+?8KFiGA<}SPYp1uwUw3;Wk{$>oh!MLS zN^H#WvvE&oLcy-F%Py2Q`lTfT;K5dBzt>6C_7}5U=elkU4rpq`;U^jVZ6C>yWA{f> zbcaZAYqX=* zgyqBjgF8%i0sf?UZkBgU7`uGRq9L(kH3=Pg4^n0N#L|g5YeI4wD3Zeh_0hf)9ls-B zGMPbpavVR!7(R?qoDM{Ubw>atE5|H^EP@EBx3*Lvm_jxpm~eAik*ITZNRpe8q<*BM zNl5#hHY*XB0wZi8Mm4^*`}&{kM^)zqbh9_LA3DNZ)1aiI3sTv<8FSsie<{EY#d8~& z;{fsfC<|W?!%u!k;PtU7t+YV&3eYb$pk8F1-yD0up|@A29x#$;r|lpdhUo{HEyKdK z;&Lf{%--lPT)Dd?3LV4M)@Wt(Cpm>VIE=PH+0FK6mfZ8$v@G#NJ5tXam#9BXr!TwR zPREIV%KjT*S)r%_%;YL-Q1x;=hR^T+{YB>54;S3(zU21Ux-Cz@4T|9#i%N&t?(Br_ z@;#>M+2VALVi1l=^w7K|9MQ+=ik;ZOplGW_GE^9 zP6`0ZecKduvbM?BBIDzN_$NbjqgBD;^wn$AUaz$Rj?Sr6d!f;!kZrB-OXDUPynF2` z0>^5&s?O1(XC4T1LiLF5uEXj?-?B8f5!1~>b6d3|$KZ(ptisSpH1XIMB;^PAOcOe3 z6hr{>_eO5egucEX@(U*A2V`Mn=@i;E5j9#IH@mZ+5@AAke>#~(B|quX+9-)_WlccZ zFHTx3n7OdrAQL~QtS$%jsa;VyK8@}#{^pQw^qXkK7)~HD98U^aZX;V9cHauDJE*{? zl3&HBl62?}@iQ>ZJK{jJo|QYU5WEA&AM|0o(#utu1e&-Lu==+NkNq5KzuVpdn_ z>jdh7Rh^xak)kGR>$|Ui;;EoZ?I-u|R1PS)om`zDRHSF#FE59WcXcX`Ee-6K$hQrl z5iPwmQcFs}$O&{6%<4PMdDj9LN@ZlU=DJ+nX%@tD?s^}tl%*Vw@DUoAo7H)QA@HXc zHXsNH@E~CS`;Qop$~th>YPY3*gEu^)+?B!&0g+z@_g zb1Rw%c3_4o-*tFW&CevY+q9n9ssn#Kxm#7Mb-@s*eNBQ_4Q3HG_ltTbJDzU} z99=-fZK1<1|qFBOOP27b}?*G%>R#!2y#Xq`dsaUzT~neek=$ zv1!!1IaP}@S?2T4TCQA>BJ7@m=NW14)tlvvzuWbsatgB|GC4P=9_>%%5yZ_d##Vd9 z5seACu0M~e-An+)jR--jwYgwYsgwepT0`bFUwo}7Y@5!_0YujbQs^A#0WX>?Ho|S@ zJN5(zvPUd|Gnw}$AB|6GwXlAlFi6kEsuM-+lzM_t@7#C zA#&0DC9~jy~|!ntt2wIs}R4puD1kmT2iBjp&v7T@{p&4>%L*t*-2JQx;Pb_9?I) z4z>pvYLVVO(~JR}kS#uBck0I~f}SKwR&yY~o-!+ljRCSf)`uqjxNuq`RIspdfoEmMMl7-$2 zIH-Vxni#6LJ4w=CM;On~h0BYuPA&&|-Y+BO_G2uy&ctzdCWasB zs#3G23uqLddD1EZ%@^N()&LCUVN(oKuZO zj--AJsY00(JAJeo2~%a?@^Lg`_1e?T&KSDX$D6g|ax-nCd%25N$7Ax}ri%Fg74>8P zvl9`o`Mj2F|F}HjSAJ;!`-sNZY^r~my{6 z9dlJTAdPgYt_Nega#i2VpN~oO2VWb=RNr5`K`!i$RQ4srqVjfj5dHab!}WbTB^x5a zxJbk)L8M!u>*QP_Y(}B4!yQ0L)7xZ<2_iZqA1rj3)t&UEsV?s_*IQAgBcpyVpK5?9#852Pm(FJ;MFJ z-|TyAAfTj+Kh}4v-1kBJb+%3(XC4zDIfA_h%rqj?3}3xQ{^k&&MfPWAsM}B+FG_Ba zgJp%BCDP#lhBU-?cM}a0GmpUEXjGRySi1MMHGJRWLT|YmzyB-Z;{Ri&1MpT`JMJS@ z{jNC-m)&%)_lA7$!#q5EVUx|?(Llo#O?=$kVU)MNV7oqVG+ldh|1hk^Ur+hXaEqv& zQZ&~>^7VUnPCWd|Ym2h7mV70D4Kr{~kZQS1nmoF+?x~tP29cC7`!eQ6a<2kozTR2| zS`p-5!#Fg6KL_0eLWnc_YT)*Hk8iQI=S9HkwIPt{QaZ!1#t@^662{L#`HTt^+z8&E zIrrUb?hS(yMN^9p1~f4@5hqLv0`$#^A~?Z()s+J4QeeeeCTxG1M1!%)TQchxc3|*5 z-N}YGVWW{ZaFE%BuHbads$S*8zMZxC&-5?0IM}~}gzQEzNB4__Uix_*Nw|n(i2CpI z8-i^_`Kf4e^bc2~qpcD~XY9p}&c9?iHC3-F;$KLa1@NNLA2R7cej|`ZAg?0sNl}q}->W=1kjOY~QX2UOb@I2O5dH z8+3dNTiB1iQ67GJkp`NDQ25MHbym4&P~Z0l*x6t9XceXQdPq5C!hy{xiBF&Vc{vap zEDe`1WaRwKa9Q-N;hiZKGoy*jv($XAcL!fUpO@q4|2OJoZTJrmOl_64Cb!eEs;%4Z zzAd2oWq$Z^tGoDK##W+*HGLJ0?(K;BXV;MF`>v!Nkf{#GY>p1L=H@Z)F@Lfh|5Qb?VUW89g;erv=vtHyy)#3vvUn+{a# zi+Q&f>&N06$y_609wpsPQ81&t7US~EY2Ls%k(Zi$sOYu$XLuPMNUWoyR#-qpQepU4 z-J^QCN|?Dvlv*Qa)%fpz^n8XS&o&}K*#i^Mz)RM!=d1dc9rmB|(f#vM+~7lmbx6gL zcr^Z~Z+`)K>0+e>!2(`7F5}s@wFn-utjR2W2J4r*uT&<5t&F6K1{~lL|)_8taJ-&u3?w4_8AP zHM2)fm0GvS&;Ko$2XZD@EpT%I7GtUnky05%I!^0%avhhah1~D6m>kPP3qWST)J~iC zKcEtyJ$bL|VWzu=Lb8HZS&)M8Uzjq;BUaa$1wyl??19cv_w0uh{b>mJyPu~JRyPIA zMNI&M8?t}~5HV;G-BMK>?H>)zrAV9<2up3ervO_iRMzVPT1Z)9QV&lZi#6J)(&Zo= zZ$uDq0FyfgfW~!{G4NtrB!hD8&e7i)u2YqKQu>h;j(}9pb&-V;QSTyVGUK%-jyU zjO}iJvNuvxI^Rjte%IDE61nTcIJEG{OpG-120|D^a_S@mqV?6W&&Ku--U%-g?Pr<{ zuZ#;*p4hu|at59{LfFv#7M%9Qj-GWnLBCI?dqP>iNmhnX+vfj$BK zyS7ZLm4E)xx|eGe)F{%9@hDj5KKw8L1{7#C*r<;M16sAs0G$|A7gW8f;1xJvFE`83 z2s5Le;4+5Es@Vq^t;pphZEDEX?)t&!>$tdly z4e?nMSx;6O&+8zCv~gzb^kkVY>DV9LJogpH#kTWN zx@SG7fU=TwuxpIfh`MjEs(G?V*cpBWTdmiIN2OMW$AU4ZO_YD@a@PQ3|?6lL3c}ggpaw1WI z2SONr{^6k%MD9!kpuYn?Q8HwIkRX#@NTMmnW8QOjTa_E#_jHw&s$U^DD=@BJxeoR- z1C(0NO8=|03YT^q-kV-&OXWW9D|_3IR#$iCzrIp*In0>>xi;-azdF6o7_{EA&fO*@ z66fH~!I@La86CONa&{P=qOCkQb@5}PCvTvwqcw_yZnx_ZVr%h|)L9F_jH@3{(qo=l>5HC69qWdIOfkrWSLB)n+1ldBu!U34Yuiydt42qL zn(oM`SA&?%_eg3XTd=Dmm!%Q3m2-f~%2RinVQ?HrM~t$}FKbJ27oM20;H}o`GK++Lw7Et&E_q(PWQh3mWQq6`y9z78>;kq{y4NOk3P_;`*jHjePsyi`qZJGXAU%_iM+*8UfxqE0g8xodphycr-s;ZOhE*)^^=CsvL2)8OU5 zMZWDA;ZpgX^%RmcrefsSuSesu6zg}vh64?~hi!f()_vLMLKYDQMVaVF-ufHpXBw%& zSQ@84;QY*%fi2Y52JvbVMHsE7al;#W*c@h5f*Y@0H12Kx_<(g@6W`26HXldb>3tqg ztiw$IH}qKlzoEx7N<-_n+wmj*PvNB|_Q2j2;3_4kgy>7n;li!u(f|E2SUhOXoxv=s zp)rHi;PF=MIv=NFBEs1R^y5~AKHK)lz?=YuN++0<^vmOMKxQEBzIu&)MFj(>l=1dKHSjq+jZ5N2o9OM{*046xFU{tuarDQ6o=^6|ry}n7DNMVct*Yc+EE)MSaD8h<` zGZE%8mtrVX?kcKCq6qG_7zlJbP@MHi|GCHWFqw`WahB@`1*c~h?;y#)zatI(Siw)> z?{9;^jeb`8_^`oM$=V6j>zq9uXHepX-DiaTbnS!a|iS_ThLQ&OEpVzND%he zOIferVao`KDBZo#lmagoLd%o_a*<6W*XUe4Rs zc3pjMmkcqxxs|E=p>IuL+JCrAfYm+ge^z(!>#A8gpA{Lz7vIs!6!IvOXHip2&=cNI zWA4K!WP3oy>nOAs9#PiZr?SRnW;aPymKg}EhJ8T&F%Y_!BS+20B*bg-FMo7mK}_0M z9Z6CEB%7in4crslv`1;?H*+DEW?u+LP&d5h_3`xzo6Cw3?%1PBcY zs#qus)U>Y%v4!UDm@)U#0G|RhD;vG>^nS z**V4d@=hOlNz8(yy;#Dz<=yVdZLN;JYzPSJ)$a8|v59GLVfDTNgzQ^`HRlxhGUxoN zR%|9AvHCl(^~O+bDpjZ)dFAvw8a-S=ll_Lib7Nx~DDGYeSIhNANk8d!EMQ56WYb`? z?SbTTuT)?A^GK%Uc}OeFgw-P|j1+ zx}Kd1I-uw*t(|Y}LCwG;W1YA$Z_G5szm&&<*W*n%z~ESy=J1p4<0Cz?ZfTc z)f50UZMMc5n2|L1+WlgS_OX!vZ@Qrt^S^@?xi;)o=h^RMv+eMZ0HqUlcWPiOsrsPq z_mi&!2&62?Sf6VgSzXSer2BW68kk!X_W* zFIucxScoV}Q(o%zn}xpS-Iy#_o5@UpHSgHcLKYr}B zDt}<*=a%f{F$M00v0r~-IY30d%+{mWFm}bV4!3-KSc7VvH|_iRee$XLE#>>9O-}V= zz68+%L8aK)Zn~KELnGYW-C?)#2&r@NzqW3L|=B=~SwQhx|5+pUHo;dGmel0o6(CBK@v_GO9O-}V<1I=wg4Lr*up_X!KWEn(T@OZG!7|a@V6Dem_@I|LeXKLu;nqI9Ee^iyz*| z-R?n`5eEEl>cUR)k&qxMckUmT6K&n!FHU(b`Br_7J}pgqlbSN9Z_F>E=g%+e1HZZr z4PbO%PY{pgh$rvDAa~K_+@}VM3l578V=5dV1x~=fc-ExtcWzq@Js^wK<_ejsUri~j z-vB}e|C6I&RGB5(>_#zN%x*||9TDY$F__q>a`!$&lJy**LC>?7;x%5g;RM?gOJt=I zu?oo+R>(LLkhU9;y_8$(*yt)!9u|^)f=ZaQuR50}OA(zrA4(#@cp?jMCF@%3{bVhf zT1m!lrJ_g}#VK27Ad9o4T+tPa!qzz<(+@NnzEAP2k1g`UV&8YCt4^cX=8LLKz`0tRUwY%S8^imEpA;>*Gz;vfgZ}f*rLASesK$CQILSArh}$}t5^V~an$ELxE1$Q-f*RkfX}qtQx%2=ogH36#fu%l+p~;Dd2{^$ zoAq}u+SLBltw<%RBUl7aqBO5J-Rg(k6O$w@gbn&j=_WZ0c3k_;ZlFv|3xDgB%sY@$ zzYR+?R*_b$uvfO6s_UpSp`H42Ui#gzC@nh+qJtFgBQ5l>_WQuy?PMJ>o)`P5aNdeJ z8B6;&DGF$UyjhF5a;HMNHwd zns#wwo$gmfe9+;3x%}EG(cjRZES=(9|9whZ1$WakpYY#@hlvYtPNOmQC=Eoj{h`yF z?gqxXKX(jpv=&roSFc>R+xhK*vt;bTUY_l2rSLyJFDtnzN z$j}ye_{g_!FpoQQwVF+JV*sbeUWXJO9To-^S^9Ol0qJ|0kzPxFy|NG4qj{WX0-v_3?N#zBH=djU(TD!+-JNSbH z<6#z86p?03OJ|_YT&E2+EQ=l3>bz=wuEXZU7$WM%thaNx&p>L>N*D$47HZ_~$p)W` z)cI<%2$b~Q5we(+C>sd5`#rwdGAULBEqEg5{gb3@R+gVq1DHe8Q~U25vSn+$Z}Yl& z(`(MXA&k|nsU*&52YPH;p(kh?MJ+0cJLl>BYipbar})FC2!bKr%y6b&k5Mh}o`1f% z(VV1xS_^3ACnU9=eP>6{N9k(4XgDQdtjFWR?}NOoM;SsU(wEOQs6LPY(eQw6r};EmMcmgS`1!|#{Mpn7=>23C_A z^@i^`N_L3&W(Y74RkV|Zhl*gJ!=k9tndj28ML2f*h4Ak6y55!PhVl8nY1*H*wgMhE z2#6v}h1Qruva2Cq6T%}HiE;dfE5g$h71zz98wSDkxa2jIN^&01&sImfMY1)WsRT0M|Z+J(5f%^WCN88=(RGao>& z1Dgz_Irtlbd!u@Bt&y$)(()VS#&*Zlw}g({t?6IB=dR~0vq8^5*Z*Vco1!yon<(RS z?2c{QwrxA<*tTsu>DV3Hw%*t_I(8=g{WELL)wzG_RMoD%3)}-C9&h(E)Ee=%!#7L@ z?_@iX@&q|ua1kV|zp^K(Ds}c?y#}$&rQq(yvpl_3x5#z8E5gfB0~5=MtiJ2Le0Js- z*MJ!+SA$B-a#>l@^sZrz6$OtPgqqTxp}xRbyIsIVOu}@w)iMB}*I?Z*Y^l@9Rx=x| z$S&@Ej>WC77<;_rhF?=jLS(mFiEi<^Vb@qDs$KZwJ_T=;Y4NC@or{$QfuX zC7&ygUc7F9OFwXE$}H4|ejig;+ZR`dFKj8TQdZJ6xqJW^^a^Mk$O!7a2ZJ71 zt{j(wbFgI;SDiSnWPb|w;~hhM)oGrhoI52WnWuV5-lO(GrQ!i1sIY_toA3UM1OjbVdY`XO-6wh9+;kMK2umtu%ztH)vCDm6gsc$ z^dr?fPY~v8+3RLciHa>@%AuJ?F#u|cyCwM_3erl8L0-BVc~En42W8RY)9dEIJr#%z z;m`xQSwliySp8yO=N`qurq)&p(y*7XGAYKfVUNY{bSRQ~3aThl%mo!>U3jWgkpzfy zcOloh(emCeZ;2+IZ<27lPj?^)oF5FGU)OkxY3%4NexQiPC?M%Mz_Akl2#w0$_WwrqIhkiMcC~Zjgmvycoc#|spCp1u-n8W7#j`r z|6U#EIehTnSHJTpJLS-7TJt5<>C0}Dyyrfv+|I3GxEwOr)A+5)O=?W6?!x`P1pMSQ zNecc9q_i)Ql^BAG1Y|w*OZSBsAZBkWe4}9ZIMc;#4IVukomdIEBE^#?ev?0f6QyZr zGE$^RhS*57g(D)=xcsm!R{tSIQVkqeR*>Kr2ur;FxQMl-@M|0X+UI;qF#%bFHn8VR z>x`-9UWve>(WHFT+X#m|*Bj^a^jbgGDenz6`FHXsOtqLWgg28t8MB7G!G0x?PM)Yf ziX1lVLnX&P63)XJc%@}owOW7iEo*6Xk5$JE8jnuo02%`yp$aG2x?Vm_T@=9Ev^|ch z@Cf70d$gn^QcqUdV{2e=5VhH6J@oHi2M>5+?~Qf`?5(%D+MTs5+rkZMlBuQ5S^xnm zXDpvLEPChhX_Hw&GxGQ(oIp@`%@3Q}wN!&$@zQ0DL z9M_~H8;)Dj_SXwr&YM=X>w2A_Q&kZk7-LITSIMp;>*9s%ln0={63p06mc3m1@}Gin zQjMwwFBd#63qAEg2LEEy?e@Of-NPtUS_7@8C{eqoD6_N}Na12N%r-9iw>}UV4Ah!J zhfVUALQ~vAZIY*FL-gUYfD=T*oZ(ASg9qgiD(%OzXB0hu_QcsQjnGYZCCHY zh;u&}_**2-htb8828N+3MtCM#oOiS0%O2R@sc?yv2Y74!tPFWBm z;kI?{4=BS?nhmhu0xJ@;fl}+1ZOTGBjQ=~vuw8n`!p_{kkhPfjeNw2UoN4;RP*Y0DP(K zRJcthz)z}Wigp@z9JywFYfC*To&@iFQQM%=U*}N1AVfy$Y3XkmXPLftbytt= z82EYdrkbTCWQ_bhf2`g(Bm^y^_4S>Y`Ci+u4K_o${lx~p7x?gp11K0+W_!OaV4dwg zsPRQRLN7kQ36=mSoXorFMb&fx(EEg)cfvbu7GrbHCq7=IIPC;ib#({MnZC!B|^qVLPF{YLE!l+lf`%E!+Z8;*Z)dOv!nh* ziK&YDfYPoFpm==U{S!zW$BTzez8F+nH9b#KGXpz5B2(} z$`6!3w`FZ)A-`+%UASv+$=Yo)w0|opXbniTb1Q5;8ymR7Ffafq-XA=Y(GK(0YvV?I z+C(#Sd86RUoHT;2%j056q;RdJb6Lafuu z$jR=X6vj0MAb9j(ArBFQy=Ht52_$^rc9!lH5j^FlWmiqcWRC4bkoM1wJ1oAX;R!qfON4L^z}5%L~Ns|-It zX+1h)A1C#@ZEvjSdcpchi1W6Q2KQok9YZ5KeD&Mp8Vi{R1Em?p#Rp`Bw%B;9vf<~y zf+RfX=l@)S`2EVJ)cfE;zwJZ%>+P=GPA+w9;3-N}us@ahI@xUDaHLS2dfbGr98~Nf z9T)6qYl1ImuNv{uEBsJ?@iTK} zH%;Apyy-92EQ(PXMfJvn12KHjVWWZLps^+4`CmxILbNh8s49r%B_sEC4ZdC<)&9M! zn*6SBug%9f?~tf-M%3LWaO1yw--RH#o3DGHgQe)l?tJ8>wvy^hy|K1iPuN>`M$!yi z&Ez~R5X`QLk;Zt|7Y>DtR%KhA-QU(lT83GNJEYCaVJ;e_}K_-R#y%mf3JDZRBQmukxOqHpxLK zjkX5{^K-9Oly%2`@|wN86FAoq{CMiW)((>G=Fe)_@iCj{+-Ns^zPDZb@dNF(l_DWF zTFCkX_s{3CL?VuaHc*O5G6*uWP)+q2NH=p}6~CF7YdVgXTA-SbnN$ zYEHT4`!0TVP+H!bbatOR?&Cv?TSl6Vr6kazk&unRn-WSKjFvsmc#x@hxhTL*)IW zxl`w}5q2ZRNgC(BFW2uN+wCVxGV*dDnxRFTD$VR~DCLT&7`T=aO8b4KbiV=lZOjcn zjoz0fuqp{|;RPuDSmaTR7bY)-b$7{xe~-xvZ+**-i{RVz*z&eUH5Rg|<8jWfQ7xRx ze)ie~Zav70XW~crhLKi^8{QJ&oA>S24uy^q|N%i}Y) z#(iV+tgDW}x0EFd0WVC3k9{H1k%}@lhu{B^kfU>E%92MO@^~qp^(J%Q4=mA~`pev~ zNj&lSqwuzb$NmOwtMMXNffG8P7tEPFAd6n4{SWwWULtDTbTrSbcA_A#%ZnW(>JPnT z+)5(JSG&ZrdlVue$&Ug7d#ntvz>-^QV7r)mt>I9nCkfFA!%^{qwlp;V)s+}!@yzfdxAaz?(kzE3im#8l8R zdGAM--j`uLgQ+(AQk<;$^}tUZhP2XEI$Zwvem9yNAFZa`tIbi4m*Vj(se%VZS`$^LV8QjW$NPItJ6i)B-xsGc$S3|p6<7VGzg2PIDpmAC8dF*^ zJuK;fM09)G6Z-e7Z-)HuSO0D&IIT3_{+aKQr(F$`eiiuk zZPiLXqoB)%;9nsuAWFja{e7gQ^BR!ccAr{i$GI)fll8F>&*j;tQ?;{xpURoPNL(cU zZqOa&HX}Q~Jw1o$zw*TNE_{k}?{H9crKl+Gm<6Kf= zf1MON!2hu+{VLa?)=?bID!sXFKVbhn$yrHFiFsAHU0w&AO2jTitntRqKNq{SW|jVk zt}Nv~7?UH$$BdR)fov$q50e8w>?UJiWCX&@%v=Y1&L_l9f3t37H*DVXqP$_hjyvkr zMnsK0m6`A52fRrRz44VS{I~jv53MroT1S9@tQLZ-2CQB&|Dx_(pof4-m0h>sLgleOPfUi&*N9w-VgX+lYc1j=nmpk&oYDP=*G-qXyKCIuxmMot{vT}Q z`1K!$7w!}L#Ctw^^_ClAzICl{$M=>QlT(YP*?v#A`nUNTu$y%dj}XZtvzRo&O-rqW zH{o&XzOZ?MHD}JGprg4+JQGZB*PI5ktyR970gB|@i@7WVy{IlTSrZOm5YK&MO>CKG zt%YZiD{FuZ28_*wEGMGYvR%+k?z&eYJ*|)}VP!v37TXPHGG(1h&FhR#kvos4{KCYJ zghCKOFM-$TXF!WWq5gIfzvf6>U^ zE?6tnKapOyFC;_1DxR;X)8UKa>pX_` zJV%;l{PulPLrg+9+0-@5r{Dcj`9vk^_aHvcp1b!|UY`G4pRW7hMXd0~tE4}euW0FS z{mMi5LsJxrDGbbu8J9a>`<#cil!}>bsp-w}_bg8JVomuNfAu3A=}SK%kXvD@%E!-6 zvHv0V|G|^i1^bsQxz>}J8SS3Ez==}+uIelpHVY*kfXq2sCD`%h$J&q$Vne9(H2bEvD^7D8eb2BP4V9O69w1NJlm{?$J@5n3O zDKZlYx!7b6VHx_NMWH=90gIc7C*>YZb4O z{II|H1G{J2?kb`d>*~(J7_cf*(jd!ebuJCJD+zNhc5gilm7LS-3-u9I)Jsz3(Xf?O zCE?md8bpwU?>WoqDTYjZeFZapkrNo1xuf>vD9XM61P*5?$@+S(EGb6`(gK08Ac;T*M5DkuyI3Jv`QeNxRi>+=GPI;$lJhLxg)MYdPA8*{1FDaYDF7xY`9M)fg`6}I|Ry!W@p?+{hbd79tz))+0Dyn=}oNB=^{ z9)ZYzA>>Ns{klyY?aPZU*Y93S`3O-morHyxl1{ID1!Kb2^Py7KN(JQP;8X1eJ~Gb?Cdbioi8abCF8 z(d7nDX)3AG=jdwSCzz7y6YjQgUoxbs%_0TO!9;u5seh{h9#k@jQA<4uWEy0;{%gUy zM3Kc|`n;?bZMe2COmrRlH?xZp$;K~&XtTeUouCx4r zpri_@lw|K8U+cV&<*L6cNu(!*|7(Gvqo?fG`nR6DlCO4K<0o$2qRJ=31#UBX-s=Y- zJ--vsym#{)xhJ262B(hCqD@FpX*J_Ol_xmQof3siU(0zCS8x{x=|f3&1HMO%zP=yv zzBtIol9N4L;DdNr3n>z}nj}S@@rN+(F;c}g1Jih)#yVJtOt9ASrI2?$_9r7Il2^e2Kb0=o!J-nen#%r= zb#w&Rj$edUz!K4Tb|L>`sSdzgzpsQhs+iJT$n8TCQUIadhN04mXvFuKYbImlWe(wL zyBc_BQuR;n#L})iEVOgmzopvrPSTXAIWw`4KnX>ZeEJXs*5vVshl0_29)0}9uk|Gn za~+Z&@b($8BWQldkmLeG)$N(AkR?0~8>*S~MYoHIltm7vYqBX__R5=Nwn(PACNCce zLu91UA78I+lTPnq=nvjRJJs6c>2aspl~|gd^93-?cDHBIFO(aKWw9og44#*Sp=Z~||ZZS48 zLT+eaep*qga)f{hu%#Do$&x+{mTb^Gab;xx-YfGC;>mPb67~{8*S=Y^k^3&frF%%t z#LFHQF@`xwGg-KPDM?hgaBAf03(pLLE1)Jx1g;FMz>3kHf`lZ+2L%ep0--eukC|d& zJ#Xg5-ZuLI&r;_Ju11I3@AdU*{k`QUZo9f)-DnRw{6Sx2UIRqY*<=WXTF&^E?ker# z4KCw++IKP$fYG;Ro_$F>+gPzk_-IpY)==)kdd4kZrKWZbcNE*#q#;M~dkl3Kg@&?A zpKOJ##ceROViWD#haBvQwt&8kdBwKzciOA#Hrj3LeYYvAV>^pFsqm?yZu%herh01% z-4C>vy?%e%Q>S6|IZuzC-_R}VGA0)&0kz(_oFU@6$#E~U2TsypEHD!&*`l2HZ_pE3 zv9&@6YuwA0oQ|=YsN!XO*?pK>FR9l`4<)o!Qv!{3(265?;uZ^X1r<@@(M>Dp-F(s2 zZk`(Zi`#uzieUMtAXSXYfdqxY#p7&N*2wzSD^@i8WxIMVA=}UOJRDszpeHC@`Bi zPItPRoCl9n>kgN)=MLwgQ|alsVx`MNJPF9k$`oek_ueIKVm+7{%58clcA0-nqN)95 zN~2~w3ms&YUy@8Q*DfLEiwIM6EQp1Lzzo*@h;mDgDJo^gQ4mB5ktuE#AJ#r0sr1S} zkq3xn?l>Uu4`iQ+vBqBR? zsU(<+JYN8VF?5LQM}NcCR!D;X*fuxn8t(z9qDGFRGQZBx?W=U8+ESNGUJLRpO+V0^ z*|nKSZMqjEf0Ty(X)bQbM1&uNa@$R6C`Hn8DvU7{rUEFh1kdv8t{L?1_V9Son?!z7 zc&p>ac?k+L0H0f}6%gZMFEa9tw;Dot4W;0%P^mVuDd3%0>3EsyZ?fie(IQE^CKDLH4y3QG5bop2~fXHAcqoni&h;`W_Xt&sme6#uttgNIT%=FV!#Dk^7vEh~^r0Cak#*-mNbq=$YAPfAnPOx^Z}4?z(F#XoeG z|37s02vg-ZLiuGszi#__a=Gq>P8+_dZU-x~X|1gITF`o!7zMOJ&V5xi0_2Y1&y2p_ zEx3g&IiN=9e2SW!Nu>WQ6(Us;=CLhL%3v11mFl`me-gZj;8%H6pbp{WImArBZvEy` zYZE2H!=MxxI4FjFGA(4?g*AE{-w08y6+|2P>S(X_m~ux>HX%w$q?5K;fZg1iB!2>T zUtwhhw0Vk&GFF zr{;|fEh|e-J$#F&3Iv0Jfmx|HrEa#{4mdqM1zmb}>RumcUvJr&-`0)N*L46LE-kI| zv4rkA)g&lQ%rBRTkJ77@>4a{xTH>bk?U8@JP+MVHhn}qg^7hyp2qFM zADmrt&tY=f@3W*#sUo4Wa87B}TyCrfOTMs`P)Q)@0gKXMHwO*!L)d!u_}PCvpJ;Ig zm-2)*{?Zl$2tdZEa)c5z+%L24d0b;*kqWvEupKrk3Wm=QFwXRT+mf#t2j!rPb9DHo zcLa&QpT$_%wt1l|TdllZ?~&`26sAv=ci9F)-L9-;c@PceD1WbJ{I({L!2im1?2YFm zl;uQE;|Kn#m*Xj?y*yWPREp+J_W?RPuL)9q;2Q#2`%chaO_z1jpowBR0m%e6CAC?) zYjC8pCj-1@1>R1n_cPwSCrSPt*g$lG2K7{Ls6&V?web<5NSa^Z4j3!0;0#TQQWr6= zzuCskrszCX3Ur3B8(ve!Nhk^HAtD34k^TLAr}QNfjob(qG~8<~y;a>zs{v;3?siLM zOpo?+sy>^)+`L_4)>@Rz6%G?=wVMsjmoC$Dcp1=6oaNZ9*Fv{-yCA;j>g*gALb+#i zvQ^Juh!`?yS=QmAt26>h3hVRl6^T-CM79*{Gn6yM$4pNl2x8H)iow4RfpeO*C0pivd$RhbP+g==L2iprYB5a_ypXwM!m9yE^%T4N5H zEUwI5M1GPha;QsgODj*#u>J8E(OZ*^FfzNw`VN-i9QOY!@;~6Qzb+42a6aSMy{)}V zo;@JP7&~rK^vw5Sv)tVHGk4t6$v8g2dx@Wq$3~Hu!lkC58HhPl;9W^C8f&wgGqGnd zZQ2v`AQ7%|w?lku^(#a$Ql1E@1L-|F?o@G(d^X7vfGpsxJK!}i#1mOH^QXoTQOIrv zV8?JTwG~zq$L6d8kxNT~>cvntFW3KOoAW}A;B_N=fh^Wn8uhCC!Wr%$K*2a!tGk@2 ziZj$n3{~~3@$W}{6;y>bSw&Dh;bL|0TpKNv(5^IEG?1{ti^xn(@2_Fo4I%jYV){Cc zT@#X#Edn=dOF%%*>2QULr17&>?rwI9%%87e68asuSU}&=Xz33NR!|2T@zt!|5MBKC zP^J_Plvw#hn#PBAZu$g%qb{3|hnt^3rVd-@xceO0sm=4^!h7-`n6teAmR6_9?>zkS zUv-g~R}?78`{R|NMvb`foSyVs0@5k>?9oM1sg5PVVI}Vj z0A0hG8_1K|b|W(f`GZ%P@3J2^$MDX2)3HX((c}h>ZHGtg@4h_V|Q4 zK3Z#FT2}6dXPRt2$dBVfb3BVq>-XzmW$CVsnGaS8)j8Fof!jIYjpMDbeO-Ef7z7>< z@Ooa@KSS?i$?Wg!iOI=S!b;@@Dt1Sn%j5KMH%;41U+rFLZ4Eb5)q6q%C3)_~y27a1 z<-zi>>hoPIU5p1^)?7KA@K$oRCk}&0$>K^46Rqk%>=zgUb&yhx2FHWQ|1}^a|JQ&p z=o>?O&39;HT&sK6lcV=?o+}Ysv0lR1UeI&t#f^S4Wu;lhrZoxn%^#E2iGrmt zjE#mA>N+cE3e_h!YCY6e)mP~HMTgtW7N8|s0)-JFV-B6~#eUJdeQb{yYc2gaL4=Mf zuQA;EG_W-m!nK19f1lJ0gj0s-Ai~g;gsCNtj?C&@FRSOd_x*(u4fdCy-Y=Hb_bRu0 zFtqKqVGnpoh(R%UOrYw&uiT899>)J(4-!BiWnx{Pmr=t~G}k8;m}MBZQq&V_H;T&< zuBj?{(V;)Fk{j14^z;oaj$iG0#}ziIXl~V#Y%zS-dex2eO_}|G9DYx;c1GR_A4uI9 zi{nM3_qvtj*bgqQza7hc(8}Mqg~{+uWElaPQ!=&xkZq(O=0M22w>K@1sWTH$sY9{) zI~_dcK&>t8h~{XzdUeya!#tkYrxB)SRj26OnV z=gs*qFZql8FE4q?bUhu!0LAJ(zOsJtv?)An{gU5u|LESpN5YxN{n&k)W)15hNtCID z%2MTh0IJO8pGu-7M#K=@O~P{Vy2Wp~l(x&5k&m;_dSXw$fcd+DL|G}P?v0`cA2Vy63-R%@=tpkxM{jlVjtBkS&b?2iqm)jk;NJNF zvRF<}%+Ebd^Bs?4oVa4*iUQujwVIR79Nw>jv>1-0k?_xBEY7aj3@+Tr6?hc1OcXy%so-!` z)Lb>u0gEmL$R}-%kdG)t8s9RC(}MfosN6`w3g0^PQ7@{lzK4nE5G~9{lRZytVFiFH z&)*pt42_2jU>_bXo#LN6%@5}e>ozTt##5lcfV-?^yI~LgE=2yXP1bL!qCNrm8zMl= zjVe4s-{0bxZUs=dypJ>(Rgp7dUF;TMT~V588qUnQr5#Qx7(1UWO83FPEwUenAxgH0>D7di z%vumcYh*Fah8>`h4?*~!t6k;c`D z)oIMVQRNt?ljWp^FZ$~bnIrBdW{r|}WpwOFk-QJ-7g`s(W z3*2UYPN^>>nVB2y^Ql@R&L8zc*|KH?dTxXLnE|nTa7dmYq_TkW9%{J-f2{wisQW5q ztFEjf>qxP_rhb%MzvmJa0@iQ>T50{Mu>v1&QI_zf&CHVm9UpvNpf`ka_rneZi`mzQ zN-8~G;7Q-NA%QS2YiM@fkmv9M($G#3wT{bk)y46jO{yY3q^(FzF`T9f}A*W;0J@Dy~Zn)eC zCH^RJ5H*8!1T6ZEIvu#4P(~6~CyTv7;JGfFDntc88VU+IZ6^rMREz$lberx%e? zW78%21$T>r$f(^H$u)IJFjx-N`=ir%KbLi?VuMwNt7&Xxwuw3W@=MaMC@^O|#dv^; zOCqo+aS}hVss^KSK>Lo)fHd2)Bn@vz9L?}nSdt2=>P^OET%5HC;Qj~pC@rNUMWy(D zxeweJ;%p>DJo}0}12KOux1nLd{i1MsII#fuf|jC-m>vu(LvN_XAwq5p>Zc|;HFzcb zqD?+k#q%M)FrfN2^ICCjEEKMV-WItd&)~Z-Lhy5>$H46!HI!Ch(>gh% z9QP14Lj76{sHe&h03KQ>J)l2A6HMWC*B%xQVf2YMRyh@-Lp)q=UTb-! zO8}MR&Dis_B8T7Zcxs8t`rvG)Cy3jo&?G>^xHOAQiqj6^_qnBX9~X}6iwKLm05ydW z<_mLNi1WYf5y)%~aHb3sn!#rJCQvx>Sq1{Ma4ye+(rqMfX1G3uA*G^@2>vjD%WMl4%G#Ax zP>B4}*ha*?U(Lx}v@yOK9S9&P`(aS=+nx9E1xP7~1sQb{oF!A1=Wa};%IsP&;AZ23 zCY1)$iTW?uN*yCLJ47{JNC33EN2%tYDG7wy18RN^&Wi{Y3mvyjtjU=fW4{rd(lEP^ z8zb*$Cv+h0;RHHN(YF=h!@he@@NqpzA`27k;CE+RKoCZ~B{EMC3nJ2=(B|$CJsLq?pYjGP zQvZzdx&dI$tX&oLx~#9K>#w|VGI250+tmKAr1Jk?Nd=0Asx`~BJ89!){qiA@<@or7 z3(b-WnxRa7vlEUT6sn3Oq$XTJHaXz(M^VDSodNpCQhvCc=DW=uK`BI$1w+8v@1 zFuwLspKr?gpk&PjZPut`ij@inP@XWlZL*mOFXZ56q=kV1ZwEBCDV+H-*rwlu01QjC zgd9;ZXcdB#yX@g;-!i@`JbpbvVG|}+u|&#)MIvPko$)ADWJbDvm$2qS5BWFgMb09p zVag7MDIxV+MVu1NV)k;hGkRHYwcX;y6TfJzka1aZD5lpO(qC*$EcK|;X|Q{S=2gSS z`0gZlzTaDY3?I60CNx9}>OPPS*5d-|qe!s9$6*N$L7;0LBd`$UP}|q@Nu!>#ced}$ z!jc}I#nE*1OFr7`;NE=!%}ZmVcKgu+m!FqDX~o0o3(0ASD*c1?NoML7df2b94tpZo za+kR3KsiKxadx=L%d5FkY$oz|og7zX#APy`|9U(GBmU={;rYixkusDOne-zgGF^#24TX3A~`zbb}^{XXQquUxx9IqR81iu@#6u$%YiOUhSGOdY` z3lMJ^!;SXBGH!zox=s%f37=^FLqfbb4kG2asWdf#=Y2FrXySdhxMp!PE6|da)!O>& zI#kvHPGZ)Y*OKi|aM-C{AkXnZPzHk8Rd9=~er3T4g1(>9j{xoFrkl$3(D*R+>`eip zaUs;MCiAUhJ(8xV?-6idO zL9+?=V6#~lQ#b^|vlH9*3{pMHUZ_8j7PYllmgEJ8!6k}jTcTJ^wOsqST9FfV{W>!R z@T$P?E-RKhtP}6J3|`+D;B5v0Ix&2c|CSr*6LETKp;=IvFz5G~{$Nh}Aee;gQcm)r z7A_3Pe#PY``(rkR+mpzeN+4OplRK{tJatrW4N#b;9c$}u$wd}P1T6+@CVTgw#lT|R!{tJKpDde?C?L}F+EW93AB)~SUq8gRr zG$uErGGeXF8VHWJClhl`6GF)q4z@&Tlc23r21yQpp!6-XNPTWolA{WmrminKm}o{O z^pQb80A_9suw%p`=L|o*iYTF?5(<81R+G)UqD+~Af=oA#6l;3$e4(xbU=e>tb``c%a-_2Pc)OAGQ`s*J^dgo+lx4h+oM7KOFg1d z;gXJ2Rc>uGLq%V0J<~|a)2UtDMkD*SB|cfdqC#mX*5jgH(h==*OUiREDn3lb+E33> zzxFUC8GuuOOTj7Os6p0x6idbyOPzGYOUf23u!V!u{>p3jJmEc~^)Iz@-27j&!@K>= z*u4amB%4>s;;wiCT;J?A!~4kG@qpR5uQQ8E_~=|F5wT7^d^tYnuA*apu?DJE9>EXjToCM@O{knuSq=UHhs9k8Uy(DR##kf?LQS|wmstZ>Se?JK@{518qXiI0 z36($zo)UAZ$oVsWv69WgUS8=3fFd>X~WWJ6K0zzAZX=%7V zfe?p@E8NuBrV*Pm(<0I=N+MPz5f)DBLlzC}@kKF_FRv1ZRGs*|687x9B}nT6x6clV!#? zF2B^k(KnQmlq7`aBU-b^2N82)vl+jb(h6o7R9kq~Wp8qtKyS6yDg4|8iT6-s12a7{yZ;^lD<&4UnI&o9i}9GN&Rtj9)}DBg(+-x7k=OfRWgCAm z{25L&@yGX^W9y+{W_O&rcH|_IbX2`YByFB6Ql~UB-Cy2I2Ox2F>0%Hh|Cu2zH)iqx zn{>3VBse-zypDpX%LG~+D%Cg-D_)D!w<}INbV`mN8J6veisybth11~pqcF%VU}TUN z8RR5ZSVkXIMhC>%jQ|Ob028cU*ttR$o{%eaURAK8+r109tjY+gg$$|{_JOT~5+g`e z_!mZCfZqspdjCwrW>Yuh;zjhfZ;Td2cI|ttsonM`+$v}>cLplIo7*f&46m7+o&Bkd zl~%@0E~ zqca{3_os@#5roFk>_V|DW8X^jfTlU`_l;lBR%QXgCEAvaK;|ni{N(xYdE-D=;XyqF zwAz$!|1qo5^^z_InfMHBoEs#c=|RkUo_~rzg2CYRCiV1)2#gj);renW2u=J!g|Ce~ zQHPp-0Oej3HaCB3r7I-yjgsjFng58cGb0*Wa@CA`mMjo)pH2DttD}><6?&;veiq&cnw`N}kc<5USqvwyRBU%{-e9 zxtxOp8rZ?jDQ9Kx{WC$-Hl~(M?4lU6C&4HFhsQ2UlK+$~@&6b^q6&KeWW2ODZQDzfYo?a6inW8Ye*;d3z#EvfV4j z)wD>>do|>PT@c!!O}$)54OhkYJP?DUEa;4n_B2t9Ts*I3?|G5-uv#!{DOxhnKi+Jz zOo~*Rz8cpP_g3E|>U+r2dj6~)=v14;Gd^AUWw&~AbFasa`Y$H{{okD6W2jbnu6FZQ zEp;nvd0TScyuCm{cfjXrI6!qQC?X`u34+)KZeUA|4DF7rMOrGx+J?f@ng!_!9dXuS zu;iVy5Y}>qRdN~c2FhtY+Rb9Wt$dx$f*Q+eW!sx!o?9M-O-#MrrQ>$am4En&8n)M)rr_$iN%v#vt+t|bWOQADN-@kAAN)N~ ziT9bia#``^Eb(2fyOq=i7dXEbU*j%zP0&STW)di<92`;C#_aR1YqkQ75?Gh7h{^!i(CS9V^o*jn#M zmur4QYttUYjm)diy91e3mKxTyK@HP#emC8k@_7Tg%#|RD9Dc!!qEo!8tdEr?-s+R&EgvCQ35eg5$@YDdz#tu4o)QiXX<=$cm>AT#j2$3mMwoK96(@+mt`0$DNq zcgxkB`R13j*M`Ku0M85dACDJ)k`}p#gr41Kkhd79Dp^;2g(oeA&ifTi^(y@;9okY60*%xOUA>JYKUx$k{n*-Gpdvrse_~mYT6Q( zB*zcohPG7Qp_=Mz)1vLO9Ly>6$%F770Z7lt7*bW&jgc!2`hcreLrLmFwsW6=bQqHN z3n(B)NHHag3aPp8xH(SN>SJeC512}buOh01kTOS19#*PACDTgK8=brS`KwW@b@@#A zrt_E|%3zoN^lj6reZyn18&qT3ESw)mD{m6^gK-OVP;dfnk&LqK12abk%7qDYEi!s zde>*R2v%CS!1(l_H?$hg+wXF#oRuV!2_F-4)%$g9M8Equ=Gzb=8_bCR4U~oZ#v|mQ zCH=TFycs4!DBWtHriVGVQ`!#8;eC8vN|X83b>bQem<KnSdOZaaNzH$as zD7c4wd@YDgOFs3ZLQ`r@CMfA5@zORalR~q6UJ@0N;&#|Ab-@QoiO);7KZO}-1mUPk z>RL-A%?B>$GV-OgP&bB%0Sf^G4IKYix*8IObG4#=pp{HFe2}j=*-a#ID8hGxoHH)9 zHrCeLg$~0ztvciLL3z6w2+@!nzJA^uIx8N1r*Nls6$o=jSLl<4aB3^t`6GQT=LT2FHv!{dGxUi(Sf6pxf29R7eAj;|18Y(z~x3vD>Z;{m#-v zSHS<93Kv0pl}u3=DHo0pHSC#6#eFercDRn?y)KBAvPQ`aN}#0OiY#m1s-!7DLlkLF z8SAAhz*n#goHKp1pdd^!#@~<8$xg!F{aZKWd(G=ZQ;}>LsY;!;TLboDXuV<&8Ow8P zfBg|Z_#;Mu@;M`$L}{PufD=6Z4LvnFc2ArC3vTQE>PwJ|J*>|-PSjite7<52Drc{^ zptrK&G>@b3O8#)XM<60l4n27E&+HW8krQmQQnvT-ocsB9ZcHiPJ%-6@PPVO4MT*PK!LoG^}*F?*|i-?4WTz_^hyxJkM0?yV0xJ;_+2#l;@DSte*4>1sq9V#k-N zPvtax?I&1xDtb*==J}GXrS>7H$3zBi69_`v?|qFyBSJVQ6KYhKl=mWSuf`MrRvK-(MU?%kB>?Ic0xR zdlZmyL@<|BNSzumU3hwgBdlxic$KSmkn{%+qX_~F#%G=%Ky$|?x$g5B&wkMUQ*R!j z|3_&i$qko&wpE(x-Gwccjd`)Ata6@~&R&9ExbvJF7Ioe}Uflb`%I!2R;q^d$3IW>T z)=?@atS_G!Aj6Ic#2FoFgyiSEhBXiebjJ&ol9G|A=93UvnhX!2tj%k-InyUG;P(y< zNdck=)fKB{BB@K}mI$RIe~c#Y#v8l{e0*)9_DyMmaamE{q;MbSD31{7Q&S~OZI*Mz zr{udYraOrGVkIhvgHmHFoXnrJ(<)8CvDKj!W!a-`e3TCEz#wu%F6$>%l|wa@&@_hr z#yGM`EWt_0{U)=R9WKz1rc!!g(r(RWBwUb2yfTHVpeTdwr0t{Y;^$yIqsRJDZ|K*~ zuO^Rid``QV^2@hrw2C?!8sQQwor393c!jf}BC8>4)9ce+-Fi;T^ks1HHCJ4{Qu^G) zhzg?`GWc=!d zbBUxcf_r9uvzIXvr+hulL9vH_{ruX;l4SaGe22~dD0l?E|4tKk!YZjj%U9n3-rCe& znhnb?Gv^}#%}r;6Lw>N&+3H}++Ep+0Uq5~>R)a`+Q{|p#$Y6F>PYH>Hi+ZKSSK$7j zyn)?lu)jv6!5~}ROW}iC1o`^I()>zOL%~T&@HY7g9a5P{sQR*^BHmf3SiXe729u@BkKGI*!rJ5+Xo%*pxHYE0mtfO(7S*P`S5tP|tt zQhbM~A~>W{81P~)m9jY#Nb8gu`gS}#hs}Qd7l$@9PyQW_F}vbUO3@DWs8*#UD(-5K zT@CGt6g@8ZfXAVNsR`!2DUV4HA(srgy%)&q}I1~t)t~b=#6;0q>v$kaS<|| z`K`0;Vc+)9l|U7iLn$xtK(uv6YJ7tR2l*KWSJd+N2{s6;N`O8Z`G{^J>BAkD7l14g zw`RZU4W)_(VP8iB!Nk*#pz|?&32Eqdr`aU>ZB91xmh4CI*Md2+eawFiF+grsP)k_l zl@9tOl3-b!nL%@F23$dWj<0JWmgrB{0F)E#tqdveuNVlUQhr!a)pN`k%Usbai!@(S zyl;xm@|UT1s0^>pB2H}vKt&97GBGR1HmNjgI-TE#UrDpKr^la+Q-SxLjg5t6zPyrq zG_Eq^J45GN>HCbT$ucqRRVJbrM%McdC-d(o;-iP5Q9t9)Ms2OChE_w`4XfOiUv$Md z)g<~n!HCbgPk;p=kLk{J-P#$o)w(BXFDd<`SghT8xO@`HNwIFgl}k%hOS=m1V&zE^1nZmYSE?sVi$+zTU`^?&ua?bwsoTu9 zJEMx(NEVUG%jOq!^;jX2N`oo-3}8P+cY0u{Gb|TH1ed=M7cPRbF3S9nh*LcJP=*LA z5XvJ~RnFfLQAas;)o;JR*8l_SpE$#36+*Tm1)@K|gc{`v)Q5l9 z-igl3MPP=)_M9gchhd3Mx>etDKu1&z7r=zBv-i)+;_3BNF&>g4H;$pBczQfmct1iv zXhSZviXf9xta&d1#u6!brYbLb2~jOd=BSNM8nAZw6IraPz@m zR}Hi)qa$8=7QWea$5ZYfjD>XGSZvn50yY7EdYyncMx9Q+1%oD!gcDAhLI!htK`4+3 zEd9bIBr|OpOGHR+_WJ;h?Wf)7-kcEpa5{PW`FRZ!cNsJ4a}6w9+aZr@;E=3(|N7E?xu(7Y$DF#{29_L+Gz#U_UkIbAkK~}Ikx@y z-i>&i+#{g~gKt)!+$b!c`MxZ;!ZW(1f0Cv)sL9xFza(tbae9pPE``{T$V&UOh@OgSKT>Pa6Aw}onO zJgt~Y!q2MNs-QG3-Z%u$Z`r2w+*Q2e{Cjo}ynd0Ylagk4mt|5}rgk!>i~XJDQOFG| z{oHPF`%>SqQq_3<3}m9y7%jez40J8cQZiI5R7A*%@3kDt(}NKw{44y;gT(*A@KSN6 zxN$o_0mOZ#Vy9bzd+dCu%J0?s#9!uxt4uGj^O*0Nh!`m3e~_S@0l-gKXD0(E;V}K% z%iZ{svQ51yY&Cw5v+7#!GKBalbU1R%}&OR;1i7F)#ZwTP%En9?8x( zFV6>7gWkB^=mAg3M2N(-cWIJI#&Y_>Y{RCcKfYWDl{dgGeF zZ3+e=faQ?{z|moq^EfD1O{&Wyrz3KhN$+p;4yYynak9oyAsQ-n_ym_pqzN&eta=V8 z_*M%1DXQY(!2wn#88?DHy2?i=#fmv*Ig4K8y^9yLj9cW{#%;IzT}BwYKge>mR_SKL zw@)>edAzc@g#uvRz%Qa{*`y>g4--_UM|S1T=bpFgy` zW1Bc;tKQ*JBP90CFW7JE(SMk9Jkp>7WtuDa7-+UjhAjK+{coiGBS>#kJDzhWa2`uS4&7(bm}~Deyv0$L-FJG4z<1&AjJF z%IQrGPVRP2O&6M*}4AIwjNp$f9sYFNgD+;8d zS8X3f%j~a;+)+^}29qDHAO`nl&nQ)O(sfH87T>705;PsO+osx8d{R&HlekhnnLP~IUjbAiicv^DJQS!8cm=N z@JUGR#YhU$jeO~nMIzWGD*-3pZz`iQAGcNwyU6<5a$|{xW#Ol2hv4itn-rWb&MNF~ z-9@Apk089$tkGg0~=$2{>hq2_c; zZC}ybDK%*#mH{Hpa1gL5{zx@gwRHXD8XFNlpIhck&U;3wb5u^;8Wq2@@8Y|DEEHrG z6QVz%|JLDwN$_OJ;+&rFW%Rn@c6}jLSfYGKXXO;KdSX$UAY$IYhPbaXL5DQ^p_iM4 zYFKs_ELCjhW3y3fQ9j>z2bGgtNO}OOG82{MMr)-7HIPKMg|1lNQvKF@KaPs1B#upaG*)l(@kQ$Ka5aU87K z<`n3*R%mJVY*fbC{zzy5w8>(_`lqi&6!{liKdBLr9w880Zu|gO!xmSU+#dC6Wh;2& zQ{%t_r-@p*31ClKv^I{E1e+0c{t?U3Ye9DbcK?S)u0A-%)H+lj#xz*B*dXQJpZPMQ zbz$8+jMBxN_L8TNW&~6`_E9hjI*hkbafki$fjEMJP@-uumIf0Rn%v3A-@KB6#joZ; zlxda<4-rt4id6?Lm*+x2$+3f9@FE7e0PM`^d(&Q3#GFSN zEty@dVH5&h%Sbl_&x2=iXQtOERpKtI%ZMIVpXc_Z|C`4R|C`4NnqU&FS(xNh6m*Ld zt>FmR-58fd%e}VDD;6a*|L{LSo(H*~8{RLjDz5L8FX~lD{6Yj9=U`H#{cL?Jiu8R; z_wqsJKT2-8X&DOOvo>}m!8xQ8-Z(dMcP1!Hj=cNa5RL-vG5{WLGE0?>2heA)1`VY$ zK!~S69EDC~h{{r=l#t1!M8xC>#gb7SfKTxKfbhQZVeF}$e%Nq|8;YYB1a16)mx3am zDG-tl1SpOpw)n8xieiPM$iu&S)3o|^!>jdY_yGfn@Rn)_ORl$(&o6$kX>+`>)JYsPuP#&{V$a^qZLSZ;rvO;hU zh331nYR*UUbV}G-c8%~B4eGkpT|4xS`yf=T^68;pCPDcVi_A*mA0r`bjM~ne*s<8V zgOyQ##w+R2XcX(2AsD)2tTsmP>PuFhd5}S|&d~LqZoGVx+XO;X^d{x#YWfc0P|2FS z4XYx1f+}QyWylc0=4(@ny60c$`V9)+X?pI8P7Z?X=bRy!*#tMg)0VPXwt|beyWD{) zVl0T`D*b+xIzTlZQ>2%~s!V;gOHRmiXx|ATzj@+t=k$Xm3M3b^(wgHbtXe^3#Fd|W za@8b}d}YIOLLfyydMVa*xDQ5iUgR)COn^P8UbiZqCZ4OOJq+I)(=0$z9(Od7n~QbH|Q zS#0~WgH>a$Y!FG0)Gn^xB}fXK`4?rp8~%p=N1r$p`oE6feNF0fs5W^EvB9sg!+E~R zET!&77ncdV0UYY{_r2FE7Cytq!{)Ckym&VycGzbS1+pj_L>v^N#Ep5!GQk>@2g#>S zSQ7Q2X12k;y$=l)Cm#@qphK^7$>y`TO_Qs?fFpXx{OKb2DV2&^qnwO+f+ZtLFUWT- zssafTCFpEU#JB7Ci7xYx^OT>a9!4nIW1?&J0!$vPG2eyItl6vbsf&F^-qqkIw&o|v zyq^IZWdSLhM(tMUw82Qg z#v38AK+!-JLxoH1*2s@2dMn14X=3fwctv20CeyFvNUL>VSTiRtk8 z)EZak9YiUK#Jdw&Dg2n&4#FYFd3`@RSnbqWfbQcVZ3XQ&a5Z(dOL*P2d8sG*zht3% z29|dFPhdR=cn&k_sHK^b3rE1ZCa0YTQo=#D%+j`dTf#t=0xH(W+Myd^vX=Pkly8Y9 z9{L}x)w?<#^iC7eRZJSC5MNC3!V7h$*=SoTE3zjuQ^gL;Pd@R z`9n1(gMBvXB>_;D99F%9f*(KN@qu{#x*Ab>S~PXyB%D?!F?fb1l_ioo>h06CgVO#H zXxuPUI#gd8uwFz0j~C{b_$ACv5O%X3v2<(S4TUlX$%gqGg3h9%pflb>7}#+ei&Rse z(wk-{z!!3|ixJH=zwB>3=f0bE(axHr2X?uu{$-vuYJQs*X3=1O@@}J9lyJ)|KHuXH zwC=^@J-Z&^pv4Qf$-97ED8;xpi3+I0KM_!FS$|@*j=2DJRh!d`T~ywzkJ zHOXfsfJXM&Oe{kFJt=&;j}(1+h!mx&ARNjhl~xMR{p-CKaPhgH{C=&P!X^-AuFcN;LI2;;=AZZL0nA_TKKXQToBpX&?S^ zv}u}gxcZCYy<$QGO~D9`oKJRzF5AY7`uC}T3&1q^s7-nv(?&T6js6!->qOOXbm9D? z5qU_~Pow=>a0(AsOyj$1&P*^@irTg06^ZO~Re6etsosp3OJRV|Q(ed|zR&Al{^$;= zf|`#%LM~u25ykr!sk|$H2F9%PH|j+yQ&hUs9joU>HGfj$PD>sQ_O`;pqY`P&{WWQC*ZE8j!3n$c6*Om zyj)eC4W<^Wh>6}sq17xmLMV_UnW;RjM3&_-krSJXIWyQ_~U+l za33enlh2#{|AMJRLjO~47P}Ls?n_i-5KCb>k#urr`aYU70^mk1aX^{J0+=aT;HkNLqI&5cxB)lJG&eW>B&QHavUnt~x^Kr~fjS1R0shH!-DcBQaew;8LD z_8>aqCdd%O$t6jJqeS|_H?21U50<{`za6ADJS{hQ2QJi<#ZJZ(7k$So*y@r`2Tzy~ z?L6+Q5!y`UFC2;^aL+)qbvu5)Ug-4f%XE_3qvG_${@Vfz1K=G#mSZ?Vzl`bL>B#o} zYe|_tnS(Waw+ucgN4`%Tc(*9u*02~v5OBhtX>!m> zFxaa`njUyJ2SSH0ABlST=L4v4Z@0)w8an3U&!Dpt>r*%iS$5O)c2sIomY@~k#w-?n zzs~!7yw~wFvS1*O+W!L3~xrOep zParG-?P&Oh=#8;#^uag22&OfSfUBQ~t>;yEA$~E7jlcA1lwq^E$!5_|RFnckl#Xri zyL)w9${uJcr*5pN4;jkJ3f(X(B!XKic&CmZrfk^ltMk$@UEfy)Lj8J^a&`YEhr|7P zr`E^|Nc6fTFPr5gfDtKW8^X=z%-W{+`9NyFEyGkTSE{YfGJm2qc9cN`Lfn8^NE|TZ6aB4(*dTZ$L5XH- zL#O=yO0)f7tOkn&5J&sg@p7SKLrNz2YPH#E;@yg5D|*cXJPcg1_2Q=7z~sz40F&@z zVpL5Fk55K&Q*dwdRbw~l4m&yFQs?PvjQd28Qh6?=Vs*ShNyAX_yj)>!|Kv3JdtvI~ zxcJ>gK^t{iYv}<>(e0*9|1EuRP=@NC^v)~%@1%r5$x+;888<6aeCMk~=(;CStgv-C zJpQIM?9-q5La${AO4O*WBQ0HmMN=JeZ@5|bBD%!ck!z8)>2R}*^-VjV%U=Q2el(H2 zyI46Ouy?Bh`P8uJrI-*Xegf7KVis=AOJUb%RI_*5CisAE+QHCwgjLOyNA~_@lLdXs z-JJs}#eVTi28csG!q=;#@7~Ez!QL13hxp=&C(aRziuE))SADIdy^y4g#;-mdXthxH ztDumm495sY?6dVM5yoBgg+N1Kjyb^nBW5pc_`dh&4-82Jw;`FGVC$g1KiX!TECB$fh-TohTqTZ%~_-E(0kAcYjAEnc;?0(qp zP^c4mhoN-?sPde($e4s@0?lpREApv<1>GJY_NWkmJz-7_cAM(DU((+Lbk^$8LR_N^ zvkOn3f>bDnKKw>UB9kfEIf5ameyBLqWc?0|{h4JejV2yunjNXLu*_A+{M;1PZQ;r`C{e9K;cvMBPKSfOZuPa zyiKUUkW$MCUy%}vHGv>rU?Q<_(-*Q5^@WXQl%^g%er6PrD6 zK|W3?WozsuAx(u}#t20Bq*H8P{s?-m3_gnguY-XC`xgSPfvyGblwtwMsPbSA33XC{ zl#yM9^of6NDZc+Y719Pz@({o>4=Jp@V;zv6 z3JgRpXS*tsZrk28$&bz!`=gT0&*htD14M^nr84xMeVy!`nn4DP08}inN62PbTstz} zlE)c&k_WbuX1v6ra;2u|d=WdhYuYwrjWPBkFTo0{zA@3%C7#i=OELsH$RHCOXe-;7 zDl0Z;YDBGi?-=|zP}P~4BLq1#*yZ+s?6iE`{Rv}x;!duR=d=GTX}diQp0^0{b*Ic=WUgVlzf7m`7Vz>$0j~Pm?c@$ zxx_z@UQ^)q9r|Ya2#pF(eoa#)z-a_8mD9(+hE)uomm}|OYjr=UYaj(w!^UPuRq%_C z!NC#B7A_z*ej&OCp}@aefBAlDX@G;h@lPMGCnhHz_Vv{SUl-B+OSTU;T%%0Qkq_+N z3{4IzZ$~T~Jp3lDC#OWJb!e+WklG8eoKhtYfNP7<&-M3ubkw%*CC#>XD`zyn zDR_uQ-F~Ucb6Y(TU5#5#wU$*_V6)$+{lcnTU!dM2MpNj@dHu=Xs{Q zmK?`m2RuwwvO4(I_?^)6uD|kbb9eD~Z)`W!3}uAYj%w2)%p>hxOmDWrTjC(2O7}V( zLVd$CkFzfi$ed7WL2KE|_=5O3h<-0!GfEO{nLW&agzgj#q?4*058Pg1@YP_xnj?Gy0xjV+BY%n}&&mU?6fX6>R?o|GN4p_X zjcwnmCXEdDf8R+F$8rFs(s*u$v6*kW76OegI!F}l4my&y_a|E}FPw~Rz(&$OQHLP? zL{j``+z^ALtqv1Yy1ZC1(#98<63Vc8d|NgzFd$4cL1%=#^f-H3V4ic7v9IxuS|<2^ zwJ(g(%AdJqJSJ*hs9L@#BYa?fxffUf{I>;)*9{WaiZ_j?*B848g)VhHRC^*roPk%X zOut;uUIiG-2KBKRBUR$PbQ_FSAiU5aZJS0YZ(s9B`}shn1JHh~nuAwv@c34Jm~jK` zceD4*WPdfbKR(jjEC_ArJ@ypwzt7vcK4-~YI0gJuGNF)-Y5vEWCOJxXnXn$GTNaw9 zig-?M3Liw92A1|zY^@-+R0Vli1M_?|mt{`%l%z?SQ9i5XI~YH*Csv_X>wIRjN>hV| zFp0Ef{rp6~>A8slwzg6(_+Z&~4R5~qY8U8vcmDd%okTJD2ghyX)EqOxW7*l-xQm;v zcBLFZ`)S)41;sq8hJBi98NuPZ=cxkS3ipA(y)-eUBLB`sP4;!9apeKHmD%J!I6{Gv48uUK;j`yn4HFJ)>5q6}OVKREOnzvkA%b=tB$ zEKUw~hrTrO@KIF@xYSwxo9XQOcocGEJT(?=n{s}$xq*g9`7B{rG=AsuQ)*Zec8UXt z%H`CpH(AN6XUkwmd?GXNw!xSW`GO-SO2Vh?*Uvq~%&Tg`iy!}<)qiZ%8dPXMTQ(?Z zc_GA!w_gIT=hUPpvRLXjt*@273*OW32kjzyrs&{qmFqaa`;idQ>=W=CWIEr-0K^4H z#~&(&_^FAIA!^ALxv~l?UQ4jhCkDseM(P%-uxqUQ<}*^)bq31HsR3{uxqUvFF5e(E zB1G{ZSKGSS;kea{DhBt z2rYlfK8Ph@A$>E*hzgG@JKPQe5w zZ%~z6|Ad)4UeGd)$gOi9`>!9=^MS(f9WyqcegtSNgOJzGWRY%oq17vHv+z@Yu_7U@ zw;0Bl)+f-&P740cZM<_+&}RUO^Jy3Yp8@%)g+ zORtYIMWKlgX4uxR%qP88LJXWPW~h65Hq(cff@#ld3jz*uk}nB2s@=m}2Ta_Ub2y>)UdEwO{D@98 z;y5Omu!<{VaGl4H_hG7&o27Mm|G6U-?=BG5pNMqv)MD5Fwp8`kz7ZS&?zW@wsbNfD zuZgM*`nOf|&6*K-vb?N4y5-#BQ;RvELW~a6PaV8SWZt)`+dL(Mnc_ zo^f1rj*&^;qJuSV7DpyZcHb~=Kia1Ht7ln_EtBMEkp%frwNDqrOJPa2WVC+#XwRpB zPL~60N^f@qHk_|QV$bv+l+XK5_kZvwx1Bj2olVtpO;aQ0_u&@p(tED@ZP%o?MQ78~ zhn^Ei$J?*Sd$y<2Mz5Ws`T)=&$b1XZU*CzTOK^g(or`59rC%N+>9Jo=>$Yhse(5|* zp-H3aO?R0CU*Ya<0z?zv=l?K>@@xuk{FL^Ih-1UzBe@UG4C#}p5T=TUyio|3xY;ibMEmt5Uh8?B|R{_9_!#C~yUM*Jc?gfw9ObQoT2kG=8$Z+~YZu zOurM=gAG_#>>RbB9G{O4_Mr!)P6bX)C>MD`q*pzT+*AaH8~e9}fW;jw$es5{8R z`U^3@E0 zM*R=5*AuEj610r^v8b8~DO|beW{>~n5<%%z#eM^?&TQFO;!)MAHY^3d(5Tb$C5z~U zE^C`JEw9p|(&l9GJ&X5K zvbiW=;NTN@XU&$(s9ZM3!3Q6fL&WGiU%Y6ROB1mNkP`Ul z*XU`8B+p8wc1wNEzEb+(nDR6Iq@Bb9=K$92iP0Gb497TGv#BT?;m5Iz=Y1-(IAI*( z6p)m?ShRoWXb9iu-QT=pzhKgB<_>7d^;H9QwnYC$8=MgiDXBSfhYCW3!+R>@WUs{A z4V4>HlX{dDrS6u)8&)ab#XmT7Sc8ILkX}j!U6mF>yPTk<#B?FtMq zp)lRm+-<-}1z~(QXgv5m&wC6^dJC<-Z-`&`j%5)(xTjWk>;P? z;CD%T-d8;Oa`j9hF^GRjN1}Xba7ScxChiAeOO0ngk0NFq~Ec=)+IVE zap~lWUiSW-J;)6r&}NCWtfZ{A2X;-`;CAtL`cMHI%*K*nfYWD`yW!t$F$`C${xI!nv#+2=SjQOa?xR8~q0GS`G@QN~I;)dcJi;b*_w zF8#zx5BXD}Z%m1NYgL0Oh7d5M0>4H+7+~DVR-2q_xLT}Cq6Qt;1y|668(a!m7;Jcg zTK5Z~X!nC{aF-b=k>SMhkC!2Iq&Hc_aTY=RiP3FsLwVF!L)EUb0iXRN=Ca`x$sMa z8hG26iS4KHb5g099R*_cnOv9G_(|`+(~nzvACGIG)&G8U!~cLwXJm0|;2h2f$26E5 z4J;Yjn+HRnl?ka%(c%T6PaD(cT2ISe+xZ(IZaouri*;#jQW;nhli~e|X|S*n*pai} zk3nI0e1-tTWUFl8;W4e_N*EkzI+qx@Pjdt&YPN>EWppIj&U*XTvpjbDA*sOtKk}|> zUGs-aavnEJMDF4@6bS~`{FrYUVRMk5pxEL0;*hvckDUBqqF|937piO52xN7qAoawi zrLiO&Uksp11-oG?g5_-WEwH-LYQy7+H^#kfwWgYRwm2O1=mMkW1mxRTM$zhSOCb^W zthB}e+DTjokE;o>ImnlZE5D$Z11vNI3wSBFmp+D}mN)MuxF~X%lsjz3SMk`uF^V8$ zri*rJrc1ormN72(uXJ{@;?BBq#G4Cqh{=9r>r(s&Yhs`q9bkIh`W z(V9WD!q(pIJ%GGt*dzM8=t$L+~a}$muU^0w>{Ozb`M~QZCWf9U0bYH zQ5wOIB#ZyNURB_=elX+y*=yPE>x%Ss|F`IR#sicYMlR6>6gei;qDV>4RfgXzmcNO} zpM>e%R}!;{IEje|DK}YJ?Jxk*tU$3*xmKX3#Zc?RHz8K?kZ2VGf5JnRBB1*JrGclp zs)0+}R?L7V@`Xu`pr?o>AO}bZ%aFTV=WXf-F#0@ZHZ?z=XXyr$S!O4gC{QcNAd7YX zeX~{xmjkhVCjvP(`Aac0idM97s`PekqPTqg(YFUc;$?s=29=psy$$tOY9q`7FUwbS z1EO?7EPsFj@r^+wG@tH3y+ZqW)?DC2+4i&UP~eBb<^K#=vj5-B$he(IkplOg z2j4eU?GLX<*-{fN?N30nO6wAu<9sjoHft(bt&BoqE@7^XmkYP$IX6Y&p;8hHI=YW3 z=flS0fzKUUc5l4FEP;dzE-$%;XD-{&>@!uC9HIz~V@3*k5Dyy=!X^CJTf7?G)Q|Ll z{pl*;^nn-~tDU!6Lyb#NV+an%u|?Ud)TF}L!Qc-1dHtD5HmicMZeR<$-#Q8UQ>GwT z=o|6xiY2AdB?&*1XNG6_B%GcV2MfQ}tqfWhe@CcF%a{=g0iWmaHb#w-v(1l(025p7 zYa#*7`@=B!8!Rs7LCug>9{{0V8xYuy%SUNjT=sjf;IR5Ni2e31z+1t}t+1#a+`$g3 zkP3zIymX?5xdZlwLP3tIqT^~iSy`tK>)C~O zL9lXB^L1F)!C#+F)xNN|KCD0)C#<&bN8Xu~OB{^V&NRI)-cHc^j}3sQg%#rbx&6Kp z=?8ppgr2vhp&@jir#%P#m&q@jY)xQ8_@BEW9Qp6s#u?;|iYFVsT+`li2=fWNacTd8uvLuV&8JkV;l?xRf!Df)K3^U0X7~?HsijVkxIG zSQM_eh}PFizR^L1KmIb539Vn@Qb^_0XQAY-c6ZGfwIAyd_Oo+V4f20$vRaBTK2;|J zLr4%wY|00i>SXk(C2fx$t2>g$9EZ6)q=f9+BXa$|&QN}}6s!buWu4k9=#l-#`di3r zIQa8@()t~Ki%m!z9H~)5`FFD|R)=qA=as+lB_JQFExU`!tD3Lilv6QM)6x6!J!bMg zn?@-E#%@a2=?|p&-OE&kln^%;bnT2 zUuO;qqH#F;x*mpGg@WD#s0KF%w$EwjukMKd1Cn_E1ExKil|pM}O@mde>^k8Z+ie2i z9|Elc5DT-JCim8Cdu0`mB@O>}B}0%q$-uM;&Z`?jDWU#20cbo>mT*+Rn1*hPesi}5 z4$6tVcg_E$Znc}kM#NDGEWoH9f!TPW)gk8qmsCFZ36Hpy`1vHT_IuFJfO`LS^QGjC zv4&?NU7i@2I|)P@I6>>Xhi!icL@yr$`;rJrwTMKhoCLs6Pi>zEzx zCacIxOy|))5atOqyswUrWgycM-(J`dimdGXZ06f%umjYokj$2T9p-S5vCBZ7(st8) zE7ALDB9q6mZDMp#W0Xr5*6qc)w%^;q6)7`7n>cns&zH}+fwBOiYTL!=vg#t>8$3n? zh$7~eYn8Iwto~sTY1|7Rplot-;R@^MxWsZB6 z&dSMZ*=RfwpcpKbss_@bvK58Bb=#>G?(_C#N%pC&@+*NVl5$2sB6p7lZrseFw=T=| zA?tH;Ro|^QqdM&$06T!-Uxw@wg5v~lk(}?RHGAyrtvf{pu0sJ8G~pjh*_@k4xrR`? z6hP;JuyoMbQJsjc7S3T{*hUe8MHy}RK@-brUVR`qk?5!&I#Nh_D9bOB*l%2j?7A`L zeOM!CYLv`ZG>{xv_5fv?e5oJDTEja(t^@fbSEPOv_;-50uh!VVvVDyp;$eh(ce3zw zxDs(LcP&Q8Q2G|i#bQ!1F-^fB1Wp4Me0Lim^sS83X}(K$^hEdx1F4DLqhqKJ@E5W zp`|zWsz@2X=xv5>mK1&TBv|ry|?Rz7PhjaC`@OG6WKK&EpF%7~(*x(H8YQ``5axr@x(_XPuqfGrZ=K z|J)Yj|IS4ot>8&2E*zW1T`qQ&@Yb*jG$2)KP{7v1vI)m&57HyMDQiB4P``e_;i>l$ zFpHqO=!#^GhBC7yQgITuI8Zw4yWLCduBV~EY&81M5u41@`@e-|sXhjSvQL0@zg=1m0Fp|=F=8NWzyQdX|To-h{4uoT|FZ9^=WU1YxxO{5g& zMR}&>FQW3u)%dBzRpTY&;HIYoI?(GKy3hkA0&ux=>!L`!PoH5{*K!XN?Y*=r@R{}1 z;$~FG|8pIn|8G&C_=lgxEgMf*AE%NaT?pn>$W|;h>~*Vvp$O*BiqIEpf}7Td!E>h(b5H_E8CjMNxJ-k?{FdYL>n@jU zpPGQ>t#8R%V_;I9qX@Y{)!O&9n01`iX7R^g9VBmI=B;{hAleScT@Zms99pzsU+W$R z@nN6>hMe>z#GPp|bNLpfJFVruBC~RStF!bgFHs2@bA^(DDcdtDfmZM6*4Xv37l+84 z>gqf3Zl9Qe0PXQFh<06+2?E;-l)B=PvSa~|$}n+8Y;Eaxjn6)#zD!B>R*AJ?5>+IH zX4VKXXj$O=fDg_$FiRN}U3qv}!}PCSw=1*2>ey*7JUnw(U#wS6;x6pZ?<;;IjBn&=EjY16Qn`R5M@fAcXx5VXQ|^M4cm+UVVy z5ncmsJHO4BeL@g^Jo@wuDWO8adS8vwPUGQ;3EvWpTP)y=X{M4p@At-7kjrLLKJ8Mb zhnkthvcr~Y)_l-P#Tg~tsA-Wjy{dkmyq6A_Zm@o9GNP2bq7j7m6ngO+{LsDbH4n-F z@uRI+ln2gSM(IIJ(m)@wprJ;mdD#obvcMlnS5fAB}W(Ts#zub{khwW3fxO)Xf{O&Y$ zY)#;~sFU%XM|F_os4M00!^4ZF_3NEY)5|9`Wn4($JKKtdhpLxw(&b$kJhpl2uIan! zLT)Pf@A)5d#Hm{fP^G3~RGrDv?9$<&KQ{WiZj{%6ZnXL@r9Rz~n~#um__%!^-+3C@ zH4H+mOhF*9YfnlM&>a5=&%l4ssO#ba(x_Fcv%IItjyyXV;zM$!kq z`jwDC=@!7@*Lsj!j>{+aws>iZ_>(EEo8sD_{xV z>pn~0dl*gRsOsCP#7aeG>%GjGFhmMQWAj$X?ea2}P)U^R-uEO>9t>JeZxk~o ztAL>}?Vh~CJoK!nm-D#`xvh~VRzD={unEvI1&9G#2%ArGm0#`V>f?Z^5p>*pQDk6u zG$|eJaBvSCB&t>-1#uiHagG*BBRGazrBzN|2Ue&doaKwLi z(fpsSieN}N?+%^#QeW4`u`=$60?YwC%MeXQglafYf*_!Yk8G;35Tm+&NR2=8JVayW zs32dWB!uv)O;`v=?GKVsRimHenY0JvKn3i#EOT_-(~RJC7ke#uUchA4p2%`77uNo$ zjU?}KM&|YaDy|WAi>l^uDB5oKS`R~!n?8N*>H+{kQ?QY`WJkhVjbZScEuQd#&Gtwd zR@O%D3T3PpC=9*1a-+X5rbIQ?KN9K)Un+C_7AQ$e{y7Kx#s4Y@94O#n(w#i+h40)uIp^LnQpOpjd>K=KHLrVM>U_uw~v(3dG zNi4dQtm*O4e0KI_{j3>{`&d~)R$mtDw&_ODo|Vc{_3_bRVkm>L7@ig_Q51iNO)FZz zv>Kn$reJDDJN^Jzx|2*z`b!I8lVM}HKET1C#~A@viQA2^`hRG;#=yFpuNm8DY}>YP zY&L0Z+qP}nY;3i$+1O3e*vSpv^m+TgU+(wc?%6XtJ7*?)6kc*?vxQn!`GWi6$=L3t zm`dPg%HDr!!p}p~{lB}o!9NeGhoZYbJJF0@A4=68+dgrPa!|#sgoQ+L*X2HMA@<-= zI>r3GI+xHmKF-sAcN6-Lka+jVuX)lG`UB~hsH?yd`r|3^At^AaJV?g{@+2cd>E)tf zr#=Jn@HqQar*>o9d+1*%J`TtH(X(u2aR2Buk5+Z%F(nb5sx8pooKMM6@ko zu`Qli;t8i{Ne11O2Gdu}InY01!mBMu%WJs{6D{sz=dEY)i&CN>f=U!g)oB1`NVq>D zNgh~@jigU=3$`im0|(CfE@*dF;_K;WYqgsOTFfGGC#uvcVb~rXUT!?3(|?vqa$M_* zxgop4g$fIhsiWyE6~(WZ);^V{_U!*Yk!Uqypc1cnAEC89d(t8uOOWl{Q#seZBEfVR z{t5@$1f6%Dy4p}18~xeo*SV1PI~n9B84^$}Xn#joU)LpASr}6U7A2OUBYX9~`MrjO zZ+{@#dmctFZ$k3L|JUySyI?^m&B*`kn~4 zDBLm$DiodtF^d^0V?qr$A%}ab5>HAU{VH&fDXdIM4e${)=Np?PZ-Gk05s(QTKfoup z3c4&jds!+u@^a&=DwlXbk}*Nm)q7|iN>%Ufgh>=1 z==rG^VG{=>PK`?0`Vd{oYh=Yx2_(Ij_@XWEv@&$HWu~JDX(s!5dVR55TO2gS3v-y0 z4{kTJ#_Ne&B(9V{*)mR1aK|xGn8v^H8CjjDSl7S0qw)o&pvQZR&`tY3Z0N21&4Z7C ziT+2HKz`!l1p=>Y34KDloo4Mo(8{%jC1tH?&(IKqY23UvP-G0SA~GdMwY5!#JJn=0 zaD*kH@6ED2Ae}0r-$0~-M}!5n0ph`j3(|%* zTd77aY9*{k>TX>XoY)BkP9jb$7~8t4Zj&3fP@f})bt|%gHNxR$aSJaVd0aU-Vv}AadrpTkrj=Gi|2wI> zeZL(&PVY7fOW@Hzp3iBiE&she{L+8E;qM9t!~6pVvVSj>tgC*R87Dc^xHPP+s7rKd z-&90$Bg=iYAGNQ#;=B8Bu_JT4vCbwggp_O+Yl>tv#NMqyp23m^4*OZmMd;6j=Qra@ z=9Fk4cumFcLrTzUKG1R z_P@3<`9*rDd5ISEcOTvTlPX|aT>ui+p?w_Yn(U3)$ z$5M%Qv)x=%B}(#!^Z_NBS?<7f0;3if0)vzUt<2_$L z9&YY~2`U*omfRM|?5;*jpz4G=<5}C2)>S}qNXOtYVxe%u@q?oM?!9%rPVBF%g!~B! zA0vRIx=(_klXMO3u1&6nfCP>}8M+M>tY<+XF{LbW9k9F_QA7 zsBBqwW>@SmnCv=xV_DrVbk{1d13_z1%fab5h(~Q3%Nm`9zH|JA&7^@Y4oDJfKc2Vs zgEaj=-Pb-7F0`Rz-Jqqv#wNd#+^U^)%??k-Vbs5M_aoO??-w&!?;MN$o;&AAB(EeF zp`J_=n43rV{jn!0@xJZT&u_3e-mO{)_<)MfajXek@P{Cee^cL=ZN#fkO6i2n>eZvTHOW(fv?SI#-nCJPkNAP)rDqp~#%Z>K0lql#Qk_3YEh;~!qS-ew}b%CcG zUw*~NazTQqE_}5I??u=OU#(EJ)nb!>FRThs15;ryzW*Rl+7t!CaUvB!X`ChzDq`%HBY~8wl zChJ~aUNHVww@?2bss&YS>3x4G3MMr#XwDpB+l*O#&fIDhx5B{2+wEj;aAY-)vf;=d26(g`F5jT3+;PBNZ26z(bNBzF7FNloXKGSo-nhHGs>IPt2C@P__ zqPXSyEe$zjX4inC3Y9-t=8)rX*R47E zJ+br`W@g@q_;<>YWdDUhHlj4LF((`cywWfnp^qz|KDv*BmtXlS;Iyf8H9}e_rFZ3> zQJ9q#ESGhpwGsl|87C^Qzw88I#SVWkHalCZt0geIwRDv^QzkzO9E?}?tRIBMqKNp{ zM8ARe?Lywo6PM7*8MOZ{#f+cPV;T25KNF)r7~btUFXYL{Rt|4NNS>XISjB~hkC7kb;j&BCt4IE;(stlY$41?asYG~ zTB@wqSwvd5sc@LROL1LK0}!9pRY0i-+ZjrUgqn61WlmB%iV@Bwf`ViU&4jHFS*u)hv^T-0S>M>5&{|J6an>LlJ4`zo}<`{yvC!aT(qI#^QuuL!t>TU#X?M14Qw(s zxFju73uGMlm*spTyif^o*CoW z>GBA3o>nECLZQdZwEOsw+0)5*b{kQwj{;9z0bgy^YX>jZl&C4y-18SNMqU!zZ?e7} zYw=Lt@le85RZ`cCzolyOa8pricglYv@<%Aii^Jr|0w3Je$G-$A~zKAC=wDm1b8Q)rce#H}c zFK1o~-S$%Hy>zAv*t`F#`U?gly0HHhMELR1VV}A7&1o1V+Ngq}WS|;N2yyToF~d+d z(J3;2va-;;S|jsJ;BQPR5AJaUG5NIsDL}|Ty#k+MFIXB;#a()j6bWb|$ReSZj2MbarIxluHk20)bJ7Fc<^`DYX=^) zj(=9d30al8Dp#-h$Z;gb%xp&cQ`?G<@>n%p7s9=*Djsb}l|r49r#zm;%90** z`I@1_dB3S(lMB~JFE=svKEHv!KCOryBboumTr`qVWPT}2QD;0CuF=dXMI|;tvpXAf z_N(Kmrx`@5wi#sF`h2y|*uH!}o@u~dqB5#PDCR_0V8`v?4Az zbx2n2%RG2eKPh#jmEU#1WcOGK5kG)q0M~v9m6$^&R!Ia64Q-2v@_3mDV~xqk5N&Lf zcIR8XnCB330;7$jj3v$dz9?i$9-vUeb(fBP(X1{m92!`*jlw0~JRta|S4O_@T&+r0 zyCQ%e&q53Vspu=5&s(<8&y~Ixg3r%+l_ABC5$}7D$$c0?#=@$m$an_VYMYS@gj>b!U#RU>6rvSJ3yU0VuS8I*4~s6 zUO4ntu3!|Dn+BFBi*9_|nGz!W$*#gaoFj-O-dPgo!D`7KQ{lw6G6f-~M55Ser)y9+ zI%n2Pu}sd9vAmo{Q2|xFDBu?rwd;*A(_WntCf`RQTBJ>CNsdiqzq^O0^K^F!b@hj% z|E{E|Vk{XSl0fh|Y89XH2l=_&B6_0~G1V6eSlWrqpTfYZ1qI&S=tsW$kb2*}?QdQ8 z7ZjO9Y^zZo0^PF@x)T|*f8!Hr!R0u;RHhvkJJj@m4ohA6|--08qI z+emAhAak4py-mFuVc3i!K2`F{-ZkPDr;8)B_#qk?w9DoN{1C~WBpL-q|hA=$c1#%k0)bD~&2kCA41BWDl2 zkML)@A77MwwwkeaS9ndL7|csyo*9a%RjkHy02C}MsyB3%VV4-j4{>J$5Gq2~VNJr9 zfQ7A(Yk%*}w5~4WzcehWci2DT!ua!jn~La@xbRO#`&T92D7Bgg^6sTz-th4sjjRGabRb%F>cId~Zy_boYN&vr z2@K0>k$Z_mdigntD0y_}Rdzu{#|ba(AK*Od`_#m6GAxCQ@TCUIpRcz~wV(yRgo~CK zcH8M(L!^_L51F-b(VkUJILxhID9Kac)GF@Kab|CCO;?kJ6IEwCE8Kkp!zx(^+sx|+ z!(s}g6dgN8RK9`s$AJ;`7~vcx6icLjnrE<=8{}C9(%K_pE<)cb#+TxwS)0lDXIIhC zK(Pj@X=W*pR4>t;0n7TVwklP+sxWHlBu)`Av(2NG;hpldn3mBjQb!@K{?M&IwdKd{ z6w$#=PID4N!3Fpo@XT)!nyulo4*zL#gFcZ`k~l*v;SDw`JkXEMe33xwz2(ucWsAm6SW!k)Y=Tm*(paMJ9VDMWO~@pTb{O$}Qkcbq72*ydG>Y zK?>0gX_We=bsUhjW#DSJf>l+pm$zA$e{c~5S&Xxl=nm?JsUcG+#NE-x`bhu^{Lp)8 z(Ik}1q_)ZK!c}1Y$zBHF(r^zE0A^4#Y=&&?T&WyCZKOv;G3`558|J}&Jbu@>1FsJG z-*bkEF{v~eW&J6g`6PvK+!m)5&9g#Ts|ibda(zoO{1O_=Z3B2-Q|-*DQREmUAK*Bx z3bfBVjbQcItDyU!cBZbY5N)YYDrUCw`7S=D2VU*A7v0=|?s* zkkb(NK$r+mMCK8<*W=L5kcpH(RX8=txfZIjycQ#lqS@k}44r?yk9heF{k*)eB!i~* z5=F}4L{9M;!ZctoLOrHLnW=_dp#kPJl!O}==MaSmv ze+0>b?k;BGv2rP>)F)E)8w4%H(=#U=9pB-`<*}Zk?q?ZdGIx%wi4W6+XZ)7b2b-<% z3~6RBBuczXK%O?9=v3}LTqk=Pv}eT_c5HSw0bkP(c~4hS%t|`IP$T`RU~5@hg{kz? zmN^ui8}gYE!$KabRB5FUota2sEY$r7JK}pfY#iVI#CGXDG`YDs^mjobeE$)$8`6Kt zpt6S!tak~w<755Zhe#>RSayn15iu{g*>M4QX2eKhf6cU;!#+ZD)C9!TN7~bP!2ulq z(!TxP)3(Ds6RgQwTQ!(JD#CKw3McN$E^07dXWIQcFyn~nB$xyx+G6g%o}@D#g89+w zF~U!pR3W(Gg5U5Q1P#3)k#jooe$>sdQH1*lDY33Dn1=_@z|xC|fHspfp?#Dm*%@h^ z@jUjY(ycj*mbG)I^jMM~f+fsr;i?2fUg;q1tMX@&CjhHv?60KO)lp$dA`u?L|Fn_2 z0!~Doa}jhP=exu8ZSln!Q*lBS6j|bO>iVQ}MMxDIrx9Fot-|B|rdBgf^hML)z6?=xs>;94#D`tw5w$k@Mhh>06Xp~S z2S9ua<4?@OJue-oFHGQfbivQYjOu4qcOm^EMI)2=o3F{uc_9ijT-RfdMau8uE;+vI zZpa?k1wj3Kk=YRYH{!*I`l;=FHUe(I=XV-O0F&hRbHm%RAm>XG!ct2MzA}Gp(d7A} z;HaMr*v)}N82o+LxwM&EVx%Uh8kh&$Hz~w8$`;5$C8*UP|1(x45;epeBja9%yqYM$ z#DHepwNF^Il2#2#qjlc`H9L{|Y=0pSN*@E+X zvx>Hc`=*~Ozu9~?^}t~mT9IdO1L@Hl1x%Gq9E~UC_G@AkA@T2ILaZ6hL5`vnj+iUr zB~j`ngb(|iy5?` zDb1;x=9yH=Hi4}DzPX^lf6)!haeA#6d`i=#-`LEJ`CAO_xBpuSBBor0wGV+b*SJjD zB(inH?{M~$oQL{LRTysfJ!=KGO$Ld)@#J)kpDvww#(M*fC&4D-AlGDx?Rp?GIxwZ#k^}2y4QIE>lE?wzC%`IBl%>EzoZu)s3FnvsCE$TE0@nIL12IYIaGR}c$T;npdQCh1Ii5QFz zweSgam@Yp0Xz}ek#l|5-n3pm~C6?u45^Q9@+sXE+S%@W800cGuA@9A{uLzh7S+h$1 zrQ`Y%c%3S|$;ae?bg&uz>gZjwNTW}kEVh_ZHDTj88W)i-X3CaSLBzFUC@BXVtqUu2 zxKnFry1y%=;QQtmuT4VcRnnf@g(1_TeBYF)OP0 zGE|0KlixW^Y4v8Ag59kP6G3u43pm0KzQsdwenV<}wgS;BNc4;W*Mz!TFj-C5fcS%X zG`KzQJ3LQGi|5%HW}x`?vqu3{^=+LRIY=D5depI!S+G#7th6K~GcWf(J9m4~i7ex+ zq!(e5^r|uzuDhT+4XSbptwIj(P)dZtADL(>5d+M^3K?xPK-O(u22C23xfGtN6!1o} zglp>(rQQ9Wh@-&M67EDI_N_UA(}~@rbAnU_1NgP2cDjSG8m^-d22lkil~l@F8o95- z3Z(vNTt?SR5AR$O>%<{YH!Gsbc8pd_YblYegWlDgDqSe8s-%`D{T*Kb<{noi5j5a* zS4ILV$_Pn%-WCbT=>AlUiRaIIx$li3GWXkxKd{K$uCI#^?=O~heEc6y0{zRJYzjUO z`V;B4oE@LYC(kfI<7%fJC4o0of`r9frC{Px7Qt$aY=#fGpS5!2tEXVSuPjpK=qi(k zvcYg{A0Cw`T-z^^>)INDYkj}V2l13b<4eg3D#ZLGiN74%h>1phkztxGWuAiTO1$FZ z2)AB4R#a0OjTc7FQ3rdn#8S}PEiZ=1g88EL^6k@zV;AWCM1vPcc)f_*I$0xhV@G(U zS}~$ir3v9C_uKEXGyD8oPZgEdx6Ed^!%$Q?@(I9V0kqbamgUnT zKE9w348*FY^7G{D#RhUbB04dvE7-$8Komx(MUl;c1=)a9Uk`-+^AR?uV#_RHo`i~< zy6G1c0_C;H8keHD=rvdK5nb7%?u_|vGYIX|6zM4zu`X3UA84l?;>Pz7Sz=>HmBzyB1CX~iB?Q*^@sZi9$Htyo{R$R=5+H(Yx2`(YmIj;~R?#*vD;plR!IBOCdb4in4hHMVlc5&3 zgZ-qIFwnI^@UgLukUDc^)vVd}s(@FJKrnOIu7Mf}t*_{@~U>p0rxvth|-Kv)ge z;383-l(=Aq%7ER&cUy+)k{%VgN*&(U0ng?U=xcW@`hk!4?$~dc1p=(_iCEN0WeNg? z7#J_T8qv&#VTv|9twHeuLi+PIsd{0C@t4X8lH`y}=jt_jSqJAD-cE;An0@V!J#ilw zZ;Arfoym&)W)q3w1WhN7$lk(tlYbAah>HIKSd!1uahCn;)gNXKCs&$SW@d`htU7v( z31xS3ilFd^GtM{tic(kG@Y*rB>S5Yn()wOP#ipSLw(E|#=z_;w3E=EPH*D+uz3GmA zDyt;3-1Yc;A+`M!gG8E~SVhQdg0tZT#!Mn)(PktE)?J%jr8lS}*o-35l^Wo}eYx|N z`r-TKbrFk?lOH-LHdFw9Qh)diaJeO9xY0%k3d|9b?CnJ<-ARpMW{^crQ zHx2HFCNx|zG7Hl;Uk5gYZoYm{9#cZs!g*1tWdQiI1BCC$hfbiXWVxU@K{Bj_E8aNX zdSCl->bfIer9(B)Y+eMG^_Scn**1c-aEaH(NlsFeSZS?^TF-ZuD6_Tzi@1H&8K$cW zX!rwoaA8eid1nqnIsJpj%u(p%Q?xkl(l~PAvInLpISoeAo5mIwq^~QSS9?q?*PGQ} zElhBlB$1RW)S5(JhB87@2T=pdoT70&1tbYQEykLJImm$4-R#XTV(1Ky>zI?h!x&k>qVTGb9tP?DDXS%1<>%(^FzWQm%NnbKMkZkZ}=VwK3XF7PS=po z6=$^F&mFtH?dYLWMjnf}fUq>6abG0%4>O=-*F#r0!9o6@sXh7=g~4iY1XY1=wv?Or zB}?r}Hk(&HoKJH4qurS#*uT}b)^|)HZLTL?ED(H#?P@;aF21HlBs_6!zN4?p4V}Ny+bVSMIRIJ8HnWqL5i*7&mvU45h+G*K|Ca(N$*qCcu#=@DvTr&z{#3)04As^{ zk#ic*+hT8)uQx$>!8AIVULvj3f-}oGmVP~pW4?Au(s^IFtHaI2;yCVbBF#>(*Tv?~ z>gd(t)olObjsqyF&_Jg%3HfzX0a#`5LkEZhj*db7)EFK= z5(T3Wm;YQ*FMho3d?A%0T}TVql0)`?Y4+Y<_;^7`VYg@WCsVnxr2c9ZDx|#bInZ(u zu7a>gqZB0QaZCmd8@FU*3O~*|KduIYY8u2klZces{^HKSf3i=*xh)vtQWKc6$P)`@ zS&1H>gg}v~9a{!h>W!!^dViO^_H>}HxU)6lsw9Uja)2n!4loi+>rpo~*}O-^x2J%Q ze76J`g<`PGPjcB83&#y4BE4UR+jgH=hwsE?LS@;iTWMlY@c@$)DsqE|M5psHpyu$)tP4|7@Z! zBoTU#2dUCax%)EmZDH?H9z!@)5Wvsg>FT;jiH=t!+vWQ%uhsPl7gwl%&|O}iV!58r z(e_X}Bd{2KY97WMKz@LIvS0miUjss=GZ_meF&7HHT5WI4^b=pl8Oq2}C z%Qkxv$rIk9LK|u6-5pfyFlshjIP-?n$3(GY;xL%maftYCSV=9yHhv6^s>M7_DX`Ht zFP_;iTJFwaLT~dZe!fe@vvi3qUlL3O$hAp>UsRfv?jklu8*Umnzd5>r*ZchRqQ|iy z$tRGciMXSl1y~FSJ&r+kNSX5QHDkzC?l$VAz!hWSnWArs8!b@j0Y_yd1B#khyh zVrNho4aDWjsVYNP9@PtIwE0(S>>dWn^I~!JFMXg?iYz6~vugw9U5e-%6K33%>F?~M z0_(}c=|8p#!)rSY_e+0|VEm`2ih%jYOMWU_eXc_l>EW*#=CUoxdmwQR4k&7g@gS-n z@j0_}LlR)Nf73?h`jv0@f~!!dS$Pz=MD75Wsl@!J*H=oy;9zJZMBueY{Y%ckzT+jz zZ9z@OPAq~0f`%aeOx^|!sG=>V3f5wgtA&?IHtuRd%neuMJXOmwo0f$P&g+#RoEK7hv>%sdeRDm!-m0@+L#Ps848H7mA z_xC@+Pglrc3B{o$B9&A;w`~SAq)DsHRMW@Y8{6uo$qGG5xTxybX*nw6A!>9uQ%(kk z?+;sd8>L1L$6X^iKI1>kH!oxU2z?Cr$C&)J2KoFE{d-jr>Yt5f@^Mm*urKS0l)K{{ zl>moK?Ip)%Q|_Y0uerc83Wm(xaF}Q4_`{UJnL)4bn-7F=C<;VJGlSf&U+D4`?b|XoHftjz$%kz@sKp@Ec-xiw z^AlV3e7-j0t3BLTTs5L73L+)v?kam3+oCDXsnIYRMkm$Zh(%I^F98|GY;YjonegZ* zR@9h_La5ouH}U&`r_qkvLv4odr?&|l;ilWBwBXNj*q9N&&`YZ_g!528oBTt&@s1!~ zRTdz(KB$=0g-2V}GZZnA?{}R{9P4;$C8h;AWM?epk*t9kqgefdaQ%vS`NFVpbu?!6 z5eu(Qk+K=}Z|FHp+e#VB5&;RTjJ?`cTI57Sn_gx{#kfkwY`PeIcu%j{j~ldS2k6?d zFVk=1tUeEW6i(~(UxlSpJr5FS(c}r+a?-OFuf_?XS0p#ZQ`luhFbz9thUF4DzC`h# zr#d?)7olj@abIL8^6#RWzukN%dZJ1N{>^Fn{54Yl(>moFJpB-Q5AK#@1az`#&*iR= z;IdKPsUdY0HCNDFc2+bEeX+ccLOv#%;vj6j4uLc+Le`R8*0R&qkzF6&@V$M5&>{xFBUy z&!qq-E5M@2ZREC-ggFkX_9~q1Q*im&WKfAEXcCaznfpAknV!zDdwri_nkWfIgO1Rb z_*E?BBtP~sXzVg9vQOKqwkmn(x09CiW!Txy0JpDU7YCD_J`M*>;X*exOnqnCiXG=a z{9j9+9R3=Ti-+IWQXGnPUucHz?Ywa z({2T9u_b4grXi#}9thLRtG;Lfz_{gZCohb0p9UgkAOf(wP1U!%b3OWvDua&Ggqt4j z(~4IC7d{t*$z$)@V~FiD1d~LwkWY_TmW@4f72wU;|9i>aUD75ZAhn?iQNaeV7gqju ztf)m8%ZlMzypzhZMB$yg5|XuGlCGUX6ScTzq-7Lg!&bTkmsuoJ3Y{5hm8{;epCG&! zl1zykvj-tyY9M1tPthYrLrGvF>ujUT7WCWK4wA9AFx0IJJ~3r<%>8qiwG*M1phm_~ zBiC#g27gIJ7R~f{%$c@6<7dBUlG)ejF#^r<+N0+gpO<8&ryKvb+oSjIB>#0=QX`iC z9Wz76(SBF6br!Ge=Togwp){HP_6I|y|sR}n20aJ7LoZ?5g72o?mAhqQ5X2Le*iy%)dWYY z@Gxp_uJ=W{r5+*r{`~yqyFG!%y4}o}7<0T^UjjC$DUH1iW!hZ{0FlwRQ0mt+;kG-3 z`F5M5{Q6`6vVXGGlYO32hoV%W!kGOuq{QC+t(SVb$PB~&_V1uSV>pC66*C&jO59 zGu>cUHkc@S;DZjlj3g^YI4Nx98ioqp@>JY`HE+GnU4fsRwwL;DQJ`D~P?fN07mvBp zZPIJn?83;5!Vn<>-|qdG%XVyH`<|4w+s*m({+T-8Iu$NW$7b@O@{%sIb}GV39=01h z`H+`enUyP%Nka8q`%ZX6k!yy9e{E=miLl?guNlF*t<0ju(nilyz4PeIG1NKrFh?s8 zP7?(cv2vn~vLCGJLA;o{b!KMj{pKCH`=e{?j*ICVu!c&AR50W}HIDch`2Y6lj~nMc z|HT>$-Q1K-7M8BmDB&Vv*d7Wb7(4l`TaGI^!9NS)K0+|cEU2!%MI187)d~$~BBq#I zkr=w;k=MM$O(b)b6x|SIcgqRk<@1<1Rw6yWrG|8`3uW!b8+W6=nC4W5O)?XTyI`vX z!^FqE$P1}?mJl<+s~~0E6!P{20jWc1k)m3Dtw!(Qj~a8@{@Uj>OPdZ!PDWKX5`!F9 zlO~f_b@2j|Vr051_QxN-9pFCiT7Gn; zMq!P)rv+&oQFVmO#dHXCbQZaGENrY5+|~^>9;L|kGE!02&HkjUwj@h~EJ7njF&ah8 zicKT=PmD^m_k6br2KxMbu|G9vP!&1V#3WMLHUhq-aU78xx2bbnPI4KWRT@dNOuU-f zDg|>kcBhjrjMF;7`re!ReFxR{-c!-VG#39sj$y(7ZB^qMudzp@%wy9Psv<3AMw?Ou zcnPA>{)K+X(r*b`@61+rIXoD|zeF4};iZURDr!VT0=MQ85l7Ft#;qh!aCM z%@~qF?T_-;3K~w|O~k_=1#E|{Ffk~5`V&aUXGGsE4kOrU5aS2Vq|%HyF0n+`f8M1& zhglRAzx$}o1U|3iz@J2CQ72iW2)s_fv%YQ;DUfV)hBsp;`6e7HjwJ_a78-UrmS9fw z8=j!2nJAw{lc8sL4J)$1mW4>`H+J2}Ghu{gI5v={Z)Z@7a5wHAK0jIE-l!%rRg%+} zCX-hs#WWk96mc8yFH}_MDvW_X(Z9Bm0a%zu$l8xlM~HL?eIuuXCXS|0i6Uaa`2*^LMyy ze)3JCq>AofK#?F(RT-rjWDNbPKhiK<@+$VpZGN}n4e2Ym@L!>l+o28HHU*wEjfl9>l z!Mp!VEctv_OGci)EIy^Ria<o>rn6rCU~8F<&l; zYxppB%ju)Vgjl!QAcMttV~8Lfx;ncKCAH%_`Mv{el!BpGVC zR^5#LW+gRG*iMs~`)ieM;;a7iP5L{~|9wG!TZ7>G-+Ac^Dzp4wuE%P8<2#vRj*y+Y zm(H(0Z{#?<7^BorMN3aAnK-!M@@W^uhTtyct2?~Nk34J@H4u-}xBb|}O;FwE{6EAF z*}l}!_6W|X8_qSt)^K+0 z*Rl6f)w-~S+;j>jBq$a_-M&4E7nG?;y)XS{tYd!3Q!IDmSKv#-Cbu@WwjOZ##wo%p zd|NbYdyIjP7sK~5vabfVnt~di>QJfh?(HhH*07KRZ+2GI^5tQbp2Tn@uw_U^VKJle z4&9p(A-zmBjox2OCKkg~S_ZL-I;D`_i9LPBz{nV0@ogBp_vSmN6AXK{UJ1K9+KU3X zD7y!fZ__j$5{K&I2ZW09C=rMrxslZO&8-Ccb*KF*b-J04*Uz!jFo4y6#Vo$j@ee$gjZ>?o4yvh>(Hk15kkD_OvVN?a6NhpCdL=|7 zuZP(QBV+2YPdvKdJLsJVrl?B zYN2$c-<6vo47hQ?ZXzxFA@?p?M#T8Fo9*)84Q~#)ksra0Sh?uUnyGlg_ejGKY+G>? zD4wq=IB;_+3nZC?qo=UCZtCFiANfm91L_lVHdq!MUP+V36bzYudq=w5l)DyEgBJYA@G zElPwbok}9rP#e^+vb9Vwk!Wv+D_CQm%?aj;P14^kB$$1#a*m=?GqxoVtD%a)$wx!P{oS;J8C!^hu{xWX+jnfQe?R}|b2U}p z-oqO5|ED3UJKX=@lN>>ej(_QU*a$YizPcEW0DfhsVG5Q&T_2l3#^XK<;F2^1U(Ide9}e9f;DNtSZs0n5c08=&MSU~vhTAWNVeccX-bqD>7JHP zQnMDH?GFuXvxPtiZ8`9n6IVsVDRJz@lRPpz4s@%Cw-axnfS+zyLBk%N5JPl!^jnT# zw%=SrG55LPWeKP-DYfTYlPU8`7N;7}Fsrl0WX)9N5!r3EC)4LsaeJCZB1%|Yzl9A) zRI}euo9AA~g{S;ygV!Xdo@>fG<703-`#@?^DP& zZ9ob)N5G413WA+?W$&y=mDg812eiZi%}d)u^j5F>RNNb91v>}GFlEat7w8KCG-@uJ zO0DcD^o%H-g@uppE#H1Vl{6r@|Bb>l&pNd%CSIlima^7tfLH3DBFTD0y|b0xh}6Ds zLs6>}p;}jK+CS=GtnG`N++C(*zZBtB6_nAvCFn0Q&W4gIFw_lQ~Z&=SDZRNK~&lSl~a+QLTg2QrH$*&`J zhHso=ZEv3(^E*ZOZ0d@f0P4lV{DI zA>}VK`khos?_O+l&A)lOVl6a(6u~(?xE$p6i)fZN+yiTivg)!h_=m#AB`pvSl?a^f zfw?CtONx9<`+Bx6SoWhGyblUo9W6-i1Thzvf2JW$ru!}YXWX#X?jPMm=JQN2%C)S$ zCpy|R%iXu}pibBDE^Jp93f%>}S_l>&{>XoGC=;>el!lQ`%PFdVwaQx7Z_S`g(>!XSu$|L!qZZ7slWL^th6(!yM+nlVpYMO9L5jtIH%_xH-iyh zY8R}Db043s?mE4iMIPK7|0B}ECV_Q?k58AiJ=-oW+s?C$eFYDkAl%Q7SsWaL2ok`; z+;ZAAle6wN5sE&MhA;Y^tqM32jHz=Eu9Iu~KBX}gilk{U=x2BdXVgvx4>U3LK#*1z za(DIDvwTyyDgsV*-KKT7tW7I{nD0(Ipi_wnGpuLV?n2P$3jN7TYoY4ifwOX1I&)ar zb68rnI8?Yp;&C+hZf}k3v74~-=KD^z0t8yYD6AZ*vPEzY32m^IH5bF?>R>BKMq5m` zHt9JfvvcK^LvOf;E$3U#?Mm4dZy-$@S=hoePRu63*h+yQRmC*(k{^`qYZSk)s zs6Sc6K7l(s^QLxr-@}cX&#Y{riygugKSq*k0Yj;KvoW`XX(Csg85f!#O%hLrKBKT* zdB1BVVk-XhM2EHAl4s}PsAlEW!I8^7OHpVa-1K`~{9e25sx#~=(VqA7EJkv9+Du`c zFEc-5-El;(kMI7uJoh;lUZ}a__c}NTNIwcm11JbHC6e!)gzRPS;WX2k`&ALvWTJ#K z8&8vy=MsyHYh2g->(b1cE@(hr7XAg2kLIkfL_b189s+y z1U^SsyCvYA24%3SPqR68)Ek?oMi6ky2k7k%hL!ukD*~_iB|KMp56R|=(*lvrG0Oac zkGxdfc>p%4xh+$GcmU9Vg7pa7GZC!_6ZeOcLJzNU!xut(ubx9>$e|s>*d8V3I5ava2 zv1*%GsH{$^te$d;!t4mog^o7c^i|Pv<6vMc&sZS7h|@JbfrCS`m8Rm{AAb*Wy<22y z=ML?xteh5e>0PvHUH_k(d2V0T*VjTtHvQet@!nmhJ;x?88J>M^e@fH216vvR>U|xSh;7RTYEX1nFO_c4->0Ot}JThXtFOLy7@{U$4r%cP=c8Y`8&s6F8>V?{Y@Y3ErbG!>!vSvH?u)fGH=um~ z+y)D&wHZuI0d!hoF($_Wbh##auN=I*ncNSYN{w-KII>=Ao6}1U@+)gNuJd}Li*Mx|F49YL2{}DO1CVtDq9#*E zmQ05hPS5n8F$u>T_rQg8zwQjX&F>-2kpd-_J~h=n=MWql_X=LP^L5vAn`dtF>AK%} z#N@5ZISV&gd3qk#a=YC0@9LbVU&g$*UugNoKR+co2pAp+h_S^9TOWIeH*gwRo}l@z zc0P;3@cIfv#5pHz+fQ{nnZlqi#~l&c!ko4O$>nqFv2_=y3eWdQn$yZreIH`0n8Q__ zg{5DGccfBz%kqXo(A*gy%$v`vYAs(iUvWrMm@QpU(7aZ3KZ+c2cb>6pQiOv3sMnJ! zNfwx6L^`#q_qJMbb5ve2cjMHfLOD=dv$1ESTAh9OHqUO8Z0Ft{D}tR?9JOSpDj&_t zE;i3$mlE=rvdj9$DLa*mPX2@Z@Bm-_!0fyey%yQA2}doUI9GYMd6L8Ahn}j0I@bB? zD8a%1vG&$sQGHR{Fo*~$jeri_4N7;-&@G{qgc3t{mmn}S2#Cbc-O?c?-7P64!q7<9 zz<0*q^Ssyl-*Hy4T?x=W40+OyMq#B|_piy|5DpqmM;Dt0xEN zwHA&8prrVJp|9=Rg;Vn=`~rRxU3ZMd=GjzES?TU&-Zrj{l(p9!BiUYLs#Z1$H+zxqSW&gRF{m`u*XOyi{7@~NBn z{xQvWi^>dqNq;7u*q%~3?1Oj?Oa(_hwBqTRU*`IYB-jXesB!7L4xMhcREX`tx+EL7 zQk1qJ?4(f$6O&fY*Y=zK+ zbrW;phpRC-wY;{5Hy@L=^ssMR+@Tur^ib8$>GQJc!rrM|o~#o{s@h<8hct$AVN=U= zY#$dzA~BU7JZ{IW6&+=;pZ`{6D{t=ULE(W2#R0A2{>rylZj!mG`;U?{8AE+scBQEw z@RJBfUQ`!^TQD{IB7SQ6LcVj=qJ?PeiE;t!?%a)!KCfsSg-q2URXPC3HPzyl!O&t7Z{yUH(M(dCArPCX|!+Qn9gn0|8Q6{ zxL@RNk$v;!)3rw(*XVjxjaakXNbOFDP-m=KhphSQA`U#}mhOWMk==Rbsl1a%?Yk?6 zFX#?%o?1Q!iyCy{K^lYU{T`4X;1FC0?B-RwHbfS_aLa$zSg2=bo^~WKv4`BmQRh=X~%#v01 zAh0t~F-DGvR74IeEi8x&Zy~WhluG@k6o9Y z`@bn2H!&8eSK)B2s{sS|75_z9_RQ;z6YkbaViuw&XlQX7sD$-F(4HvWUn-gB&Z%_{dlAnTV<43!;C#aKIb@MJ+Zyy;oorfhI#4-rO9fh*F%AY5E*!Y z3nsf^9naEK^+M(pKSb88=Hd$PV2|i9P=5DbbZV^GKdd3CC1G#y4#8Qqz?HgiXUN>K z?yGj;Zr#W0nc}XCp$@ekgDNp>8H&aSEQ8wC28FF+EurPX%r}3cP2w4ut?WhQ!&LV2 znc={zSk>V9;Ftw9pL))mrFYUYwFT&-dOJ1FlXy#WaHOFJJW97ZWcEU1p2$f0)!EAE zm%97G{d%mfS`g(ff+ZI@(fB|*VWD=coBO)Ht* zX^J)5Tqf1ph76yU<{HK3YM18PAjI1<*f#RV68X6DVo01Hkrj_Y8$3w^R3N|sRE@IIZrK>qT;=L78fm+iwKG?F%Hyt{V zX+L94^tb}iqO}V3wbahMnQZqj4#-xmUKw+yBmC(V14_hCZ+ME@0&jsU3Fm%_Y7fb< z)ohz%HTUJo%*(Go$Cmho)7P^r=(0*y&6&G$&+HauwHEI!_6@c=ufC63=+p^ln{*%99-NT z;7%4V+$PnsPmaVtR#YH5Zq znD$NHUI;k&spCDEPXYP3L|!af*tKqL>e;RDZ5Bx04*GNfz*Xuq4FFu@OouXEO#K%r zJ-$0_e@r;E;nMpa^^|WgMR*2X+u?8@&zM;<`{c(W`(k!JNt~m4amfSneV>??Wrzb4 zcz30lSJE&uJK|IQ&+T_@g@Hws7}*JjO9!=hZi#FhmMxCACrZikREvAbVkltGQ~= z=D{SY)gzMrBYo#kV3g0XIcc&Ya)L@=y`+7-F5PC6ur5M3Z0U-B)XWi6Lce_zM^$m_M?htZo>qeLH6)yJ#Dg+-K@ zMa~{7T8g=oQ-g+CjvbdhjbV+l;ZB)7bbwjk5DGfxpx z6fjKI-DH3!K!}ww!(}10?CSTI7iQPXD|mU-cEbr?X>uNiR+EMS zhf7+;;j2o_scU$)jTq12=7y=|`*4TR5e*l2W>uAg5y2Ce7W?*-26aF0tC8Mj^&ABj zz-}W@c8g{gnSt!VC=~ql&0@lTJ?eHn99dpaD?0aO;})h{Sno49T(ILP!W+_fXYXQZ z-5BX9B8^ixRqp?Wa~V%*?tZ^wa+PED1?V@9mJ^ono{7@zmatY`bG4{pXVTA?M1?NP z&;%b5ZOgY2yfZfKrqAIeWIOF}sYf2?M(B@5H>!9AQc0o0IldB>_+?+0U#aqnUJR=& zZ!NEIiSK+>x4)d4*-{%Dl`uVOEqXvF`Gz*2ETDE57jg~bVlYBpnz zVn~n3{+){;!KFiiyunx^W-uBl`6rHB$+EylM0{YDJYl7Qd08)wC1O zp7)Q9Z*1WFIjJeXrX;Q@4;)gHZaH`ddE=n3)ZP?-ucDb7UpOLqQVXBW=Vz@Png1gt zjPdEh&Zhryz!w|r*cv&}xp|LWA#kW?z*d@1gNh1g+r)zV*uKxE^P7yb zpMvm!gZWCicnvhPz(i>iAZVOBbw_d;y52-LwCQi89e!PM*m?IwK=J1i1%J8TWMe_d zFJ4=cz}gesC0>nM<8qH60wd|Db4Q{#p>n&D%|W>3Kj5S7u9lvzxU^3Al1^@`24E@u z_Law<8f_e1@s*tx78Ls(EsYv?cpU?~iZR#od}|8@>~QZL+oE~Xdo%2UEveSb8E1Lc z{?druO4G-WX^np7tJ$Z@nV(;b^<34Y_Fd!6IQ|pcc7rPlci@N-`|Bed?^9n1FS1aC z@@Er^4HlQ-in5p=g7FIZ6bD(YN0lZ&#C%a7nY_Chnaq|~8KA66{Pw`1GqG!=J)XE{|U`|ZhW1pzj3xC2+)LEOC8R3w2?7F)T zF=!y#p99ynu(BjGPc}X;THV%RE*=1@rJF+7zMA42e7ksKBc*6VE~-V`09G&(rr3{4 zIv*4LSW}S5+w^^|$;xc!i&4Xh^S-I~TH02?5nR%5Vu~s(j1Jg_XELa5xb{BXHYOpj zXuX1@{z7KoR)^n@7m&_tg-;H1@N*snZ7M!Fpzv_!Xk3_q$6K;(RS`Nhy#;YK*@_DUh$)F2p#-eP{=Ek#;fyX3OafxIuuB&E>k^GTNq>uG$tzz&* z8x@DhM#d)}SJ$YP6=XpptqwURPHhte4yO?ZoL;FZqw`uLqr-&z>+PcP*b}uiyiEI9 zkNVgjj_?~`%I3F<|5|%O!s3iokZ!k_l+yVzUH3|RHu+6wfv!|x-96*PM)#M2l+EZC zhZWbJLCYjbm+c0L5N{MY#u^HgZ)i9S_tUXdW_HadGYi@Wl*)K-Ys-XRN!Ps2Z4$Oj zozcDj>>`wI%IGc-g8T`;Z#jGPi*<6IpFV=uwp&lp!D-^Wyo7iafurLj(a!v|OGo*A zCSC4+R_M*ejBBRRh3o}E!a+8(TK;1yrS%Vx+5$bDduvOaqV3l%!l?uhFV)vB7OCzl ztSm!v^_}zYLo(A^LIUrm7*Db2$9V%Ns^_ygCYt*-6 zUN`vaor8->_l|xzH`M#7^yeJ}lZ|R25t46JM=!08Z5#LXY!>*r(uYY~ZsjXZn3HRE z-RH8i(rJ{<(VA@1i>AMx*YC4o?^Kj#-?b!bnbn@P?sGSLm^n|>&zGk;qz*x;y|xVA z-Tb~lj+Nf12m>aF2K|WA*`@PRX&cMksmDott&ex~+j>OpfO({%wbzZRkG=aBh>Je% zFURGCh8+{3i?q=+kJzkGP62Nf|hJ zv(M^d)Ye|V`2qv*fJxU~SLes`XN43!{fn|vnE10_6Zjfz$VTiJuRZjuHx_ZW=u{3rJhJ`>W3zs)8T4o-vC|HP;s?{DE2+NMqr>t1XCjEAX@1Co7~W z<$WK;3%4IVyBHe!pmKy?bFWmAnaqENAS2fA66U|%PigFF$h|4}MC^(9AlqnJrWvq3 zth@ffrk3}^lCW?&2QKsTwTy~7QO2$-<#wd$yR#-|BU8*%VTh!`luxsz z`1V6S5lScKxykF5p8$2V`>WPQ=4h`tJv@F&U$V}EGxZlQ_~*edA1m)`b%tZ|<1nrj zN7uC?Ty>Fc&HLMa;mUC2BX&IT^Sz}>S?WDj0YX^o!1T+cgGmzWA{ZPxafVOz5xXwO zZztLDs1vTb_E1ni;NnT*kp|P5;F-YzG<3#!N#KnMSPL8kl)tUQ@hbv;7xfO4)6-2O zi+7%H5V#Kk*~3FnvUl$c-y`oPjgdWuUcWfg)ZvTYb|w2$Ipdh1&N+(2%1nwI%$~;+ zx@&{F>ksuPj#i~GXEzbgbgEjKmOk3ex@gqB($}Lo&0Hc)FV64Gf!WYn>E|_GH;&a7ugGNwA^)_RfRIf z)jlAnaQwC#w3&MTGAsOh*0zuPjK#(9_u{5MmEe=pZrN}rj#CUl#^W^-K(O1n!enLj-PGuHlFmcPFc--dJP8N?TULe z?!Y%pJZGW#7x4Ax=ZgW+T{KreuMH{B+<%7l5Ryc04D$`l>wT`_RZWGr#^=?2tR29u z{V))Ev$C5d`Rlf&a_9vb>3bIxKvV_v{#tbUC0>If^K z-l@mKl5U<9kTuQ|MQe2Qx9%rM5^i#)U9{|1*va!9H0~Cu8q`asAIy7kjO>ZWmu<#V z9A$l%OB}1aOHF1=%Te^AS}?P;D}tKu)3UuLxwLhwU?k&7n&h&w)|<@e%opzK#332C zLLS)V{Sg??ktN5Q3%JJ(qx3!sCHFrMFHa;I^P5>bzgZshcQdtjw$}N;{-@&;@f@N& zY$D;?1S`(&-QmHCBlp_b=|l!@k2t1-PeIzxqb!S<`!~3_lfaFR%tcL>=jY;!9`77G zkg>O?^)~(yrp$m$FaG%{!@{D(HGFAv--qd7x$S+sXv3b6)rq($-b$P07}9+|McCTg z;?S>ryLHIQLN0f{N!MB~0jq}3k~KN4z|fv7d#JuZF;1I2D|Jz_6P$bXOjY+cXtIPyH=wp(C2wS-1Iu>x$p+^phUyV?2Y=yZ1YV;`o}6iQxPo2}(oRVx3>o+&}3$QM0J zU|t&W|EAX}Z1WxQYrEeal1L*yei2_jU9rTfZS^B%znec-(I@pCv&lpX*H6kN$1nl? zCUR?`c%#D)6*`Ed$e`nKdTAdQm!I{(j@%|$lvPrm2V(i(#MnducP68@MS&mXR%pFT z1EtGHb3^yHv-tWhHR>?ieBlgXIUDuv1yD$i1zeM#N1y(+&`b`0WH|9ND3-`HA}M`v zn1EusU(v_g$S1m5v8Ltxouys-mNH+{<@#p5%^iu}4WJ_v+kX}0%T0mX?ZfqNg{|lB ze=aSh<-QXnJF+!*ELvkefO=5wn}qUC;bMtuZ-%B8%y2zS%iZS_G)SAT+Z}AZ*lSB= z@z8O(aZAgw=LHrUMfPvaJW^BY5&NNIub$FAe9Z6jSbb;+9nZgiwJ$1twMh??-^D9s zXp3`_sea>l0XIj{t$IV%+2VX*zO|*Y=KP?w*z3aBGZh6$13#j=ifAu;`H*>)L=(zk zfAyLS?QPzTOFkSBWnp$0;&*BI!#DD*X{lV5C@pnlNmqIDT>SoafB5AB3Y^WyQM;jj z(om)PPhP+f^Aly|!m2JV)TBP=@4gy0TRHeU2RZBeM>scOu*eUksxH+2ZeB^9@J0E0 z$LfEZ48B@_AD{wJ$8X6%e&K=+>-{95OBTJ!Xusdtw9-elJqRfO+p6Ji%~+#ZTLCZ@ z|0ja9ST6*I`qi)IvE}EgL%=AsoG=j3c|v>tVdrD){q<7%+4?xUwZNgsURBw zTs|NxTKGQ@-tqPAZ5FeypM|@CuLf9umcyWaPZ3~OhFa*M&w)`=QU5ItV~FZH(|EDQ z<4{?qvWWGbvB5#SxasWL;D3ewU&;`M#*Tgyd~%s;a75rf`Gb*4CeT@^i=pYDA!^6sqi^$;b2B(BfM5 z9*ddEdHY6f6e(({ho8}EPXDjP9>?}P;sFpUW#r2|7eX+w_)||83ueSpUN0WT}sGP+#v&QuX9qobZbU-!}Pf>h7kGylB}&oAereH^|o?QQ{s(TY(WGc>6uZ#PNa zrGWf@&EEtYa$S^S|9py?k%xr}#idi;e6ZkiS;xD5D{G z@y`OQbLPKK{Isb}*{C_ozl-!2%dQ1lOjQsR7 z;ACL`zi<#o=yEo6K{UZM8*5Kx>Vg_h_pii}s=uATQjXCuMw!ecJ_^wW+Rh_x87)WN zi7ynO27U_rcfA6>fXcgu;{Ao?(Ph+W1@`}<0tDMhIC*hS-M*=!$)ma9E5Kr*N$vc1 zRoWUsqqazx!0={)z!nqf-xZWpuW|ofB`RLD;oebBZJG89wHWVCJ!;NR{{WU$FU7^z z$*rDq@ih0eH-nlDDBMX?W>_A>lKgu}lP|KA3)OYWe|9XO)cp@=cO<}#MFmDqZIwBM|i zBubW#jOMJTCDdLin5F$U>cQnZn7IpKJ%&jcufQn&H(x{PPYm^s|N9V)=n+545m5(C z-Tx1(`+t{vFt!=B6~OT{KJ<=&?mB(t4`klgOpNnEVBWuQ0DLr=B?VD`?naMBqyIA)y}$<(P5po8&QSy#D?uXVe&L}hF#XSd z(rm{(2a*9z)Bnzp2VT=aL;SA>6hx+H(E{84&xc)6ENJ6o|9fp$)V2Nocd3dV&&|iq z0r6-!|Ic{Q^lHlsK!7;XD@{R~$XXfbV?wwTM4os83{zmvxhYXNm4@*Bxjqwz@Tmfi z>WWu{;L2f%&$$y3;93ZZY zgve#<>3Y1TaEA!$#INB4^4q@D^tvoU`-kE8$i`%2kp08 zgcUBIGHaL&^=N%&S}!tp=1)8zwbF^IcUNyehmMYA#A=Q);WOntAmn^A#`OJHx*hQH z<^#GM@@eX}P(!XkrfTS>ztxPy~?-)Z)GVw%$5py{8(u+Gr zLL&vpN&~1)FCtu*1rC7}5>M2G!!Ta?I~CsDKu#Iyu0v*uFeT*WFt?=LVM_3evw&jB zSYXq_3>mSmxfpXC2q(PtIGwAU4IfBwYtHicH$L&cUms1Ch8QQztj{D8&iT(X;EEq3 z0Xfcno+yCtBX=0Cmpm?;K15Vvs_qNzG7qs{MggP)7wSA5=F=6YX z^>iEInn+WtI%Cd7QCMZs5yspi@Ex*5M?G=;Y6a~b-~B(oktC4QsOJ}FOeCwK#=4Q; z_TFn2-?YZGs^~@_RplgK=g2<^L7f9b$Die^0DHKLwC$2tR!YLduOcG`_3PB z&AYR8TM4Tk;GPL3nY)Cuq^Tb{rMITQ-sSS}y1=fAdaz z@vt3TRmB7h60sbS5MNzS_uA@((;8D2N9j4TMDV#Dl(lPN@~?XrcZ_97CWMZ1qg_#~ ztr>-*wEbwAYO2PQn`-lfkWNG%)Qk*LO&l3Xl<1KT&;;hos-(>k?0Nq3*lj7>)9Rs6 zq!fFi+U%43fTC~G+%KFhN9~HecdMk?6FtAb-BG70*AUTWPi#|!NJg<1t@Y|*zOe}* zik&%AWlucT&pxF}${@W6Neq5To=~0XMD>9i4A$P^*4|z+rg*Kb>hyKZ-{BcOQ zD10);&_l0N7_;JSpzF@bap55Zk&dU|*UPqh11O@+Isk`0K>>7q6L#wC6o=N=7hVK}iaBn7v!{TP zBq5;dSfL@mhg00#o=db`?r5<;w&~=doc+G;vQnVWcvPhiK_%yE?=^ z$X?L^KkkO9XB+||eeYrgd|j0djo7V|LNbOS$4x(*FL6KQgvJMw-v-g%5jz>Sl+kXc zk|Wqnv%c)85|wrTnOHP^lJojL4irOjF)l0#+@0qD$z0TH%OKl}ifd_y!oJYR>B|^7+GX1ZE;$ zx(?4Ca@dUq?+t#sgE|746kI2%jHr47$guwHsCe#B^Lq-Zc|usuEg!^buW$l^IYQK^ zl&Cs-^a+h$pm8&j&lNAqkI!^#;BiS?p8}BK9Ng6YR!!&q&bgM9r^EK-dR?_XU)XI5w`L6~1C-)Bng7(axMnf) z+DIii;6wS=?edAQI^k1%`0|fIb5iwkFE13pE9xW@evFJ$EqU4a$ozP|Q{L%krHW%g zj=xhKB-K0}bNlas=s3mwd)JjMM)aY;qpX6djc0qczrU3l64OFmNv1DlWv2QsgRVL= zvUyhiKuuY_Nc%{H`~JCmdnFr@h(^-hr$#B=NW*FCxyJWjz6f3!imnL(}WBgFq{InnE_u z-vQXjiR8H3D~>SdwWJ0-8T!Q9QEnn7*B+vj&dPSCIV|*ws|s1YtpY`4pSQ52xzJV% zc$VF{u=ve`k|V3Vl)LQ)A}&g{Hu8Kz_)fFx-uP;6(O6pN>FLcBD_gJr;o+x{)B&Pq zNiS3COp)E>ll+r#1aL#e0CaM;q^a<}xPq8BGI}1g(OJ+Kio~;KjNt9wV zb!REH)n+PJ6SiI?+WM9qeeJTIX7BZi!!0>t!aAY?P08OHkX*0(1*9z;)>z|sX{t^Q zIXENptdB)>2dtgA^T))kbr~h7>j?-we7_J9&Qf{87Vw>xD!GB=J5jB_VV7xvN*>in zE?gkF2z2dvWKjO0MDHs*aZKdaYJNceJ8oZ?P0_D(B{GxzoiyYy@&r6R9(>qcWn`~e zQ*x_guKoAS{Iz(pn3Q<((VmSa9@vl*7iq02z@7C$IDRsBw-Id8MV&GExo}rl$VBHi zb?GbT_F7QgNBUX7l;KrmrD|`VO^nY1@x`ZHuDo>8d2lITHr&dCs_f@L(btQaNXH#32&d>ul$9<;=4YA3DzCX;YrxSb*j(<@+YIgfBQ~4PHNw2 zuKOXqsCpSoW$~kpUTI+Nook}F;>LQ^q=!}w&A zoLb~DU$obbpF#ZYcxEa+6TH_*L+%qo{qtGU#G9VxS>M&4ES59}`;+M&BYCmb$W#N1cMsEz2nLE7lF4eqI`F#E>&@*D<;)qXD zH@ALukw{13M7)IPZxvEmmVmn&#NSeK<_pAfsk290oP@ZKa%1tu_8(dOwyMqeRt}8>4(S`L-3=d7SS6Y?Bm(gx%kVS6?}S=%9#7 z4Y*W07m0bzq4sVG&2E+!%tfBHsK{dE>*MN`_Y#tGWebv-MCLA?$ynkq2&idtlI5mj zNOEkQrC0kE!2w~`HqShT=0KVPhj??O5(SdP6OKkyay~Ns952=NM1eeTqzJ&y2fkeo zv>uO^AXe`6q8C=)TXnhU=o^3KE0cU1t*<@NYx+j$W`Z!tsZA+#Sk6a zS^D6!YXx-+VoPapHQh)tU)BN26L<;s-@YK%ejfm|=`9UtIY|HTE-fRKlq2jsA1e4t zm9=;yt{|10%y+7A0hrv!3*eD5{U2Al9&oftCyr$QWK0OX3l2Q%dB-`p8Imi}3)BkR z+DWfGUUI*NyZUF7!T@Y*^6q)3%W(^2exqzmK@q%_Z5|Lq^E552yj3c3M48@dijHJy zHOu&|1DQGRv88*;x8Ha@U_Zbf{5>TK$0Sb{6{wic&`LE$#bZX(tubObeRTR~#N^l$ z55U_UU8FJfpt`5Q5Ba~&6#2cGfx{O6h~bR^1Pl>FQ72Gma@Q|G@lTD6g7O;i9&UBc zd3!*lajDX-UG+Hc+F61?QDLBd>%T4~u!Xvcz%8ieBYv~*Tw*f1ky~;Ttd;<_|ETm7 z|FpiiZx=ua8(Ziogm57$xhGFJZIWXv*9_PW*7D)bzU}w}7_$O6fCe`_HMQE|XOEa? z``;6??%*(baplAjvXE^<9!-IEQcWyB#-88zblBEQdCPlYc-Aj91ui102egiynH1k{ zDS*|7P7nooR343lSSPr2JCsk)=x46{%V&y076yW=Ky{U}8iYbtghi(7rVQ#p1#Hk& z(GOICBrcttZ3u}8>uH(TlLin!-lng}hBZ|=kVi6p3*q;|il$>DtmFtQ0!|=$oX0{# z{R7s2sjU?T@pX)FxhTGBtGeO(1eKAJYgQWSM*7}~!(l&2Rm9m$3K{JIx0zl(1f*?S znR$Djw|7xT`{mC9ZP)QLe)>q`b^>$W;UhxCp$EhjV8VR4qdozA7;H6V#}kS$9Dl_# zIi3mX91>~!BKE4lW-yG7rKDZk+=uzs;b&hIh7BA9dA3}u`xp6_%oK@FSx;9`J8LZ$ zM7(;Gv{Afs$$97kY-6OTT#8)J4d;#?@n!iCdzsw6N~z{kZE#ksN61q9khb~d+f}jW zqSP=Nk~m)e431J+E(AHb9_K;g%zDj-x=V2-F174{`ZCfL_EK5)3jFw)b!z%23Qn)O z830RX@@bE(zIvxA@O}{xlgSMD%PY4;n&ctb6cy=k=w}WwRYU@0{yTlBTbQ5PJEE>A zYYHLk#X?W_bS*sKo9%?IqfOWaY6EA*dDI@*whuq>TTAL-vR>_%7lw#dlcP^%Ag z!*iE19eRF4wPWne}P~+UZ_d8HM3Px1F_!C(xz`uxR*%b=>TYc*9XWL=| z8`jGVgG^+IsKpl$S0mV;D?=Z4#;L_gN7OC_0W0^9o98kk&6BD7?O3kCrg0nuVh6Z} zhFAV_bHs%WgCQQayT7|c^sbqQ+N>T-T z^8!|X+E^uvkeOb5zU@zD?jizta&wYfVM(xd(IT|&{^13_mwyaftq{(ePrhngaue84 z#O-AGuFb<4?Rk)sy4M2079E1AZj?jChHcAGn6wxCReKOi)A=@erf6pUj-XVtM%$6z z)}X7P3Qxa}PQ_sAwW;uv-_BGxizhSDd{Kh^)~~*tcGt$w(BVgeNVrTWdWo&N0oI=4b9 z(OIU$>q1}pC;_>Os`RO>nn-(TEWfS=ETOXlXSxz^kJIrt71or z09)lN#A%SVo!;s-%{i`%PQ-pTI!e$&#(gWXBcn*DeG83+>9Hdq2=soH@+D z7^oi6X)WK((+qC1M5eent-s7roIi~v7@`X5==EGpkq8n=tcD`qdx0-K`dN)@9&CLK zP{&W%ypq@B?3M7v@ztYtz9LDq1GYTz6S6G^<|)9}evU6>qiFasx}4BfUE!jJdG;V; z{fZ%~s^&>(ox}&Gw_g|NQvTI+_@)uc7TeMO+VEA=r@Y3e&&;t#G43;-IbT(|)Pqm` z$(+B0g=y5A-k4AIU$lUl|DNUH$OD`gSc8hhWWo)A4HMG+2QJRfkV92<@BP09s#H;j zyzTiNrt#c!4`nv50V4S_YlXX;?TeN`$o4~ggE&EI_=EKKB`4V^L2IrRp&_W(mzUBytj^@f*u9&FEX2!ohN@V@Ye6 zM1-U`h4X%k$=;hMHJe8s6AC;u1L%Xua75=0emlwPc5ICHy^kc_5C7BH!Z*rKGxP;c z9H}4xD7$!~XRv`^t-nNJnm{8W7V-xJgUjGq()Fh@(wf^|5|q8{1NK5}%1Z*7zb)lE zWe-O{G9%TFaUMZjX=S(c(7&ur->g999`&!fRI{afX~1+C^0U0ggF0dW&Rji_bIO!d zPM$^@C?X3t{WmeL-kyc6JXn(45W2jAP_C&*zC2;TH%JY7nY||V9JJhKY6#L4rSYB( z@#R|0)>HIcxg(8-UZ)4FS{pD5Xv9}wTJhhlNHYW8v;3SuqaoC!9GFF2D*{ZR6Q1(gC|YCKSeje zzx-KS{iAXj;DW~31apoS@{dUXHfK~XQ5~-eA$wcTKWvG?V!9LX3&H}bHzsI&03NN)?E`*;8RP(Di$;H;)9#sC5}YXN)XeO(5z7= za+of5@oxc4Ox*KRGE;v5RObQIF<58DA6LXBPF4v2^`8!cp=C;7wce&vok&tn3PIGi z)*}503HP5opTmZLM96UbEs{(*bCv^w$1oi_i@>CQY^(#H9ku$E$$7k}j zR~Rm^_n(_rxEXTPFAYO28GPsnkMV08`)mFm`!BS25*tVC4!+!5t zSz|!zyJz%t*N^XiSx=Wem%yOmOUHWuo~gT<#Ujbpv=Bb*jZ1=B@s9sJh!8FhiS*-Y z7*sb2hGodyA%Q|A?EweY%SQ8p~p?82{LM0C)8+ko22>n8)E&)Kfi9P$2 zfElPZKT`j|f6}-|OW1&!Ly~uC{JD<>AH_>CIs>?{NfmU>2s!1+jLcxeMV{r@;#~)= z4xp!o+2X>w>IptZ=ZR;2vQv*siRXFxF&Z6Ek$Qle8l<#Edj|Uw=t+gZ-{F{80X16E zOJ1}k(I4(or!m+~x9u#8K$c@2xIsQee?T6FCqf|$5*hpMSUq+oIFne5rn&gzZ0Z4* z54X}6A6bHpkLW5B>9CDH+34NE*yRMN=c`_b2MD@bS zfJXl!GE2eX3$EKu{^|9r7$To#ShZIQ-|6lH!x}8gnBzG#un`xq*#enEXgEQ!rScCA z-P0eCiweDG|4|<$J{-aG6~9PE#q^2NUxNerSS?FD9;~fS&&RxbTQ33Lj`J6>$d6%y zZs=~>e`0Lk_}K-Tt5G_-Cl)?>D`)}!*!bx&5ovxl^L7t1ScYQ}@C3kDdLIDQEB$5V z<8Kk=hH(Qx(PqVO?LsJk`%gjnRc9(xFHqrJn%8Nryjp+Qd*U6i%TMZ7{HCk1cS_MC zBt*M3b1Eun!x1eK$>>K~KAc6q9rJ$cKWcC>Lpnl%<}t@!9WC3!@iTGt-bLth3`4?RArnL<;2SfJzaUKe6Ze5rio*!T@qQ6uE3c|QD(llC! zR59^Y7dl%a$nSaD`*3p75jbP@n zNugBkUvgQ*=s(ysx696bQIRrv@89>(@Bm*ow|B}JJqiQxqg{O3*Qf#%0h3Q9XsTXGs&(S8zaebEFi8ZQsM7e8;oURM;;$897WmjfR=uw)@Tj*oJazi_MZ-YpCpDt>mDG&i+OG!7< z1#;RNSL#(AEvR#;VdBJ7t&YA)a;Z=$$A3Nm4D4VYeW*#Vz1DfB@i5oicziTX- z?#gopV*^bCnTkW)rqsv}bwa8yKK<+Y)6WB?E}ma|>9-5xqo%odafd2(;AI<`(&<%Q z7(W&$%;s+(b9m3+nb51iA7|`b3W3@#LMWEc>#!F;vtrwJ|IRY?|EkLpy`G2p7JD?< zbyc55FwC$o0fvZzEnwWWM#6sR23{;Gf|yb1^Bj36OSfLulGUYo3qbcymLMU*M*>tt zfKb=~H1v1~Zk6~|=xEQ8mz_E3K$?TTAGI2yWdTIF_I<^6PwNn(m9PH1Wn;lAKfGS{ zeyHnr083DImPoL%xm8q2y5_vj%!>lbx!iC>SzKuo0EORZJ_5E2=p!nixKeslUB664 z6TIF;`y`CXR3K3mpfLP^SF`&A!2oh<0eS#M+z?MJ#FbtJ0eebk{d$w8icmf5KoW`# zxFktQqsn;p1GL}{ua|ccm6K6jZ@V3IA^H9cxZy?9Aw|=rkW&&RY|Y{TpoobKJJq|O z%Q=L++}lk^Bml%`z(p!gj)zqPWKryt|HWdscNNg!R0419o(B7JG|0}JJ+C7NbKGb{ z5zl#2cyes-*n*0;#}Yyh8Zx4|cjm6y2uWnhEChn2)CxkL*jj}UN%k`-b=w^qd>!~} z1vIo@_5hHB30PnE@1ymkBC3|R&XR!&}cKOmQ0>8%cC zrBZ}WNdiEvo+gPISTGtPG6uFx(07e427s8*SmKFagk@pDpf^CMV2A*8nyCf}pv@_4 z^1ponWEjRde;h)jp7xGqDHKA_$mW(2bj3p_LK`&G3f$0OY5{^_0D)`%#t3vhZw>kh z(2TLJsOWNxtcZX)JFehoET0GXa=E->>;mHM{U~e!9n3Je=}mX-)jvb=gTE&)Ti)re zK@mKgmwghM%?qxumNy(5)_ZFE=|!IlM-M%#-SvBzd1mG74SJxLoC@&W?~JCszCZ7% zvc>+!0hqtJ9pr4><2hCaB>yghW!D&x1085F#(=!L=V|RJ_VmF6oxEJWaj*)Kr0ZhM z5l}9V)*S?*sq)Q-vYj^P!w(=soCDqRQC`pn9S1_N;szNhVHwjHJ7&saIJ@4Eh(j^H zOm22{pBQ>qg1v~*by5&Fg%s(GUIA`5aCShn1n1u>`UKQ(s-+ zHU^}01OQ2bQ|0)}Sasg)%}N&r{n-VYe1av}nqusRZ^<JC24YMfYXN(63LNP;%TruV+b_C4^_D4>K)2aLJF z_kKG6Pj7D-7WEhXixMJAO2g0~k`mG|3=I;}DJ=qmgp@MGP)bOusC38BB}jvkqB0;L zNDZM33^5?xXXF2O?tOosd(OP$VdfjV*4m#~J0uzX)y;c{z@H;vIf9|Aw$+fWzz*Ot z6Ou73VCZS7vQJ%Z%fJ^T9{d9lJuh`2#V!4P3u=Z^k5g^{rllfEFY0~*_AUYd-yVd) zw}`RqsJ+al8-QF{7DZ7SN@@NLe*HnJoYM_TIOoW1y-{t7|Cb&0%*HS$v?8#%PVTc3 zzY9keKsg#x!b2FJWyscevr(DyW#%Bqws_PEX-2F7TgB$*bU!a8Rm%0nkLZ|{4cdgS zCV3?Z*zZeb<%ZK$blT92ZhV2a=_NX+l%|ued|pLVlUpK>J$m8Ew?Ut(hj~vUqM6TC z$Gw_$&JMsNGq9;nqOcTM?FC@Cvx@w3JJwBprmB%JhnTo@^>xxq*ZsPyIgWMr$&68D zA|QOdl&yJ{3S?8xi~FtEt+TI=F}4=cJCc?u-yJrm;aaMlIp7qC19#ZdLni@Ln|I|P zBf20;h$YV!nKEVt?@+9rh^aezfwwVaLM&b-rK8%}8Hj-1#F~~{KC3dFpo5`!$9-1~ z%dQ%*H1OyH&S0Ikwsb1myHU2fk0q*?-)8eeRds2Go`IIzsEcVb&-w@Z_!0;<-Se(s zt}b37!B8X_=r;Z4FO+^RJ-bvh*JU~Tgff!=0m93_8bCjufR!OX~*J7Ug}r4uB&R1F602$5kIkp zY{v(Hx(w0Z_4XY-GL;jAsHqZ5o(4S{_(L;#tK4v@HjY6Z%@VfT1y9z3BF1w479(sk zovxe&R#wnoh+;Jg8NrxpH%s|mhc>7jiUVsT~du5N}&?jMB|-H+b43BbF3mbRX!;-HtItOGzb zSWY^*QWX)jo(4q3d4HgQdJvMlK?dH=J8wW@pyhk7 zMuO1ZQ56iH&Q!8y{cKR%r6tGT;< z&HW}|EbPog?CCRSZ{asl42e^X#mLs=^`SK3#tZzAB(nO5!UnKEkr4=nP}yh}NUN&3 z_;{O?8@7(_`L>ltSu8*YZdu+Q0!)%5bO^f8{pu;z2y?(od&}F2Tvm>Nu(T`s;Rt_Z zLOt}#yohfUoF1J)A6u8!be3JGOP;}(ajKFm&d}YWk#aA)SzvXp{N-H=W&-BxAd`1d z&dhc0*5&FbZUY-xbI?zr(Wy-(-GRJRvspC@;K1xRfWJ2!)laNj>e-QIca(IyaFKMg zQO^ggNeS)!n3*TX@AC~JNn&b1!|adZ2&w6m05xnPi`Rz=j{FuwnYgZ4so~KE zeoXSANiQhd9+41Tce#A>MK+{;2beiYl4+=uTKIFmJ5sNwioeg)8Zq%0{>0y|Q_o^l z2b{GJZh93)d4yNn4Nu-+G6s#NHODSM7jSYd-8&Kj*?D4l8{zWY65=VF#0kZjEBhk| zr&v^Ojc(W74=$*nA5Cqz%(a{6K5qR>wEJUf-3n-{hxCWPnO-ak{ z^epJ=GycGY>F2xKRubq3>yibfJHzNv-1)>n4A~pKgn&gM!pV!B>G)_yZb24Os`($i zaguanH;|t5>VLBIr#8XMjJ0_O2#<%Q83K&SRU>yerXMym-2?QI@lc4VM_dy-$MTp} zbIG)M}qr^X6lyN6rYUqDD_b)zsDw$v@+H72#78VemnI#m5yGUe^7Wkt>~~ zd5~3YH)zk`uL2n6`+5~>MQppNCxFRLVzNJr1~#MEm0mSJ>oC*;@xEZp3hybNNVz&& z{#(X>$+^PTW$6BNNu2$-98r^(BEBT!iF=&^>~rQ|PO2!37v`Ry@MuhR1wK{TNovx4m-Qo3&q?v8@U@J{#W|*v4A$mtm-2&VL00KB-=9 zYD9Zr&%5^ut4zgvwzjW0szF*b)cVWP9as(nlh>Lo%BpzWTLWRu#(VtL=KXzMeX=rg zjsILALf1#cI@QhMl*QlnT?HZG#WnXCu;lFrJul2d7s|OJvN#8qF%KS5<8^I4zBJ+} zahw3h0sp0FeG*6YI_f+oOgRCF4GHi#-u{#A$6PIuP<;YA_4yfFr7@xKGXUK{X@N!H zfB!q_>x+5tuGdc#xPA;bcZ`1}6h8kGlR=0lYVx1cpb-6~%;2M))fjCp@bdigq<|YA zw@c0Y*fBW0ae0G}_x~75jC3J#`avgcVR50i7MDSt1wgG`8XfRf_l3C@AF-<|6}<9-U{%3Kr^2dZP6>6 z?>=}f;|iXr(a{J3`tSM3R3OK9=!OGKW5!QGmVwip6ap7j?fd7`=(=`SH3zbFa0^^J zGp34^X{;t@C4uuDG2Y#q4==j`7+?Cl#=hNpPd)NhZTA`1l!_XSnjcbt>{yK}kb;Vp z`V%Hg1b|Rc1bCum|Jf-Rzt_B`iMh+2uSveZEjzuGrFzJM7nav;aA{P+3tI@PAaNt> zmY{04NcrBbdJF~s`_G?af5@iTg9-%O&QQoA1|N@%?!QLv?r#u}T3E_=M|j`OA;J@- z{;vngz`g-Mv@4v!V-v9}xBuPf)&IU&lb(@yPS5IBwOxRnWQc`QfF}FT1R`p?Ckf>5 zw)}&|bUVPxb(7%IfdW2^RlGeP<|+Z|YbX^;J9>z%~i#&vi>g@xIJ?YpL zhih(lO7xtLm$oE4`v4iF7~R4#LR$^T;8ah)Acv16y|3q^Jx0 zG$0F*_nN~jTR>EAc4NfnymcH|$?hM7hnI>RQ@{z<-R}vx95(hd1k8X7dL7evKVhnm zN@~eEDaSi#3ol&>n`(JatvnE+=a%Db;P!n0at$x+WN8uVM}-2|y8K5E-g4 z0EU`IQGv(sy@J=H2ML^zybEawA*XhtnaHK>I%j6*rgU$jy!nDsc=JQH)MqK>2ymHZ z(kqfIjIY6rJ>Oz^r39QSMBUmAikY3QI8I z_;*kI0kXC4uVohZm2CM&ZCR81fo>}Xk4f3@z`4>S0j7+P3)n!Yo{)9T~gkFPBuoxo1 zBEBtix>FUN@YEpEJLH`h=`vpY?B%`9OYu}*6W-ZWOjZ&gyoRFsa?Q;e!hswK@@O)F z9CN7N!>3jzT8yxce3IkpK?p;8y^cauwImigT__VINYvzQ9rCyY7rjZG+$>bNxt=cr zNI**R(xN+JZNhCJ2MuJ-osQYnTlnaXIoh#E< zDa#DCW)%@+6if;?MaRb)QmQX}T+0l6z93L4KNu_Gfcl#*~H)V6**#JKou+KfO!mAG`#SxbVgA?5T-< ziSli)u?qk`1*cTiA2ki~;Q*6(y?ym`Q`S%%2*pZ@fn8;TX69mG%E6X^Pq#a2}tUb}&}DrM(j0K>=* ze43!Bc5n%1ZBinVesVi(j9fiDrgWZ|f#Yur$3t&2cD(ph5=7yjmglq%VMNFwE%WVe z9asGU6AY#P3mTH`5<$ngwx^^~B9P3iCsby$Y zO%kb*o%j>eDx0&a`<~f#BN5rb?K0bxsBB9qvS#vbIb@M(;i+@ns%7IQOQ-T3_}Ed+ zXGi(QlAT2YUBeuPDvuiI;EsWf@eKM)>4gFA2-Y#jMYdzDeFwm~bZ-NZYd@078p6_$ zW0>}8WFEa?@KD4ANPGM4*T>ZRF3UeFIIiNHmKpXBOL7@(M6f1}o)y0;m#*)??YTkE z+xkC%ES_bwY`_&ATjfKw;t*K9yaY7*RRMbX~(PW z|9Ob@$w`$d8eWdVjh+ja`35WJ#GR&djOU-Q`ocP`Y(hSUv%UNrtT|!Q(7q<~{V2F0 z(U(}}>XEN>;1z-)n|TelBr$8gfqc1(Q>E6y{)Ds=R<=hoernTQ6Y!N|y-1U#kE12T z*E9RPg%iNM0RwM`Onv*lCVK64tFb%^$6D?DnQ?a~06q6xEV$Ta0+^qMZZMtOboS~_ zfDI{BaT-|WWGA*V!#4#TqOeWH)vSjt3^%(Lwr%u%e-=bYWBRsj3S@-G1B2~|m!}Vw zbp09IhERLmsdYajO_#g+Kjh#=OYDXuI=R4IR|wH@LGt=#Z$yXN1CkTXAyfS1v-Msa zgU;v<1=}??NnWDg? zF=Ty_5tY_bL8px=BVAKr;0m%%#i}LFKv@er(5ek*0_@Wh*~R=_eszM@Ihgp$4c^xw ztn;ngPg?-p5QrAmJY&VQa!j3QH!QryPg2gt?>}1q0~W(CP=u%wtc#G8yB2qc^48_P$?99woC+NL1o`k)u-e8{({CNLfPAgXtM(Zg7Gd3J8XcKGg zDz?Ub{!ND2sYF_=tPzt+P{>a*HR(3Ye;1#Ju7`z{WSqKYn=*iJ>bf2|=^_XJ8C_~5( z=NnP_&jQEYuzg{xsC%yOv(!%nba!J<g+c7dGk`#s5+8&X#(T``vtA$$zCA`V8d~a%DR7PwuoG5gCD^ zj`l^rCl>X_gh=YRuy@=Pd$tY8+>5XL$;5lHZgP&vGokc#WiD_8v-iv6?$Cq4la=^Y zNGZZ=u3hiZgh7VwX_mfux4Fz-l5x?`fMBnazLf&F;BjGA^TEBFOK9tEhPEy5MOslg zkL-}aEFVR**F3JNYdF%Bt$!Y39&}O^T4_QxM;G!=AtalbDRhF4up@SQs-8T{zdA{D z1rxuWunWz^q&^Gna}DXB37uQDJ@LGPF)Bm)g^HYH;@tR@aW&2PfvvC=m&z^1s5@A( zyjrxmHLyh39GEz38PsOB8v2kElLWI4JgQ6xXCEq9)J~xA*DdG7MQE8k9pN~0dC z9|<)JY+$t9UPhMYuqB%StSh2<*Dx2shO29(26b7q*m#(P9AnG{LleHG^D z?Y1(SjlJ2%GRM4+gdKVzA9bL@Flc5(9MGU>xaHzjBF(c%JiXdcPx%AKx1U&+-{EU} zXY*@%DhSaFNRh))?%yWL_OU+T1J{nmJ%Z?d>j&W=IhVe@-RH2x*PWXN8e71 zgg)j;WxrU9PnX_UZdSEqm1957|JFBHT~b3fuEQ4OZ<~Sw(|I_5RKb3aoMGufJD8MN zaWA7nXxP6#BX`W+7^`mIN12NDbWj|Y9^4Tg>xOrJ`_%;VopYSxHh##;z_$KOcZar~ zvreb2?`#J{?kDmzX~XX0Ia%L_1~5UFt$h9rWt{IZS;D-YWs2zyY=*$3973b`(l-V|$+sdG|L#mM6wH)2(x62;q^2TPU=<9TP(0-lAv>q;-Z~B7q-Cy~;T# zRYN)*uk_EutA1%+6}UB3zwVZV`-LOuRkTtZc($-#wl;u`5k?qhN86%*iQ|pz$g-%( zcA))v@i9jErHQ|lCW?sI77qh(t@juvFHTrgH-}1*0cazxmC1ue>9g#9@sFBBkcc_+ zW7~sc{~p;mbrYS*%)`=~4 zUf%MxNw1Ds`TK%Iw~Rv@2TJuOI}k>GJ6(W7DJ_CN@3ee0cp5$>l|7-*aPj)krhBW6 z;v2)*lZP_Mk%RLaJG-&j{ktkTrE@@j^XRaLWBlS|E{=E-1_E<`VOF=)z|-wEdY&Rg z7gSxXnZCE}Ky6~T-7}=*gq5n=QDmS!x^n_9MlZK9>s(GRr!m$^DM!XPTwnO8QgFTg zVHxU;#Q5YSri(4NyH6?#svHrlF^nRKm!CdewxLU3XXRbpvOJw$m;JhsLA60D=uV!v}%5iVQ_&`M- z5I9*j$$art=)SH8*+4j$x$g4#RPRXq{2Zm#RE?a1gjWZ-1^(o#3X1KBF593z^5@`= zI^HU)BPC!TE#s!{6=OZLh1w7EmIZnLTmIVUi6lG^dXUx9kEtzs{D|3I~j?YV2Ef8UU#IbHlkoo*XyQnC;M*As$W(`VmOS;436#685A!IDZH^VDD+g~?F;8B`>!WsuNeEtL)ZA` z29B&XJGox<;){#*Z5KNjci%kBTz~%~%fyg{K}-I<(K8!J?&fC>qS3=cEBW~+@fFr5 zj;)*9yUmU!R$NtUc!eve%d{KMb^Wg^dC$_EkYTjHDvaQcWGgV6{t9PCRlarFL<*m2 zJ@$g%nT&RRZqhxN;=SM7TX+m(IY`^!yv^&AN*bnMUSCF4d!w4qHd%IV^sngewTKeu zyQ?2Jbzx+sVSScbY4vi-tm)pc_gt;Qr4Wh^@5UGD3f&=3)#`j+MD)BIL{*!ZX zB*gmK4;`CX`-z}$ZIQrZkfURr>ccm|bIxK|8l-Ud1N-9GheFqT%|v&hdrG)HeLfLXiXCs|A@7}XlwVV*A+xt8BO;Ymt!?Ed5ZYMQRi5}`-1nQb> zZuh0?Klde0&Ek3|n_ygZ5HT<-@f~ZjaikpYz`C1J9sObr~iprO0FSq$x_37 zN(9p%$?0!Rw8-s1`|Xo?tfiLBpEO0~0J`i&-?hUOBFTI?;ku{7MWxiIF#epB+qE_a zeaI(|GAF_-wy^HBMC?_|wbyoa&~{gnhYWx-&UVZ`!wgV&uco*qcFfG^WKyi?~<-l4EHqtTR48 z&dU<(4mG@3dv>y9c}Fmu{H>em`l`jYUvY_L0)$x92aDRqx-a;Ux;Q zD&CdicUI>hvOVo;+8J7Uxmh(Hry>p3)lo$aie;Ic-H|80&Q*MF>1Uy?HX5LBf>|yK zIX`NX_{NeS(boPc=50vgHj#_!s&D&}1EN%s0SKxNELO$&6tcs~<4xI*yT3G3wSCgV zK9hf(hbxX!fGf=Ztzfqd{<>tU_~4TVNP^EiEw~HnnxGga05Z;T8DuXXwNIjWvKP~1 zJdG#`rJVwPrZsp&d-gIG)cp^QLuRe`n3c)@Y`1L!cli<{h3nroiWu#X+_kVI^qQQD zdJ$ri(Z9H@F^{H@aiyh%%?Cccg|i#xUFzORwOtW6>FU`@)%!bn=6y$>^$k2{ttf8? z{-$`X3hi5Br>SUo(8DO-^nN#m;mI`LpmX}*wF+iYN5^3YrEJt|v2jrdi9hTkF7n6l z5m8&g`o_m3!OgD22=UD6{QDj_NC!P}MXU1Nij(T=p@)fVDF!NLscXeQL zrW&D=zD9(vF?&H)@@W4TgtXuYEVPy6y02D)M}uaraNNY_x#%W31&k1->h&q_?Z6xQ zqtmZDCKQGsTlbZms2^^$OdQCG+6_k#s^uGftR^HSY8wmOmvtc~e_7bI8Zz{rPN3=2 zCt}nZ(}i_HCB~yUpLeN1W<_k)Yb8$5{&kU~Tc$ckGGCVn}NUf&%Q762}KXZ5wf!ppdauTo5pKn|ZVtJ`O zJK-T_OWCTFRvo2KGO^o2#}ytT2Vk`x?kV?CezRC#aoi= z&`G@e2aEo>AFPLVyk8kQ@7RK7_GQHGb`5nbovp7^VvZa#quh^LP=9Sy{k-3&<4lCTtW;ymTS&@Wuq8aUjyIE&zB#eK``Igz#=OQfM7l$1i#eU>xWaHe3j>>3xMB4g@kManU7wM)e3qT&F`R=4DP_3r&)(e| z7#5tw{B{Q{7M%9-gnP^%m`fc)WW3azr=fd_I41Mw9!xO@5xG_;meip(i zO6|>taeb`cM5lT7Oo*p(QZ?@Zv(QsaTXqJKxbIG^9UUyZr^EgIx5YM)Fm_X@)}nu( z$OuyhiM$$rfLzP2pa+j)j28d97Pf{yJB7X-v+6ETxq5`fdU&AU>toR%>&RE=~G-MgFbUSao$h` zdnD}P9m)y^l&!*(4pN7T5R}SVcO_Zqs-s^FTfzm59g;T&RZZUU zLYo#&THe3S)j*x^hk$+E;x_4(kvqOaJ{Z%K@5aHNn#+EHN$d|}Vz3;Cuwtq)#@p=5 z{JdR%GP%!QCKYF6zft<-Y#WMjz6*MwmdWY(cuW?4Bj^1NZR$O%El-=VUw5!=*H4A` zUsDZ3rb0wPc~kz|CgD(Cp6F*|t^$7qRsx_^Y03N79NbOPJH*YtGSy=_?}f>&i9uvi93P6u^uw-DLt+9} z?%gj(Q#$YUkPQEn!Iqc@XM$tfn`7p=bB-$aYSRPU2722j+CK!pmnjYDxKp-KND}`v ztKM>Qs4Sqwch(o{oc!2oY62$N?l6g3X|4WT!Ja=+kIHoH4uRe*qJc@2#bDbG(v*`N zO*4ISf)-`2P}Wsp(x(nAjc(RNC8uwftic%j8)PNJ$t8R9f@UU_e&LrD)Tt1u1r}t@ zyqZTUg35&PSfsDNzx1Xb$dK^xEY}Y=FVR(+1GRzYCbX2MUVP>UV^BuE292{hjY@;E z^3P_ajLe|GBq>(w^X2A_;^ai$j|vryei8@ z`F-FU9;>pJ8@~*%vL@z}9N1u~ZY=n7G=a*C_CVSH(SX?|e6zmI?N(kRAdchZ0kAwv zV`xQKcZ*I^YK&HT6ToYOx5 literal 0 HcmV?d00001 From 5685fea7f410c1b4f4bbfbd7bf1d15f000a7b4da Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 15:41:39 +0800 Subject: [PATCH 272/304] test(chat-format): modernize coverage with Qwen3.5-style templates Replace the legacy Mistral-focused chat format tests with self-contained Qwen3.5-style Jinja template coverage. - verify ChatML system, user, and assistant message rendering - cover enabled and disabled thinking generation prompts - test image and video placeholders with vision identifiers - validate tool definitions, tool calls, and tool response history - add clear error coverage for invalid message structures - verify model-specific stop token criteria - keep the tests independent of tokenizer files and model weights Signed-off-by: JamePeng --- tests/test_llama_chat_format.py | 368 +++++++++++++++++++++++++------- 1 file changed, 288 insertions(+), 80 deletions(-) diff --git a/tests/test_llama_chat_format.py b/tests/test_llama_chat_format.py index f031bf72b7..4860a4c7d6 100644 --- a/tests/test_llama_chat_format.py +++ b/tests/test_llama_chat_format.py @@ -1,89 +1,297 @@ -import json - import jinja2 +import numpy as np +import pytest + +from llama_cpp.llama_chat_format import Jinja2ChatFormatter + +QWEN35_EOS_TOKEN = "<|im_end|>" + +# A compact Qwen3.5-style template keeps these tests independent of model files. +QWEN35_CHAT_TEMPLATE = r""" +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, is_system=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if "image" in item or "image_url" in item or item.type == "image" %} + {%- if is_system %} + {{- raise_exception("System message cannot contain images.") }} + {%- endif %} + {%- set image_count.value = image_count.value + 1 %} + {%- if add_vision_id %} + {{- "Picture " ~ image_count.value ~ ": " }} + {%- endif %} + {{- "<|vision_start|><|image_pad|><|vision_end|>" }} + {%- elif "video" in item or item.type == "video" %} + {%- if is_system %} + {{- raise_exception("System message cannot contain videos.") }} + {%- endif %} + {%- set video_count.value = video_count.value + 1 %} + {%- if add_vision_id %} + {{- "Video " ~ video_count.value ~ ": " }} + {%- endif %} + {{- "<|vision_start|><|video_pad|><|vision_end|>" }} + {%- elif "text" in item %} + {{- item.text }} + {%- else %} + {{- raise_exception("Unexpected item type in content.") }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- "" }} + {%- else %} + {{- raise_exception("Unexpected content type.") }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception("No messages provided.") }} +{%- endif %} +{%- if tools %} + {{- "<|im_start|>system\n# Tools\n\n" }} + {%- for tool in tools %} + {{- "\n" ~ (tool | tojson) }} + {%- endfor %} + {{- "\n<|im_end|>\n" }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content( + message.content, message.role == "system" + ) | trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception("System message must be at the beginning.") }} + {%- endif %} + {{- "<|im_start|>system\n" ~ content ~ "<|im_end|>\n" }} + {%- elif message.role == "user" %} + {{- "<|im_start|>user\n" ~ content ~ "<|im_end|>\n" }} + {%- elif message.role == "assistant" %} + {{- "<|im_start|>assistant\n" }} + {%- if message.reasoning_content is string %} + {{- "\n" ~ (message.reasoning_content | trim) + ~ "\n\n\n" }} + {%- endif %} + {{- content }} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- set call = tool_call.function %} + {{- "\n\n\n\n" }} + {%- for name, value in call.arguments | items %} + {{- "\n" ~ value + ~ "\n\n" }} + {%- endfor %} + {{- "\n" }} + {%- endfor %} + {%- endif %} + {{- "<|im_end|>\n" }} + {%- elif message.role == "tool" %} + {{- "<|im_start|>user\n\n" ~ content + ~ "\n<|im_end|>\n" }} + {%- else %} + {{- raise_exception("Unexpected message role.") }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- "<|im_start|>assistant\n" }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- "\n\n\n\n" }} + {%- else %} + {{- "\n" }} + {%- endif %} +{%- endif %} +""" + + +@pytest.fixture() +def qwen35_formatter() -> Jinja2ChatFormatter: + return Jinja2ChatFormatter( + template=QWEN35_CHAT_TEMPLATE, + eos_token=QWEN35_EOS_TOKEN, + bos_token="", + add_generation_prompt=True, + ) + -from llama_cpp import ( - ChatCompletionRequestUserMessage, +def test_qwen35_basic_conversation(qwen35_formatter: Jinja2ChatFormatter): + response = qwen35_formatter( + messages=[ + {"role": "system", "content": "Be concise."}, + {"role": "user", "content": "Hello"}, + ], + enable_thinking=False, + ) + + assert response.prompt == ( + "<|im_start|>system\n" + "Be concise.<|im_end|>\n" + "<|im_start|>user\n" + "Hello<|im_end|>\n" + "<|im_start|>assistant\n" + "\n\n\n\n" + ) + assert response.stop == [QWEN35_EOS_TOKEN] + assert response.added_special is True + + +@pytest.mark.parametrize( + ("enable_thinking", "expected_suffix"), + [ + (True, "<|im_start|>assistant\n\n"), + (False, "<|im_start|>assistant\n\n\n\n\n"), + ], ) -import llama_cpp.llama_types as llama_types -import llama_cpp.llama_chat_format as llama_chat_format - -from llama_cpp.llama_chat_format import hf_tokenizer_config_to_chat_formatter - -def test_mistral_instruct(): - chat_template = "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}" - chat_formatter = jinja2.Template(chat_template) - messages = [ - llama_types.ChatCompletionRequestUserMessage(role="user", content="Instruction"), - llama_types.ChatCompletionRequestAssistantMessage(role="assistant", content="Model answer"), - llama_types.ChatCompletionRequestUserMessage(role="user", content="Follow-up instruction"), - ] - response = llama_chat_format.format_mistral_instruct( - messages=messages, +def test_qwen35_generation_prompt_thinking_modes( + qwen35_formatter: Jinja2ChatFormatter, + enable_thinking: bool, + expected_suffix: str, +): + response = qwen35_formatter( + messages=[{"role": "user", "content": "Solve this problem."}], + enable_thinking=enable_thinking, ) - prompt = ("" if response.added_special else "") + response.prompt - reference = chat_formatter.render( - messages=messages, - bos_token="", - eos_token="", + + assert response.prompt.endswith(expected_suffix) + + +def test_qwen35_multimodal_content(qwen35_formatter: Jinja2ChatFormatter): + # Qwen3.5 assigns separate sequence numbers to images and videos. + response = qwen35_formatter( + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "image.png"}, + }, + {"type": "text", "text": "Compare this with "}, + {"type": "video", "video": "video.mp4"}, + ], + } + ], + add_vision_id=True, + enable_thinking=False, ) - assert prompt == reference - - -mistral_7b_tokenizer_config = """{ - "add_bos_token": true, - "add_eos_token": false, - "added_tokens_decoder": { - "0": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "1": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "2": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - } - }, - "additional_special_tokens": [], - "bos_token": "", - "clean_up_tokenization_spaces": false, - "eos_token": "", - "legacy": true, - "model_max_length": 1000000000000000019884624838656, - "pad_token": null, - "sp_model_kwargs": {}, - "spaces_between_special_tokens": false, - "tokenizer_class": "LlamaTokenizer", - "unk_token": "", - "use_default_system_prompt": false, - "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}" -}""" - - -def test_hf_tokenizer_config_str_to_chat_formatter(): - tokenizer_config = json.loads(mistral_7b_tokenizer_config) - chat_formatter = hf_tokenizer_config_to_chat_formatter( - tokenizer_config + + assert "Picture 1: <|vision_start|><|image_pad|><|vision_end|>" in response.prompt + assert ( + "Compare this with Video 1: " + "<|vision_start|><|video_pad|><|vision_end|>" in response.prompt ) - chat_formatter_respoonse = chat_formatter( + assert response.prompt.count("<|vision_start|>") == 2 + assert response.prompt.endswith("<|im_start|>assistant\n\n\n\n\n") + + +def test_qwen35_tools_and_tool_history(qwen35_formatter: Jinja2ChatFormatter): + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + response = qwen35_formatter( messages=[ - ChatCompletionRequestUserMessage(role="user", content="Hello, world!"), - ] + {"role": "user", "content": "What is the weather?"}, + { + "role": "assistant", + "content": "I will check.", + "reasoning_content": "A weather lookup is required.", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "London"}, + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call-1", + "content": "Sunny, 28 C", + }, + ], + tools=tools, + enable_thinking=False, + ) + + # Tool calls and their responses use Qwen3.5's XML-like markers. + assert '"description": "Get the current weather for a city"' in response.prompt + assert "\nA weather lookup is required.\n" in response.prompt + assert ( + "\n" + "\n" + "\n" + "London\n" + "\n" + "\n" + "" in response.prompt + ) + assert "\nSunny, 28 C\n" in response.prompt + + +@pytest.mark.parametrize( + ("messages", "error"), + [ + ([], "No messages provided."), + ( + [ + {"role": "user", "content": "Hello"}, + {"role": "system", "content": "Too late"}, + ], + "System message must be at the beginning.", + ), + ( + [ + { + "role": "system", + "content": [ + { + "type": "image_url", + "image_url": {"url": "image.png"}, + } + ], + }, + {"role": "user", "content": "Hello"}, + ], + "System message cannot contain images.", + ), + ], +) +def test_qwen35_rejects_invalid_messages( + qwen35_formatter: Jinja2ChatFormatter, + messages, + error: str, +): + with pytest.raises(jinja2.TemplateError, match=error): + qwen35_formatter(messages=messages) + + +def test_qwen35_stop_token_ids(): + # Verify that model-specific stop token IDs terminate generation. + formatter = Jinja2ChatFormatter( + template=QWEN35_CHAT_TEMPLATE, + eos_token=QWEN35_EOS_TOKEN, + bos_token="", + stop_token_ids=[248044], ) + response = formatter(messages=[{"role": "user", "content": "Hello"}]) + + assert response.stopping_criteria is not None + criterion = response.stopping_criteria[0] + logits = np.empty(0, dtype=np.single) - assert chat_formatter_respoonse.prompt == ("[INST] Hello, world! [/INST]" "") + assert criterion(np.array([], dtype=np.intc), logits) is False + assert criterion(np.array([1, 248044], dtype=np.intc), logits) is True + assert criterion(np.array([1, 2], dtype=np.intc), logits) is False From 998b7b5a5318ab6d53c66a7cf79f7760e5b9f83c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 26 Jul 2026 17:34:37 +0800 Subject: [PATCH 273/304] fix(_internals): clean up native resources on initialization failures - Register native model and batch ownership immediately after allocation so later validation failures cannot leak llama.cpp resources. Free a loaded model when vocab lookup fails, and route mixed-batch setup failures through idempotent cleanup. - Initialize sampling-context resource fields before fallible setup and make partial teardown safe to repeat. This prevents missing attributes from interrupting cleanup when sampler-chain construction fails. - Clear model, vocabulary, and sampling parameter references after native context and sampler resources have been released. This prevents closed wrapper objects from unnecessarily keeping models and related Python objects alive. - Add failure-injection tests that verify model and batch handles are freed exactly once and partially initialized sampling contexts release their resources idempotently.Extend lifecycle tests to verify that parent references are cleared and that repeated close calls remain safe. Signed-off-by: JamePeng --- llama_cpp/_internals.py | 96 +++++++++++++++++-------- tests/test_llama.py | 152 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 30 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index a41764d7d9..9b37ebcc74 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -54,11 +54,14 @@ def __init__( self.params = params self.verbose = verbose self._exit_stack = ExitStack() + self.model = None + self.vocab = None + self._lora_registry: Dict[str, LlamaLoraAdapter] = {} model = None if not os.path.exists(path_model): - raise ValueError(f"Model path does not exist: {path_model}") + raise ValueError(f"LlamaModel[__init__]: Model path does not exist: {path_model}") with suppress_stdout_stderr(disable=verbose): model = llama_cpp.llama_model_load_from_file( @@ -68,15 +71,20 @@ def __init__( if model is None: raise ValueError(f"Failed to load model from file: {path_model}") - vocab = llama_cpp.llama_model_get_vocab(model) - - if vocab is None: - raise ValueError(f"Failed to get vocab from model: {path_model}") - + # Record ownership immediately so every later failure can release the + # native model. In particular, a failed vocab lookup must not leak the + # successfully loaded model. self.model = model - self.vocab = vocab + try: + vocab = llama_cpp.llama_model_get_vocab(model) + if vocab is None: + raise ValueError(f"LlamaModel[__init__]: Failed to get vocab from model: {path_model}") + except BaseException: + llama_cpp.llama_model_free(model) + self.model = None + raise - self._lora_registry: Dict[str, LlamaLoraAdapter] = {} + self.vocab = vocab def close(self): """Manually free LlamaModel and Vocab/Lora resources.""" @@ -574,6 +582,10 @@ def close(self): self._exit_stack.close() self._exit_stack = None + # The context no longer needs to keep its parent model alive once the + # native context and its callbacks have been released. + self.model = None + def __del__(self): self.close() @@ -1025,6 +1037,7 @@ def __init__( self._token_buf = None self._owns_token = False self._exit_stack = ExitStack() + self.batch = None # llama_batch_init allocates either batch.token or batch.embd: # @@ -1046,25 +1059,30 @@ def __init__( f"llama_batch_init({n_tokens},{embd},{n_seq_max})" ) - if mixed: - if bool(batch.token): - raise RuntimeError( - "LlamaBatch[__init__]: expected batch.token to be NULL for " - "mixed embedding batch initialized with embd > 0." - ) - if not bool(batch.embd): - raise RuntimeError( - "LlamaBatch[__init__]: expected batch.embd to be non-NULL " - "for mixed batch." - ) - - self._token_buf = ( - llama_cpp.llama_token * self.n_tokens_capacity - )() - batch.token = self._token_buf - self._owns_token = True - + # Take ownership before validating or allocating supplementary Python + # buffers so close() can release the native allocation on every failure. self.batch = batch + try: + if mixed: + if bool(batch.token): + raise RuntimeError( + "LlamaBatch[__init__]: expected batch.token to be NULL for " + "mixed embedding batch initialized with embd > 0." + ) + if not bool(batch.embd): + raise RuntimeError( + "LlamaBatch[__init__]: expected batch.embd to be non-NULL " + "for mixed batch." + ) + + self._token_buf = ( + llama_cpp.llama_token * self.n_tokens_capacity + )() + batch.token = self._token_buf + self._owns_token = True + except BaseException: + self.close() + raise def close(self): """Manually free LlamaBatch resources.""" @@ -1894,6 +1912,19 @@ def __init__( self.model = model self.params = params + # Initialize every resource-bearing attribute before performing work + # that can fail. This keeps close() safe for partially initialized + # instances. + self.prev = None + self._cur_p = None + self.sampler_chain = None + self.grammar_sampler = None + self.reasoning_budget_sampler = None + self._logits_view = None + self._logits_ptr_addr = None + self._single_token = None + self._single_array = None + self.vocab = llama_cpp.llama_model_get_vocab(model.model) self.n_vocab = model.n_vocab() @@ -1940,7 +1971,6 @@ def __init__( self._build_sampler_chain() # Grammar sampler - self.grammar_sampler = None if params.grammar: self.grammar_sampler = GrammarSampler( model, @@ -2256,12 +2286,12 @@ def close(self): # Free grammar sampler if it was initialized. # This releases underlying llama.cpp sampler memory. - if self.grammar_sampler: + if getattr(self, "grammar_sampler", None): self.grammar_sampler.close() self.grammar_sampler = None # Free the sampler chain and all attached C samplers. - if self.sampler_chain: + if getattr(self, "sampler_chain", None): self.sampler_chain.close() self.sampler_chain = None @@ -2279,7 +2309,7 @@ def close(self): self._cur_p = None # Clear token history deque to drop references. - if hasattr(self, "prev"): + if getattr(self, "prev", None) is not None: self.prev.clear() self.prev = None @@ -2291,6 +2321,12 @@ def close(self): self._single_token = None self._single_array = None + # A closed sampling context must not keep the model or configuration + # graph alive merely because the wrapper itself is still referenced. + self.vocab = None + self.model = None + self.params = None + def __del__(self): try: self.close() diff --git a/tests/test_llama.py b/tests/test_llama.py index df379df7fe..b233ea5266 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -19,6 +19,158 @@ MODEL = "./vendor/llama.cpp/models/ggml-vocab-llama-spm.gguf" +def test_model_init_frees_native_model_when_vocab_lookup_fails(monkeypatch): + native_model_handle = object() + freed_model_handles = [] + + def model_path_exists(_path): + return True + + def load_native_model(_path, _params): + return native_model_handle + + def fail_to_get_model_vocab(_model_handle): + return None + + def record_model_free(model_handle): + freed_model_handles.append(model_handle) + + monkeypatch.setattr(internals.os.path, "exists", model_path_exists) + monkeypatch.setattr( + internals.llama_cpp, + "llama_model_load_from_file", + load_native_model, + ) + monkeypatch.setattr( + internals.llama_cpp, + "llama_model_get_vocab", + fail_to_get_model_vocab, + ) + monkeypatch.setattr( + internals.llama_cpp, + "llama_model_free", + record_model_free, + ) + + with pytest.raises(ValueError, match="Failed to get vocab"): + internals.LlamaModel( + path_model="model.gguf", + params=object(), + verbose=False, + ) + + assert freed_model_handles == [native_model_handle] + + +def test_batch_init_frees_native_batch_when_validation_fails(monkeypatch): + class InvalidMixedNativeBatch: + token = object() + embd = object() + + invalid_mixed_batch = InvalidMixedNativeBatch() + freed_batch_handles = [] + + def allocate_invalid_mixed_batch(_n_tokens, _embd, _n_seq_max): + return invalid_mixed_batch + + def record_batch_free(batch_handle): + freed_batch_handles.append(batch_handle) + + monkeypatch.setattr( + internals.llama_cpp, + "llama_batch_init", + allocate_invalid_mixed_batch, + ) + monkeypatch.setattr( + internals.llama_cpp, + "llama_batch_free", + record_batch_free, + ) + + with pytest.raises(RuntimeError, match="expected batch.token to be NULL"): + internals.LlamaBatch( + n_tokens=1, + embd=1, + n_seq_max=1, + mixed=True, + verbose=False, + ) + + assert freed_batch_handles == [invalid_mixed_batch] + + +def test_context_close_releases_parent_references(): + context = internals.LlamaContext.__new__(internals.LlamaContext) + context.ctx = None + context.model = object() + context.params = object() + context._exit_stack = None + + context.close() + context.close() # Closing an already closed context must be a no-op. + + assert context.model is None + assert context.params is None + + +def test_sampling_context_partial_init_can_close_idempotently(monkeypatch): + closed_resources = [] + + class MinimalModelForSampling: + model = object() + verbose = False + + def n_vocab(self): + return 8 + + class TrackedTokenDataArray: + def __init__(self, *, n_vocab): + assert n_vocab == 8 + + def close(self): + closed_resources.append("token-data") + + class TrackedSamplerChain: + def close(self): + closed_resources.append("sampler-chain") + + def get_sampling_vocab(_model_handle): + return object() + + def fail_sampler_chain_build(_sampling_context): + raise RuntimeError("sampler chain build failed") + + monkeypatch.setattr(internals, "LlamaTokenDataArray", TrackedTokenDataArray) + monkeypatch.setattr(internals, "LlamaSampler", TrackedSamplerChain) + monkeypatch.setattr( + internals.llama_cpp, + "llama_model_get_vocab", + get_sampling_vocab, + ) + monkeypatch.setattr( + internals.LlamaSamplingContext, + "_build_sampler_chain", + fail_sampler_chain_build, + ) + + sampling_context = internals.LlamaSamplingContext.__new__( + internals.LlamaSamplingContext + ) + with pytest.raises(RuntimeError, match="sampler chain build failed"): + sampling_context.__init__( + params=internals.LlamaSamplingParams(), + model=MinimalModelForSampling(), + ) + + sampling_context.close() + sampling_context.close() # Closing an already closed context must be a no-op. + + assert closed_resources == ["sampler-chain", "token-data"] + assert sampling_context.model is None + assert sampling_context.params is None + assert sampling_context.vocab is None + + def test_llama_cpp_version(): assert llama_cpp.__version__ From 866bed9565c1be3fd3c2985d71a62c05c9f1eea3 Mon Sep 17 00:00:00 2001 From: Emptyngton <40150265+emptyngton@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:56:43 -0400 Subject: [PATCH 274/304] fix(loader): guard HIP_PATH and VULKAN_SDK dirs with os.path.exists os.add_dll_directory() raises FileNotFoundError [WinError 3] when the directory does not exist, so a stale HIP_PATH or VULKAN_SDK left behind by an uninstalled SDK makes "import llama_cpp" fail outright on Windows. The CUDA_PATH branch above already guards each candidate directory with os.path.exists(); this applies the same pattern to the HIP and Vulkan branches. Valid directories are still added individually, so a partially removed SDK contributes whichever of bin/lib remain instead of raising. --- llama_cpp/_ctypes_extensions.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index a9a2c02e50..3634720681 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -118,13 +118,19 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list # Add HIP runtime DLL directories when HIP backend is available. if "HIP_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) + hip_path = os.environ["HIP_PATH"] + for sub_dir in ["bin", "lib"]: + full_path = os.path.join(hip_path, sub_dir) + if os.path.exists(full_path): + os.add_dll_directory(full_path) # Add Vulkan SDK DLL directories when Vulkan backend is enabled. if "VULKAN_SDK" in os.environ: - os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Bin")) - os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Lib")) + vulkan_sdk = os.environ["VULKAN_SDK"] + for sub_dir in ["Bin", "Lib"]: + full_path = os.path.join(vulkan_sdk, sub_dir) + if os.path.exists(full_path): + os.add_dll_directory(full_path) # Add package-provided library directories. # From 8e1ea5ef1b91a88bb26e9bc809b8aa645785c479 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 27 Jul 2026 22:43:32 +0800 Subject: [PATCH 275/304] Update Submodule vendor/llama.cpp 8bb9093..b77d646 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8bb909374d..b77d646751 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8bb909374d04d40621340aee5ba2245860027fdc +Subproject commit b77d646751d01c0962bc203b6809e9d94f7d50b7 From 194dfb29e9dc504f942949328a52c7d4f372d445 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 28 Jul 2026 00:17:11 +0800 Subject: [PATCH 276/304] fix(ctypes): support GCC/Clang mangled symbols for optional llama_ext APIs - Add missing `_Z` Itanium C++ ABI symbol variants to ctypes function lookup lists. This improves compatibility with Linux and macOS builds where C++ symbols are exported using GCC/Clang name mangling. Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2f402cd74b..317a435489 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -5092,6 +5092,7 @@ def llama_opt_epoch( "llama_graph_reserve", "?llama_graph_reserve@@YAPEAUggml_cgraph@@PEAUllama_context@@III@Z", "__Z19llama_graph_reserveP13llama_contextjjj", + "_Z19llama_graph_reserveP13llama_contextjjj", ], [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32], ctypes.POINTER(ggml_cgraph), @@ -5115,6 +5116,7 @@ def llama_graph_reserve( "llama_ftype_get_default_type", "?llama_ftype_get_default_type@@YA?AW4ggml_type@@W4llama_ftype@@@Z", "__Z28llama_ftype_get_default_type11llama_ftype", + "_Z28llama_ftype_get_default_type11llama_ftype", ], [ctypes.c_int], int, @@ -5134,6 +5136,7 @@ def llama_ftype_get_default_type( "llama_model_n_expert", "?llama_model_n_expert@@YAHPEBUllama_model@@@Z", "__Z20llama_model_n_expertPK11llama_model", + "_Z20llama_model_n_expertPK11llama_model", ], [llama_model_p_ctypes], ctypes.c_int32, @@ -5150,6 +5153,7 @@ def llama_model_n_expert( "llama_model_n_devices", "?llama_model_n_devices@@YAHPEBUllama_model@@@Z", "__Z21llama_model_n_devicesPK11llama_model", + "_Z21llama_model_n_devicesPK11llama_model", ], [llama_model_p_ctypes], ctypes.c_int32, @@ -5166,6 +5170,7 @@ def llama_model_n_devices( "llama_model_get_device", "?llama_model_get_device@@YAPEAUggml_backend_device@@PEBUllama_model@@H@Z", "__Z22llama_model_get_devicePK11llama_modeli", + "_Z22llama_model_get_devicePK11llama_modeli", ], [llama_model_p_ctypes, ctypes.c_int], ctypes.c_void_p, @@ -5186,6 +5191,7 @@ def llama_model_get_device( "llama_set_embeddings_nextn", "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", "__Z26llama_set_embeddings_nextnP13llama_contextbb", + "_Z26llama_set_embeddings_nextnP13llama_contextbb", ], [llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], None, @@ -5212,6 +5218,7 @@ def llama_set_embeddings_nextn( "llama_set_nextn_layer_offset", "?llama_set_nextn_layer_offset@@YAXPEAUllama_context@@H@Z", "__Z28llama_set_nextn_layer_offsetP13llama_contexti", + "_Z28llama_set_nextn_layer_offsetP13llama_contexti", ], [llama_context_p_ctypes, ctypes.c_int32], None, @@ -5236,6 +5243,7 @@ def llama_set_nextn_layer_offset( "llama_get_embeddings_nextn", "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", "__Z26llama_get_embeddings_nextnP13llama_context", + "_Z26llama_get_embeddings_nextnP13llama_context", ], [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float), @@ -5253,6 +5261,7 @@ def llama_get_embeddings_nextn( "llama_get_embeddings_nextn_ith", "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", "__Z30llama_get_embeddings_nextn_ithP13llama_contexti", + "_Z30llama_get_embeddings_nextn_ithP13llama_contexti", ], [llama_context_p_ctypes, ctypes.c_int32], ctypes.POINTER(ctypes.c_float), @@ -5271,6 +5280,7 @@ def llama_get_embeddings_nextn_ith( "llama_set_embeddings_layer_inp", "?llama_set_embeddings_layer_inp@@YAXPEAUllama_context@@I_N@Z", "__Z30llama_set_embeddings_layer_inpP13llama_contextjb", + "_Z30llama_set_embeddings_layer_inpP13llama_contextjb", ], [llama_context_p_ctypes, ctypes.c_int32, ctypes.c_bool], ctypes.POINTER(ctypes.c_float), @@ -5294,6 +5304,7 @@ def llama_set_embeddings_layer_inp( "llama_get_embeddings_layer_inp", "?llama_get_embeddings_layer_inp@@YAPEAMPEAUllama_context@@I@Z", "__Z30llama_get_embeddings_layer_inpP13llama_contextj", + "_Z30llama_get_embeddings_layer_inpP13llama_contextj", ], [llama_context_p_ctypes, ctypes.c_int32], ctypes.POINTER(ctypes.c_float), @@ -5311,6 +5322,7 @@ def llama_get_embeddings_layer_inp( "llama_get_ctx_other", "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", "__Z19llama_get_ctx_otherP13llama_context", + "_Z19llama_get_ctx_otherP13llama_context", ], [llama_context_p_ctypes], llama_context_p_ctypes, @@ -5330,6 +5342,7 @@ def llama_get_ctx_other( "llama_model_target_layer_ids", "?llama_model_target_layer_ids@@YAPEBHPEBUllama_model@@@Z", "__Z28llama_model_target_layer_idsPK11llama_model", + "_Z28llama_model_target_layer_idsPK11llama_model", ], [llama_model_p_ctypes], ctypes.POINTER(ctypes.c_int32), @@ -5349,7 +5362,8 @@ def llama_model_target_layer_ids( [ "llama_model_target_layer_ids_n", "?llama_model_target_layer_ids_n@@YAIPEBUllama_model@@@Z", - "__Z30llama_model_target_layer_ids_nPK11llama_model" + "__Z30llama_model_target_layer_ids_nPK11llama_model", + "_Z30llama_model_target_layer_ids_nPK11llama_model", ], [llama_model_p_ctypes], ctypes.POINTER(ctypes.c_uint32), From 1e49f9da22b2125ed9d032ad88a9fd9afa8e4f2c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 28 Jul 2026 04:04:19 +0800 Subject: [PATCH 277/304] feat(tools): add cross-platform ABI inspection utility Inspect PE, ELF, and Mach-O exports and normalize platform-specific symbol names. Validate optional llama_ext ctypes aliases across Windows, Linux, and macOS builds. Keep artifacts and timestamped privacy-safe reports local to the repository. Signed-off-by: JamePeng --- .gitignore | 11 +- tools/abi/README.md | 164 ++++ tools/abi/__init__.py | 3 + tools/abi/__main__.py | 5 + tools/abi/artifacts/.gitignore | 4 + tools/abi/artifacts/README.md | 20 + tools/abi/output/.gitignore | 4 + tools/abi/output/README.md | 7 + tools/abi/scan_dynamic.py | 863 +++++++++++++++++++++ tools/abi/tests/test_platform_artifacts.py | 85 ++ tools/abi/tests/test_scan_dynamic.py | 194 +++++ 11 files changed, 1359 insertions(+), 1 deletion(-) create mode 100644 tools/abi/README.md create mode 100644 tools/abi/__init__.py create mode 100644 tools/abi/__main__.py create mode 100644 tools/abi/artifacts/.gitignore create mode 100644 tools/abi/artifacts/README.md create mode 100644 tools/abi/output/.gitignore create mode 100644 tools/abi/output/README.md create mode 100644 tools/abi/scan_dynamic.py create mode 100644 tools/abi/tests/test_platform_artifacts.py create mode 100644 tools/abi/tests/test_scan_dynamic.py diff --git a/.gitignore b/.gitignore index fad7f43313..b5d60bf894 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,15 @@ local_settings.py models/ docker/open_llama/*.bin +# Repository-only ABI tool inputs and generated reports. +# Keep only the directory instructions and local ignore rules tracked. +/tools/abi/artifacts/* +!/tools/abi/artifacts/.gitignore +!/tools/abi/artifacts/README.md +/tools/abi/output/* +!/tools/abi/output/.gitignore +!/tools/abi/output/README.md + # C extensions (llama_cpp bindings) llama_cpp/*.so llama_cpp/*.dylib @@ -208,4 +217,4 @@ docs/_build/ # Installer logs pip-log.txt -pip-delete-this-directory.txt \ No newline at end of file +pip-delete-this-directory.txt diff --git a/tools/abi/README.md b/tools/abi/README.md new file mode 100644 index 0000000000..ea509f2f7a --- /dev/null +++ b/tools/abi/README.md @@ -0,0 +1,164 @@ +# Cross-platform ABI inspection + +Author: **JamePeng** + +This repository-only tool inspects PE (`.dll`), ELF (`.so` and `.so.*`), and +Mach-O (`.dylib`) exports. Its primary purpose is to collect ctypes symbol +candidates and verify optional `llama_ext` bindings across MSVC, GCC/Clang, +and macOS builds. + +## Boundary and safety + +The tool is intentionally excluded from wheels: + +```toml +wheel.packages = ["llama_cpp"] +``` + +It is not imported by `llama_cpp`, has no installed command, and keeps LIEF +out of project dependencies. Run it only from a trusted source checkout. +LIEF parses native binaries, so do not scan untrusted artifacts. + +Install the maintainer-only dependency: + +```bash +python -m pip install lief +``` + +The tool and its documentation use the same MIT License as this repository. + +## Artifact layout + +Run commands from the repository root. Put builds under +`tools/abi/artifacts`, or replace that argument with an external absolute +directory: + +```text +tools/abi/artifacts/ +ā”œā”€ā”€ windows-x86_64/ +ā”‚ ā””ā”€ā”€ +ā”œā”€ā”€ linux-x86_64/ +ā”‚ ā””ā”€ā”€ +ā””ā”€ā”€ macos-arm64/ + ā””ā”€ā”€ +``` + +Names are not significant. `--select-symbol llama_decode` identifies the +llama library by content when dependency and backend libraries share the same +directory. + +Artifacts may come from local builds, an installed or extracted wheel, +[project releases](https://github.com/JamePeng/llama-cpp-python/releases), or +[upstream releases](https://github.com/ggml-org/llama.cpp/releases). Record +the source revision, compiler, architecture, and build options. Upstream +artifacts may not contain fork-only `llama_ext` APIs. + +## Scan exports + +```bash +python -m tools.abi scan tools/abi/artifacts --recursive +``` + +The default output is one same-named JSONL file per library: + +```text +tools/abi/output/ +ā””ā”€ā”€ 20260728T153012.123456Z/ + ā”œā”€ā”€ llama.dll.jsonl + ā”œā”€ā”€ libllama.so.jsonl + ā””ā”€ā”€ libllama.dylib.jsonl +``` + +Useful options: + +```bash +# Select only binaries that export llama_decode. +python -m tools.abi scan tools/abi/artifacts --recursive \ + --select-symbol llama_decode + +# Print instead of writing per-library JSONL. +python -m tools.abi scan tools/abi/artifacts --recursive --format text + +# Write one aggregate file; its filename receives a UTC timestamp. +python -m tools.abi scan tools/abi/artifacts --recursive \ + --format jsonl --output all-symbols.jsonl +``` + +`--prefix` is optional. By default all exports are retained: + +```bash +python -m tools.abi scan tools/abi/artifacts --recursive \ + --prefix llama_ --prefix ggml_ +``` + +## Check optional llama_ext bindings + +This is the primary ABI validation command: + +```bash +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive \ + --source llama_cpp/llama_cpp.py +``` + +It statically reads ctypes decorators without importing `llama_cpp`. The +default `--scope optional` checks declarations marked `required=False` and +returns exit code 1 if any candidate is missing. Other scopes are available: + +```bash +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive --scope required +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive --scope all +``` + +## Compare and create a manifest + +```bash +python -m tools.abi compare tools/abi/artifacts \ + --recursive --select-symbol llama_decode + +python -m tools.abi manifest tools/abi/artifacts \ + --recursive --select-symbol llama_decode \ + --output llama-exports.json +``` + +Cross-platform comparison uses `canonical_name`: + +```text +?llama_graph_reserve@@... MSVC +_Z19llama_graph_reserve... Linux Itanium ABI +__Z19llama_graph_reserve... Mach-O symbol table + ā†“ +llama_graph_reserve canonical name +``` + +Records retain `raw_name`, ctypes `lookup_name`, `canonical_name`, ABI, +address, ordinal, library filename, format, architecture, SHA-256, and UTC +generation time. They never contain the artifact's absolute source path. + +Every run receives a timestamp, preventing normal output from overwriting +previous results. Generated artifacts and reports are ignored by Git. + +## Verification + +Unit tests are independent from the project's default test suite: + +```bash +python -m pytest tools/abi/tests/test_scan_dynamic.py -q +``` + +The opt-in integration test requires Windows, Linux, and macOS artifacts: + +```powershell +$env:LLAMA_ABI_ARTIFACTS = "tools/abi/artifacts" +python -m pytest tools/abi/tests/test_platform_artifacts.py -q +``` + +```bash +LLAMA_ABI_ARTIFACTS=tools/abi/artifacts \ +python -m pytest tools/abi/tests/test_platform_artifacts.py -q +``` + +Without configured artifacts, integration tests skip. With +`LLAMA_ABI_ARTIFACTS` set, a missing platform or optional ABI alias fails. diff --git a/tools/abi/__init__.py b/tools/abi/__init__.py new file mode 100644 index 0000000000..0a387e6759 --- /dev/null +++ b/tools/abi/__init__.py @@ -0,0 +1,3 @@ +"""Cross-platform shared-library ABI inspection tools.""" + +__author__ = "JamePeng" diff --git a/tools/abi/__main__.py b/tools/abi/__main__.py new file mode 100644 index 0000000000..acfe21acfa --- /dev/null +++ b/tools/abi/__main__.py @@ -0,0 +1,5 @@ +from .scan_dynamic import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/abi/artifacts/.gitignore b/tools/abi/artifacts/.gitignore new file mode 100644 index 0000000000..dbea0aa373 --- /dev/null +++ b/tools/abi/artifacts/.gitignore @@ -0,0 +1,4 @@ +# Keep downloaded or locally built native libraries out of Git. +* +!.gitignore +!README.md diff --git a/tools/abi/artifacts/README.md b/tools/abi/artifacts/README.md new file mode 100644 index 0000000000..30863ea8c0 --- /dev/null +++ b/tools/abi/artifacts/README.md @@ -0,0 +1,20 @@ +# ABI artifacts + +Maintainer: **JamePeng** + +Place trusted Windows, Linux, and macOS build artifacts here for local ABI +inspection. Filenames do not need to follow a fixed convention. + +```text +tools/abi/artifacts/ +ā”œā”€ā”€ windows-x86_64/ +ā”œā”€ā”€ linux-x86_64/ +ā”œā”€ā”€ macos-arm64/ +ā””ā”€ā”€ macos-x86_64/ +``` + +Downloaded and copied content is ignored by both the local and repository +`.gitignore`; only this README and `.gitignore` are tracked. `git add -f` can +still deliberately override ignore rules. + +See `tools/abi/README.md` for commands and artifact provenance requirements. diff --git a/tools/abi/output/.gitignore b/tools/abi/output/.gitignore new file mode 100644 index 0000000000..f12e9f061e --- /dev/null +++ b/tools/abi/output/.gitignore @@ -0,0 +1,4 @@ +# Keep generated ABI reports local. +* +!.gitignore +!README.md diff --git a/tools/abi/output/README.md b/tools/abi/output/README.md new file mode 100644 index 0000000000..3601ac132e --- /dev/null +++ b/tools/abi/output/README.md @@ -0,0 +1,7 @@ +# Local ABI reports + +Each run is stored in a UTC timestamp directory. Reports omit artifact source +paths but may contain binary hashes and non-public symbols. + +Generated content is ignored by both the local and repository `.gitignore`; +only this README and `.gitignore` are tracked. Review reports before sharing. diff --git a/tools/abi/scan_dynamic.py b/tools/abi/scan_dynamic.py new file mode 100644 index 0000000000..8ee11fd8c6 --- /dev/null +++ b/tools/abi/scan_dynamic.py @@ -0,0 +1,863 @@ +"""Inspect and compare exported symbols in PE, ELF, and Mach-O libraries. + +This repository-only maintainer utility supports collection and verification +of cross-platform ctypes symbol candidates, with particular focus on optional +llama_ext APIs. + +LIEF is imported lazily so that ``--help`` remains available when the optional +dependency is not installed. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import sys +from collections import defaultdict +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Sequence + +LIBRARY_SUFFIXES = {".dll", ".dylib", ".so"} +__author__ = "JamePeng" + + +class ScanError(RuntimeError): + """Raised when a shared library cannot be inspected.""" + + +@dataclass(frozen=True) +class SymbolRecord: + """One exported symbol and its cross-platform names.""" + + raw_name: str + lookup_name: str + canonical_name: str + abi: str + address: str + ordinal: int | None = None + + +@dataclass(frozen=True) +class BindingDeclaration: + """One ctypes decorator declaration extracted without importing llama_cpp.""" + + python_name: str + candidates: tuple[str, ...] + required: bool + line: int + + +@dataclass(frozen=True) +class LibraryScan: + """Metadata and exported symbols for one binary architecture.""" + + library: str + format: str + platform: str + architecture: str + sha256: str + symbols: tuple[SymbolRecord, ...] + + +def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def generation_timestamp() -> str: + """Return a sortable, collision-resistant UTC generation timestamp.""" + + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + + +def _enum_name(value: Any) -> str: + text = str(value) + return text.rsplit(".", 1)[-1] + + +def get_format(binary: Any) -> str: + value = str(binary.format).upper() + if "MACHO" in value: + return "Mach-O" + if "ELF" in value: + return "ELF" + if "PE" in value: + return "PE" + return str(binary.format) + + +def get_platform(binary_format: str) -> str: + return { + "PE": "windows", + "ELF": "linux", + "Mach-O": "darwin", + }.get(binary_format, "unknown") + + +def get_architecture(binary: Any, binary_format: str) -> str: + header = binary.header + if binary_format == "PE": + return _enum_name(header.machine) + if binary_format == "ELF": + return _enum_name(header.machine_type) + if binary_format == "Mach-O": + return _enum_name(header.cpu_type) + return "unknown" + + +def normalize_symbol_name(raw_name: str, binary_format: str) -> str: + """Return the name used by ctypes/dlsym and cross-platform comparison. + + Mach-O symbol tables prefix external C names with an underscore. dlsym and + ctypes callers use the source-level name without that platform prefix. + """ + + if binary_format == "Mach-O" and raw_name.startswith("_"): + return raw_name[1:] + return raw_name + + +def detect_abi(normalized_name: str) -> str: + if normalized_name.startswith("?"): + return "msvc-cxxabi" + if normalized_name.startswith("_Z"): + return "itanium-cxxabi" + return "unmangled" + + +def canonicalize_symbol_name(normalized_name: str, abi: str) -> str: + """Recover a source-level name from simple global C++ mangling. + + llama_ext functions are global functions, so their MSVC and Itanium + spellings can be mapped without a full ABI demangler. Namespaced, + overloaded, and templated symbols remain mangled to avoid false matches. + """ + + if abi == "msvc-cxxabi": + match = re.match(r"^\?([^@?$]+)@@", normalized_name) + if match: + return match.group(1) + + if abi == "itanium-cxxabi": + match = re.match(r"^_Z(\d+)", normalized_name) + if match: + length = int(match.group(1)) + start = match.end() + candidate = normalized_name[start : start + length] + if len(candidate) == length: + return candidate + + return normalized_name + + +def _symbol_address(symbol: Any) -> str: + value = getattr(symbol, "address", None) + if value is None: + value = getattr(symbol, "value", 0) + return hex(int(value)) + + +def _exported_symbols(binary: Any, binary_format: str) -> Iterable[Any]: + if binary_format == "PE": + if not binary.has_exports: + return () + return binary.get_export().entries + + # LIEF's exported_symbols filters undefined ELF imports and non-exported + # Mach-O symbols, unlike dynamic_symbols/symbols. + return binary.exported_symbols + + +def _iter_binaries(parsed: Any) -> list[Any]: + # A universal Mach-O may contain several architecture slices. + if type(parsed).__name__ == "FatBinary": + return list(parsed) + return [parsed] + + +def scan_library( + path: str | Path, +) -> list[LibraryScan]: + """Inspect one library, returning one result per architecture slice.""" + + try: + import lief + except ImportError as exc: + raise ScanError( + "LIEF is required for ABI inspection. Install it with: pip install lief" + ) from exc + + library_path = Path(path).expanduser().resolve() + if not library_path.is_file(): + raise ScanError(f"Not a file: {library_path.name}") + + try: + parsed = lief.parse(str(library_path)) + except Exception as exc: + detail = str(exc).replace(str(library_path), library_path.name) + raise ScanError(f"Failed to parse {library_path.name}: {detail}") from exc + + if parsed is None: + raise ScanError(f"LIEF did not recognize {library_path.name}") + + digest = sha256_file(library_path) + results: list[LibraryScan] = [] + + for binary in _iter_binaries(parsed): + binary_format = get_format(binary) + records: list[SymbolRecord] = [] + + for symbol in _exported_symbols(binary, binary_format): + raw_name = getattr(symbol, "name", None) + if not raw_name: + # PE supports ordinal-only exports. They cannot be matched to + # Python bindings by name, so keep a stable synthetic label. + ordinal = getattr(symbol, "ordinal", None) + if ordinal is None: + continue + raw_name = f"#{ordinal}" + + lookup_name = normalize_symbol_name(raw_name, binary_format) + abi = detect_abi(lookup_name) + canonical_name = canonicalize_symbol_name(lookup_name, abi) + + records.append( + SymbolRecord( + raw_name=raw_name, + lookup_name=lookup_name, + canonical_name=canonical_name, + abi=abi, + address=_symbol_address(symbol), + ordinal=getattr(symbol, "ordinal", None), + ) + ) + + records.sort(key=lambda item: (item.canonical_name, item.raw_name)) + results.append( + LibraryScan( + library=library_path.name, + format=binary_format, + platform=get_platform(binary_format), + architecture=get_architecture(binary, binary_format), + sha256=digest, + symbols=tuple(records), + ) + ) + + return results + + +def select_scans_by_symbols( + scans: Sequence[LibraryScan], + required_symbols: Sequence[str], +) -> list[LibraryScan]: + """Select binaries by exported canonical names, independent of filenames.""" + + if not required_symbols: + return list(scans) + selected = [] + for scan in scans: + exported = {symbol.canonical_name for symbol in scan.symbols} + if all(name in exported for name in required_symbols): + selected.append(scan) + return selected + + +def filter_scan_symbols( + scans: Sequence[LibraryScan], + prefixes: Sequence[str], +) -> list[LibraryScan]: + if not prefixes: + return list(scans) + return [ + replace( + scan, + symbols=tuple( + symbol + for symbol in scan.symbols + if any(symbol.canonical_name.startswith(prefix) for prefix in prefixes) + ), + ) + for scan in scans + ] + + +def extract_ctypes_bindings(source: str | Path) -> list[BindingDeclaration]: + """Extract literal ctypes decorator candidates without importing the module.""" + + source_path = Path(source) + try: + tree = ast.parse( + source_path.read_text(encoding="utf-8"), + filename=str(source_path), + ) + except (OSError, SyntaxError) as exc: + raise ScanError(f"Failed to parse binding source {source_path}: {exc}") from exc + + declarations: list[BindingDeclaration] = [] + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Call) or not decorator.args: + continue + decorator_name = "" + if isinstance(decorator.func, ast.Name): + decorator_name = decorator.func.id + elif isinstance(decorator.func, ast.Attribute): + decorator_name = decorator.func.attr + if not decorator_name.startswith("ctypes_function"): + continue + + try: + names = ast.literal_eval(decorator.args[0]) + except (ValueError, TypeError): + continue + if isinstance(names, str): + candidates = (names,) + elif isinstance(names, (list, tuple)) and all( + isinstance(name, str) for name in names + ): + candidates = tuple(names) + else: + continue + + required = True + for keyword in decorator.keywords: + if keyword.arg == "required": + try: + required = bool(ast.literal_eval(keyword.value)) + except (ValueError, TypeError): + pass + + declarations.append( + BindingDeclaration( + python_name=node.name, + candidates=candidates, + required=required, + line=node.lineno, + ) + ) + + return sorted(declarations, key=lambda item: item.line) + + +def check_bindings( + scan: LibraryScan, + declarations: Sequence[BindingDeclaration], +) -> dict[str, Any]: + """Check which ctypes candidate would be selected for one library.""" + + exported = {symbol.lookup_name for symbol in scan.symbols} + available = [] + missing_required = [] + missing_optional = [] + + for declaration in declarations: + selected = next( + (name for name in declaration.candidates if name in exported), + None, + ) + item = { + "python_name": declaration.python_name, + "required": declaration.required, + "line": declaration.line, + "candidates": list(declaration.candidates), + "selected": selected, + } + if selected is not None: + available.append(item) + elif declaration.required: + missing_required.append(item) + else: + missing_optional.append(item) + + return { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "declaration_count": len(declarations), + "available_count": len(available), + "available": available, + "missing_required": missing_required, + "missing_optional": missing_optional, + } + + +def compare_scans(scans: Sequence[LibraryScan]) -> dict[str, Any]: + if len(scans) < 2: + raise ValueError("At least two library scans are required for comparison") + + symbol_sets = [{symbol.canonical_name for symbol in scan.symbols} for scan in scans] + common = set.intersection(*symbol_sets) + libraries = [] + + for index, scan in enumerate(scans): + others = set.union(*(symbol_sets[i] for i in range(len(scans)) if i != index)) + libraries.append( + { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "symbol_count": len(symbol_sets[index]), + "only_here": sorted(symbol_sets[index] - others), + "missing_here": sorted(others - symbol_sets[index]), + } + ) + + return { + "common_count": len(common), + "common": sorted(common), + "libraries": libraries, + } + + +def build_manifest( + scans: Sequence[LibraryScan], + *, + generated_at: str | None = None, +) -> dict[str, Any]: + generated_at = generated_at or generation_timestamp() + symbols: dict[str, list[dict[str, Any]]] = defaultdict(list) + libraries = [] + + for scan in scans: + libraries.append(_scan_metadata(scan)) + for symbol in scan.symbols: + symbols[symbol.canonical_name].append( + { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "raw_name": symbol.raw_name, + "lookup_name": symbol.lookup_name, + "abi": symbol.abi, + "address": symbol.address, + "ordinal": symbol.ordinal, + } + ) + + return { + "schema_version": 1, + "generated_at": generated_at, + "libraries": libraries, + "symbols": dict(sorted(symbols.items())), + } + + +def collect_library_paths( + inputs: Sequence[str], + *, + recursive: bool = False, +) -> list[Path]: + def is_shared_library(path: Path) -> bool: + name = path.name.lower() + return path.suffix.lower() in LIBRARY_SUFFIXES or ".so." in name + + paths: list[Path] = [] + for value in inputs: + path = Path(value).expanduser() + if path.is_dir(): + candidates = path.rglob("*") if recursive else path.iterdir() + paths.extend( + candidate + for candidate in candidates + if candidate.is_file() and is_shared_library(candidate) + ) + else: + paths.append(path) + return sorted(set(paths), key=lambda item: str(item).lower()) + + +def _scan_paths( + paths: Sequence[Path], +) -> tuple[list[LibraryScan], list[str]]: + scans: list[LibraryScan] = [] + errors: list[str] = [] + for path in paths: + try: + scans.extend(scan_library(path)) + except ScanError as exc: + errors.append(str(exc)) + except Exception as exc: + detail = str(exc).replace(str(path.resolve()), path.name) + errors.append(f"{path.name}: {detail}") + return scans, errors + + +def _timestamped_output_path(output: str | Path, timestamp: str) -> Path: + path = Path(output) + return path.with_name(f"{path.stem}.{timestamp}{path.suffix}") + + +def _write_output( + text: str, + output: str | None, + *, + timestamp: str, +) -> None: + if output: + output_path = _timestamped_output_path(output, timestamp) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(text + "\n", encoding="utf-8") + print(f"saved: {output_path}") + else: + print(text) + + +def _scan_metadata(scan: LibraryScan) -> dict[str, Any]: + return {key: value for key, value in asdict(scan).items() if key != "symbols"} + + +def _jsonl_rows(scan: LibraryScan, generated_at: str) -> list[str]: + metadata = _scan_metadata(scan) + return [ + json.dumps( + { + "generated_at": generated_at, + **metadata, + **asdict(symbol), + }, + ensure_ascii=False, + ) + for symbol in scan.symbols + ] + + +def write_library_jsonl( + scans: Sequence[LibraryScan], + output_dir: str | Path = "tools/abi/output", + *, + timestamp: str | None = None, +) -> list[Path]: + """Write one same-named JSONL per library under a timestamped run directory.""" + + timestamp = timestamp or generation_timestamp() + destination = Path(output_dir) / timestamp + destination.mkdir(parents=True, exist_ok=True) + grouped: dict[str, list[LibraryScan]] = defaultdict(list) + for scan in scans: + grouped[scan.library].append(scan) + + written = [] + for library, library_scans in sorted(grouped.items()): + output_path = destination / f"{library}.jsonl" + rows = [ + row + for library_scan in library_scans + for row in _jsonl_rows(library_scan, timestamp) + ] + output_path.write_text( + "\n".join(rows) + ("\n" if rows else ""), + encoding="utf-8", + ) + written.append(output_path) + return written + + +def _scan_text(scans: Sequence[LibraryScan], errors: Sequence[str]) -> str: + lines: list[str] = [] + for scan in scans: + lines.append( + f"{scan.library} [{scan.format}/{scan.architecture}]: " + f"{len(scan.symbols)} exported symbol(s)" + ) + for symbol in scan.symbols: + raw_suffix = ( + f" (raw: {symbol.raw_name})" + if symbol.raw_name != symbol.canonical_name + else "" + ) + lines.append( + f" {symbol.canonical_name} [{symbol.abi}]" + f" @ {symbol.address}{raw_suffix}" + ) + for error in errors: + lines.append(f"ERROR: {error}") + return "\n".join(lines) + + +def _compare_text(comparison: dict[str, Any]) -> str: + lines = [f"Common canonical symbols: {comparison['common_count']}"] + for library in comparison["libraries"]: + lines.extend( + [ + "", + ( + f"{library['library']} " + f"[{library['platform']}/{library['architecture']}]: " + f"{library['symbol_count']} symbol(s)" + ), + f" Only here: {len(library['only_here'])}", + ] + ) + lines.extend(f" {name}" for name in library["only_here"]) + lines.append(f" Missing here: {len(library['missing_here'])}") + lines.extend(f" {name}" for name in library["missing_here"]) + return "\n".join(lines) + + +def _bindings_text( + results: Sequence[dict[str, Any]], + scope: str, +) -> str: + lines: list[str] = [] + for result in results: + lines.append( + f"{result['library']} " + f"[{result['platform']}/{result['architecture']}]: " + f"{result['available_count']}/{result['declaration_count']} " + "binding(s) available in selected scope" + ) + if scope in {"required", "all"}: + lines.append(f" Missing required: {len(result['missing_required'])}") + lines.extend( + f" {item['python_name']} (line {item['line']})" + for item in result["missing_required"] + ) + if scope in {"optional", "all"}: + lines.append(f" Missing optional: {len(result['missing_optional'])}") + lines.extend( + f" {item['python_name']} (line {item['line']})" + for item in result["missing_optional"] + ) + return "\n".join(lines) + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Inspect and compare PE, ELF, and Mach-O exported symbols." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + def add_common(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument("paths", nargs="+", help="Library files or directories") + subparser.add_argument( + "--prefix", + action="append", + default=[], + help=( + "Optional canonical-name filter; may be repeated " + "(default: keep all exports)" + ), + ) + subparser.add_argument( + "--select-symbol", + action="append", + default=[], + help=( + "Select libraries exporting this canonical symbol; may be " + "repeated and does not depend on the library filename" + ), + ) + subparser.add_argument( + "--recursive", + action="store_true", + help="Recursively search directory inputs", + ) + subparser.add_argument("-o", "--output", help="Write output to this file") + + scan_parser = subparsers.add_parser("scan", help="List exported symbols") + add_common(scan_parser) + scan_parser.add_argument( + "--format", + choices=("text", "json", "jsonl"), + default="jsonl", + help="Output format", + ) + scan_parser.add_argument( + "--output-dir", + default="tools/abi/output", + help=( + "Directory for default per-library JSONL files " + "(default: tools/abi/output)" + ), + ) + + compare_parser = subparsers.add_parser( + "compare", help="Compare canonical symbol names across libraries" + ) + add_common(compare_parser) + compare_parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format", + ) + + manifest_parser = subparsers.add_parser( + "manifest", help="Create a cross-platform symbol manifest" + ) + add_common(manifest_parser) + + bindings_parser = subparsers.add_parser( + "check-bindings", + help="Check literal ctypes decorator candidates against libraries", + ) + add_common(bindings_parser) + bindings_parser.add_argument( + "--source", + default="llama_cpp/llama_cpp.py", + help="Python binding source to inspect without importing it", + ) + bindings_parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format", + ) + bindings_parser.add_argument( + "--scope", + choices=("optional", "required", "all"), + default="optional", + help=("Binding declarations to check " "(default: optional llama_ext APIs)"), + ) + + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = create_parser().parse_args(argv) + paths = collect_library_paths(args.paths, recursive=args.recursive) + if not paths: + print("No shared libraries found.", file=sys.stderr) + return 2 + + # Scan all exports first. Selection must not depend on --prefix, because a + # caller may use an anchor outside the displayed prefix set. + scans, errors = _scan_paths(paths) + if not scans: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + if args.select_symbol: + scans = select_scans_by_symbols(scans, args.select_symbol) + if not scans: + print( + "No library exports all requested selection symbols: " + + ", ".join(args.select_symbol), + file=sys.stderr, + ) + return 2 + + scans = filter_scan_symbols(scans, args.prefix) + if args.command in {"compare", "manifest"} and args.prefix: + # A package lib directory normally contains ggml and accelerator + # backends. Empty prefix matches are not comparison targets. + scans = [scan for scan in scans if scan.symbols] + timestamp = generation_timestamp() + + validation_failed = False + + if args.command == "scan": + if args.format == "text": + output = _scan_text(scans, errors) + elif args.format == "json": + output = json.dumps( + { + "generated_at": timestamp, + "libraries": [asdict(scan) for scan in scans], + "errors": errors, + }, + ensure_ascii=False, + indent=2, + ) + else: + output = "\n".join( + row for scan in scans for row in _jsonl_rows(scan, timestamp) + ) + + if args.format == "jsonl" and args.output is None: + try: + written = write_library_jsonl( + scans, + args.output_dir, + timestamp=timestamp, + ) + except OSError as exc: + print(f"ERROR: failed to write JSONL output: {exc}", file=sys.stderr) + return 1 + for path in written: + print(f"saved: {path}") + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors else 0 + elif args.command == "compare": + if len(scans) < 2: + print("Comparison requires at least two libraries.", file=sys.stderr) + return 2 + comparison = compare_scans(scans) + output = ( + _compare_text(comparison) + if args.format == "text" + else json.dumps(comparison, ensure_ascii=False, indent=2) + ) + elif args.command == "manifest": + output = json.dumps( + build_manifest(scans, generated_at=timestamp), + ensure_ascii=False, + indent=2, + ) + else: + try: + declarations = extract_ctypes_bindings(args.source) + except ScanError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + if args.scope == "optional": + declarations = [ + declaration for declaration in declarations if not declaration.required + ] + elif args.scope == "required": + declarations = [ + declaration for declaration in declarations if declaration.required + ] + if not declarations: + print( + f"No {args.scope} ctypes binding declarations found in " + f"{args.source}.", + file=sys.stderr, + ) + return 2 + binding_results = [check_bindings(scan, declarations) for scan in scans] + if not args.select_symbol and binding_results: + # A package directory may contain arbitrarily named dependency and + # backend libraries. The library ctypes would want is the one with + # the greatest declaration coverage, regardless of filename. + best_count = max(result["available_count"] for result in binding_results) + binding_results = [ + result + for result in binding_results + if result["available_count"] == best_count + ] + output = ( + _bindings_text(binding_results, args.scope) + if args.format == "text" + else json.dumps(binding_results, ensure_ascii=False, indent=2) + ) + validation_failed = any( + result["missing_required"] or result["missing_optional"] + for result in binding_results + ) + + try: + _write_output(output, args.output, timestamp=timestamp) + except OSError as exc: + print(f"ERROR: failed to write output: {exc}", file=sys.stderr) + return 1 + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors or validation_failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/abi/tests/test_platform_artifacts.py b/tools/abi/tests/test_platform_artifacts.py new file mode 100644 index 0000000000..b5f88dcc3f --- /dev/null +++ b/tools/abi/tests/test_platform_artifacts.py @@ -0,0 +1,85 @@ +"""Opt-in integration tests for real Windows, Linux, and macOS artifacts.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from tools.abi.scan_dynamic import ( + check_bindings, + collect_library_paths, + extract_ctypes_bindings, + scan_library, + select_scans_by_symbols, +) + +ARTIFACTS_ENV = "LLAMA_ABI_ARTIFACTS" +DEFAULT_ARTIFACTS = Path("tools/abi/artifacts") +REQUIRED_PLATFORMS = {"windows", "linux", "darwin"} +STABLE_LLAMA_SYMBOLS = { + "llama_decode", + "llama_model_load_from_file", +} +BINDING_SOURCE = Path("llama_cpp/llama_cpp.py") + + +@pytest.fixture(scope="module") +def platform_scans(): + configured = os.environ.get(ARTIFACTS_ENV) + artifacts = Path(configured) if configured else DEFAULT_ARTIFACTS + paths = collect_library_paths([str(artifacts)], recursive=True) + + if not paths and not configured: + pytest.skip( + "No ABI artifacts installed. Set LLAMA_ABI_ARTIFACTS to run " + "the Windows/Linux/macOS integration test." + ) + + assert paths, f"No shared libraries found under {artifacts}" + scans = [scan for path in paths for scan in scan_library(path)] + scans = select_scans_by_symbols(scans, ["llama_decode"]) + by_platform = {scan.platform: scan for scan in scans} + assert REQUIRED_PLATFORMS <= set(by_platform), ( + "The ABI artifact set must contain llama libraries for Windows, " + f"Linux, and macOS. Found: {sorted(by_platform)}" + ) + return by_platform + + +def test_windows_linux_and_macos_llama_exports(platform_scans): + common = set.intersection( + *( + {symbol.canonical_name for symbol in platform_scans[platform].symbols} + for platform in sorted(REQUIRED_PLATFORMS) + ) + ) + assert STABLE_LLAMA_SYMBOLS <= common + + +def test_macos_macho_lookup_name_removes_symbol_table_prefix(platform_scans): + decode = next( + symbol + for symbol in platform_scans["darwin"].symbols + if symbol.canonical_name == "llama_decode" + ) + assert decode.raw_name == "_llama_decode" + assert decode.lookup_name == "llama_decode" + + +def test_optional_llama_ext_abi_aliases_on_all_platforms(platform_scans): + optional = [ + declaration + for declaration in extract_ctypes_bindings(BINDING_SOURCE) + if not declaration.required + ] + assert optional, "No optional llama_ext ctypes bindings were found" + + for platform in sorted(REQUIRED_PLATFORMS): + result = check_bindings(platform_scans[platform], optional) + assert ( + result["missing_optional"] == [] + ), f"{platform} is missing optional llama_ext ABI aliases: " + ", ".join( + item["python_name"] for item in result["missing_optional"] + ) diff --git a/tools/abi/tests/test_scan_dynamic.py b/tools/abi/tests/test_scan_dynamic.py new file mode 100644 index 0000000000..ebcf75fd47 --- /dev/null +++ b/tools/abi/tests/test_scan_dynamic.py @@ -0,0 +1,194 @@ +import json + +import tools.abi.scan_dynamic as abi_tool + +from tools.abi.scan_dynamic import ( + BindingDeclaration, + SymbolRecord, + LibraryScan, + canonicalize_symbol_name, + check_bindings, + collect_library_paths, + compare_scans, + detect_abi, + extract_ctypes_bindings, + normalize_symbol_name, + select_scans_by_symbols, + write_library_jsonl, +) + + +def _scan(library: str, platform: str, names: list[str]) -> LibraryScan: + records = tuple( + SymbolRecord( + raw_name=name, + lookup_name=name, + canonical_name=name, + abi="unmangled", + address="0x0", + ) + for name in names + ) + return LibraryScan( + library=library, + format="test", + platform=platform, + architecture="test", + sha256="test", + symbols=records, + ) + + +def test_normalizes_macho_external_prefix(): + assert normalize_symbol_name("_llama_decode", "Mach-O") == "llama_decode" + assert normalize_symbol_name("__ZN5llama", "Mach-O") == "_ZN5llama" + assert normalize_symbol_name("llama_decode", "ELF") == "llama_decode" + assert normalize_symbol_name("llama_decode", "PE") == "llama_decode" + + +def test_detects_abi_after_platform_normalization(): + assert detect_abi("?function@@YAXXZ") == "msvc-cxxabi" + assert detect_abi("_ZN5llama") == "itanium-cxxabi" + assert detect_abi("llama_decode") == "unmangled" + + +def test_canonicalizes_simple_global_cpp_names(): + assert ( + canonicalize_symbol_name( + "?llama_graph_reserve@@YAXXZ", + "msvc-cxxabi", + ) + == "llama_graph_reserve" + ) + assert ( + canonicalize_symbol_name( + "_Z19llama_graph_reserveP13llama_contextjjj", + "itanium-cxxabi", + ) + == "llama_graph_reserve" + ) + nested = "_ZN5llama6detail3fooEv" + assert canonicalize_symbol_name(nested, "itanium-cxxabi") == nested + + +def test_compares_canonical_names(): + comparison = compare_scans( + [ + _scan("libllama.so", "linux", ["llama_decode"]), + _scan( + "llama.dll", + "windows", + ["llama_decode", "llama_windows_only"], + ), + ] + ) + + assert comparison["common"] == ["llama_decode"] + assert comparison["libraries"][0]["missing_here"] == ["llama_windows_only"] + assert comparison["libraries"][1]["only_here"] == ["llama_windows_only"] + + +def test_collects_versioned_elf_library(tmp_path): + library = tmp_path / "libllama.so.1" + library.touch() + + assert collect_library_paths([str(tmp_path)]) == [library] + + +def test_extracts_and_checks_literal_binding_aliases(tmp_path): + source = tmp_path / "bindings.py" + source.write_text( + """ +@ctypes_function( + ["llama_ext", "?llama_ext@@YAXXZ", "_Z9llama_extv"], + [], + None, + required=False, +) +def llama_ext(): + pass +""", + encoding="utf-8", + ) + declarations = extract_ctypes_bindings(source) + scan = _scan("libllama.so", "linux", ["_Z9llama_extv"]) + result = check_bindings(scan, declarations) + + assert declarations == [ + BindingDeclaration( + python_name="llama_ext", + candidates=( + "llama_ext", + "?llama_ext@@YAXXZ", + "_Z9llama_extv", + ), + required=False, + line=8, + ) + ] + assert result["available"][0]["selected"] == "_Z9llama_extv" + assert result["missing_optional"] == [] + + +def test_selects_library_by_symbol_not_filename(): + scans = [ + _scan("custom-backend-name.dll", "windows", ["ggml_backend_init"]), + _scan("renamed-native-output.bin", "windows", ["llama_decode"]), + ] + + selected = select_scans_by_symbols(scans, ["llama_decode"]) + + assert [scan.library for scan in selected] == ["renamed-native-output.bin"] + + +def test_writes_jsonl_named_after_dynamic_library(tmp_path): + output_dir = tmp_path / "output" + scans = [ + _scan("libllama.so", "linux", ["llama_decode"]), + _scan("llama.dll", "windows", ["llama_decode"]), + ] + + timestamp = "20260728T120000.123456Z" + written = write_library_jsonl( + scans, + output_dir, + timestamp=timestamp, + ) + + assert [path.name for path in written] == [ + "libllama.so.jsonl", + "llama.dll.jsonl", + ] + run_dir = output_dir / timestamp + row = json.loads((run_dir / "llama.dll.jsonl").read_text("utf-8")) + assert row["library"] == "llama.dll" + assert row["canonical_name"] == "llama_decode" + assert row["generated_at"] == timestamp + assert "path" not in row + + +def test_check_bindings_cli_fails_when_optional_api_is_missing( + tmp_path, + monkeypatch, +): + library = tmp_path / "renamed.dll" + library.touch() + source = tmp_path / "bindings.py" + source.write_text( + """ +@ctypes_function(["llama_ext", "_Z9llama_extv"], [], None, required=False) +def llama_ext(): + pass +""", + encoding="utf-8", + ) + scan = _scan("renamed.dll", "windows", ["llama_decode"]) + monkeypatch.setattr( + abi_tool, + "_scan_paths", + lambda paths: ([scan], []), + ) + + exit_code = abi_tool.main(["check-bindings", str(library), "--source", str(source)]) + + assert exit_code == 1 From 7708b3de2596fb3262df565ce46c14a4db9ab254 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 28 Jul 2026 23:55:47 +0800 Subject: [PATCH 278/304] Update Submodule vendor/llama.cpp b77d646..7e1e28c Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 52 ++++++++++++++++++++++++++---------------- vendor/llama.cpp | 2 +- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 317a435489..2901c6e2e0 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -504,6 +504,30 @@ class llama_split_mode(enum.IntEnum): LLAMA_SPLIT_MODE_ROW = 2 LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_load_mode { +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# }; +class llama_load_mode(enum.IntEnum): + LLAMA_LOAD_MODE_NONE = 0 # no special loading mode + LLAMA_LOAD_MODE_MMAP = 1 # memory map the model + LLAMA_LOAD_MODE_MLOCK = 2 # force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_MMAP_MLOCK = 3 # mmap + force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_DIRECT_IO = 4 # use direct I/O if available + +# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); +@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) +def llama_load_mode_name(load_mode: int) -> bytes: + ... + +# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); +@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int) +def llama_load_mode_from_str(str: ctypes.c_char_p) -> int: + ... + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -743,17 +767,15 @@ class llama_model_tensor_buft_override(ctypes.Structure): # struct llama_model_params { # // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) # ggml_backend_dev_t * devices; -# + # // NULL-terminated list of buffer types to use for tensors that match a pattern # const struct llama_model_tensor_buft_override * tensor_buft_overrides; -# + # int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers # enum llama_split_mode split_mode; // how to split the model across multiple GPUs +# enum llama_load_mode load_mode; // how to load the model -# // main_gpu interpretation depends on split_mode: -# // LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model -# // LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results -# // LLAMA_SPLIT_MODE_LAYER: ignored +# // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() @@ -770,12 +792,8 @@ class llama_model_tensor_buft_override(ctypes.Structure): # // override key-value pairs of the model meta data # const struct llama_model_kv_override * kv_overrides; - # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported -# bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used @@ -789,15 +807,13 @@ class llama_model_params(ctypes.Structure): tensor_buft_overrides(llama_model_tensor_buft_override): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs + load_mode (int): how to load the model main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights - use_mmap (bool): use mmap if possible - use_direct_io(bool): use direct io, takes precedence over use_mmap when supported - use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used @@ -808,34 +824,30 @@ class llama_model_params(ctypes.Structure): tensor_buft_overrides: CtypesPointer[llama_model_tensor_buft_override] n_gpu_layers: int split_mode: int + load_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool - use_mmap: bool - use_direct_io: bool - use_mlock: bool check_tensors: bool use_extra_bufts: bool no_host: bool no_alloc: bool _fields_ = [ - ("devices", ctypes.c_void_p), # NOTE: unnused + ("devices", ctypes.POINTER(ctypes.c_void_p)), # NOTE: unnused ("tensor_buft_overrides", ctypes.POINTER(llama_model_tensor_buft_override)), ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), + ("load_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), - ("use_mmap", ctypes.c_bool), - ("use_direct_io", ctypes.c_bool), - ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b77d646751..7e1e28cae3 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b77d646751d01c0962bc203b6809e9d94f7d50b7 +Subproject commit 7e1e28cae36d41fe7bbe9dae7c9625de6565c063 From 00591a5b6a635682914e410224e3736d21ecb088 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 29 Jul 2026 00:03:14 +0800 Subject: [PATCH 279/304] feat(llama): support llama_model_params load_mode - Update model loading configuration to use the new `load_mode` field from llama_model_params and align with the latest llama.cpp API changes. - Remove deprecated internal handling of legacy loading flags and keep backward compatibility by warning users when `use_mmap`, `use_direct_io`, or `use_mlock` are still used. - This prepares the Python bindings for the updated llama.cpp model loading interface while providing a smoother migration path for existing users. Signed-off-by: JamePeng --- examples/low_level_api/common.py | 3 --- .../low_level_api/low_level_api_chat_cpp.py | 3 --- llama_cpp/llama.py | 22 +++++++++++-------- llama_cpp/server/model.py | 4 +--- llama_cpp/server/settings.py | 18 +++++---------- tests/test_llama.py | 3 --- 6 files changed, 19 insertions(+), 34 deletions(-) diff --git a/examples/low_level_api/common.py b/examples/low_level_api/common.py index 8adb2923cc..601f5cebdf 100644 --- a/examples/low_level_api/common.py +++ b/examples/low_level_api/common.py @@ -60,9 +60,6 @@ class GptParams: instruct: bool = False perplexity: bool = False - use_mmap: bool = True - use_direct_io: bool = False - use_mlock: bool = False mem_test: bool = False verbose_prompt: bool = False diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 1f4f5b3e79..96c4121f4f 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -76,9 +76,6 @@ def __init__(self, params: GptParams) -> None: self.lparams.n_parts = self.params.n_parts self.lparams.seed = self.params.seed self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap - self.lparams.use_direct_io = self.params.use_direct_io self.model = llama_cpp.llama_load_model_from_file( self.params.model.encode("utf8"), self.lparams diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index f733d7afb9..3ce635b547 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -104,10 +104,11 @@ def __init__( cpu_moe: bool = False, n_cpu_moe: int = 0, split_mode: int = llama_cpp_lib.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, + load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_MMAP, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, vocab_only: bool = False, - use_mmap: bool = True, + use_mmap: bool = False, use_direct_io: bool = False, use_mlock: bool = False, check_tensors: bool = False, @@ -215,11 +216,10 @@ def __init__( n_cpu_moe: Keep the MoE expert weights of the first N layers on CPU. Useful when VRAM is insufficient for MoE models. split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. + load_mode: How to load the model. See llama_cpp.LLAMA_LOAD_MODE_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. vocab_only: Only load the vocabulary no weights. - use_mmap: Use mmap if possible. - use_mlock: Force the system to keep the model in RAM. check_tensors: validate model tensor data use_extra_bufts: use extra buffer types (used for weight repacking) no_host: bypass host buffer allowing extra buffers to be used @@ -352,10 +352,19 @@ def __init__( self.model_path = model_path + if (use_mmap or use_direct_io or use_mlock) and verbose: + print( + "Llama.__init__: WARNING: " + "Legacy load options (`use_mmap`, `use_direct_io`, `use_mlock`) " + "are deprecated. Use `load_mode` instead.", + file=sys.stderr, + ) + # Model Params self.model_params = llama_cpp_lib.llama_model_default_params() self.model_params.n_gpu_layers = self._parse_n_gpu_layers(n_gpu_layers) self.model_params.split_mode = split_mode + self.model_params.load_mode = load_mode self.model_params.main_gpu = main_gpu self.tensor_split = tensor_split self._c_tensor_split = None @@ -371,9 +380,6 @@ def __init__( ) # keep a reference to the array so it is not gc'd self.model_params.tensor_split = self._c_tensor_split self.model_params.vocab_only = vocab_only - self.model_params.use_mmap = use_mmap - self.model_params.use_direct_io = use_direct_io - self.model_params.use_mlock = use_mlock self.model_params.check_tensors = check_tensors self.model_params.use_extra_bufts = use_extra_bufts self.model_params.no_host = no_host @@ -3445,12 +3451,10 @@ def __getstate__(self): cpu_moe=self.cpu_moe, n_cpu_moe=self.n_cpu_moe, split_mode=self.model_params.split_mode, + load_mode=self.model_params.load_mode, main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, vocab_only=self.model_params.vocab_only, - use_mmap=self.model_params.use_mmap, - use_direct_io=self.model_params.use_direct_io, - use_mlock=self.model_params.use_mlock, check_tensors=self.model_params.check_tensors, use_extra_bufts=self.model_params.use_extra_bufts, no_host=self.model_params.no_host, diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 6b3fd1dd15..0d509bbcf9 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -294,12 +294,10 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: # Model Params n_gpu_layers=settings.n_gpu_layers, split_mode=settings.split_mode, + load_mode=settings.load_mode, main_gpu=settings.main_gpu, tensor_split=settings.tensor_split, vocab_only=settings.vocab_only, - use_mmap=settings.use_mmap, - use_direct_io=settings.use_direct_io, - use_mlock=settings.use_mlock, check_tensors=settings.check_tensors, use_extra_bufts=settings.use_extra_bufts, no_host=settings.no_host, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 350ccc2323..62ce3b5044 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -32,8 +32,12 @@ class ModelSettings(BaseSettings): ) split_mode: int = Field( default=llama_cpp.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, - description="The split mode to use.", + description="how to split the model across multiple GPUs", ) + load_mode: int = Field( + default=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP, + description="how to load the model", + ) main_gpu: int = Field( default=0, ge=0, @@ -46,18 +50,6 @@ class ModelSettings(BaseSettings): vocab_only: bool = Field( default=False, description="Whether to only return the vocabulary." ) - use_mmap: bool = Field( - default=True, - description="Enable mmap to use filesystem cache.", - ) - use_direct_io: bool = Field( - default=False, - description="Use direct io, takes precedence over use_mmap.", - ) - use_mlock: bool = Field( - default=False, - description="Use mlock for force system to keep model in RAM", - ) check_tensors: bool = Field( default=False, description="Validate model tensor data.", diff --git a/tests/test_llama.py b/tests/test_llama.py index b233ea5266..d0053feaba 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -260,9 +260,6 @@ def test_real_model(llama_cpp_model_path): # 1. Setup Model Parameters params = llama_cpp.llama_model_default_params() - params.use_mmap = llama_cpp.llama_supports_mmap() - params.use_direct_io = False - params.use_mlock = llama_cpp.llama_supports_mlock() params.check_tensors = False # 2. Load the Model From bab30611b4035bd69765d4856f907c763a6a69fb Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 29 Jul 2026 00:11:27 +0800 Subject: [PATCH 280/304] docs: document `load_mode` migration - Replace references to the legacy model loading flags with load_mode, document all supported loading modes for the Python API and server, and update the performance tuning example. Signed-off-by: JamePeng --- docs/server.md | 20 +++++++++++ docs/wiki/core/Llama.md | 40 ++++++++++++++++++++-- examples/notebooks/PerformanceTuning.ipynb | 12 +++++-- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/docs/server.md b/docs/server.md index cd6f86c513..3bdc0c7e69 100644 --- a/docs/server.md +++ b/docs/server.md @@ -37,6 +37,26 @@ CLI arguments and environment variables are available for all of the fields defi Additionally the server supports configuration check out the [configuration section](#configuration-and-multi-model-support) for more information and examples. +#### Model loading mode + +Use `load_mode` to select how the server loads model data. The corresponding +CLI option is `--load_mode`, the environment variable is `LOAD_MODE`, and a +multi-model JSON configuration can set `"load_mode"` for each model. +`use_mmap`, `use_direct_io`, and `use_mlock` are no longer server settings. + +| Value | Mode | Description | +|---:|---|---| +| `0` | `LLAMA_LOAD_MODE_NONE` | Use no special model-loading mode. | +| `1` | `LLAMA_LOAD_MODE_MMAP` | Memory-map the model. This is the default. | +| `2` | `LLAMA_LOAD_MODE_MLOCK` | Keep the loaded model in RAM rather than allowing it to be swapped or compressed. | +| `3` | `LLAMA_LOAD_MODE_MMAP_MLOCK` | Memory-map the model and keep its mapped pages in RAM. | +| `4` | `LLAMA_LOAD_MODE_DIRECT_IO` | Use direct I/O when it is available. | + +For example, start the server with memory mapping plus memory locking: + +```bash +python3 -m llama_cpp.server --model --load_mode 3 +``` ## Guides diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 305624c8fc..00add6ea4c 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -3,7 +3,7 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-07-26 +last_updated: 2026-07-29 version_target: "latest" --- @@ -36,13 +36,47 @@ Initialize the model and context. Note that model loading will immediately alloc | `cpu_moe` | `bool` | `False` | Whether to keep all MoE weights on CPU | | `n_cpu_moe` | `int` | `0` | Number of first N MoE layers to keep on CPU (compatible with `cpu_moe`) | | `split_mode` | `int` | `LLAMA_SPLIT_MODE_LAYER` | Model GPU split mode:
ā€¢ `LLAMA_SPLIT_MODE_NONE`: single GPU
ā€¢ `LLAMA_SPLIT_MODE_ROW`: row-level split
ā€¢ `LLAMA_SPLIT_MODE_LAYER`: layer-level split | +| `load_mode` | `int` (`llama_load_mode`) | `LLAMA_LOAD_MODE_MMAP` | How model data is loaded. Select one of the `LLAMA_LOAD_MODE_*` values described below. | | `main_gpu` | `int` | `0` | The primary GPU to use for intermediate results or the entire model. | | `tensor_split` | `List[float]` | `None` | Proportional split of tensors across GPUs (max `LLAMA_MAX_DEVICES`). | -| `use_mmap` | `bool` | `True` | Whether to use memory mapping (mmap) if possible. | -| `use_mlock` | `bool` | `False` | Force the system to keep the model in RAM, preventing swapping. | | `kv_overrides` | `Dict` | `None` | Key-value overrides for the model metadata (supports bool, int, float, str). | | `numa` | `Union[bool, int]` | `False` | NUMA strategy (e.g., `GGML_NUMA_STRATEGY_DISTRIBUTE`). | +#### Model Load Modes + +`load_mode` replaces the legacy `use_mmap`, `use_direct_io`, and `use_mlock` +arguments. It accepts a member of `llama_cpp.llama_load_mode`: + +| Value | Integer | Description | +| :--- | :---: | :--- | +| `LLAMA_LOAD_MODE_NONE` | `0` | Use no special model-loading mode. | +| `LLAMA_LOAD_MODE_MMAP` | `1` | Memory-map the model. This is the default. | +| `LLAMA_LOAD_MODE_MLOCK` | `2` | Keep the loaded model in RAM rather than allowing it to be swapped or compressed. | +| `LLAMA_LOAD_MODE_MMAP_MLOCK` | `3` | Memory-map the model and keep its mapped pages in RAM. | +| `LLAMA_LOAD_MODE_DIRECT_IO` | `4` | Use direct I/O when it is available. | + +```python +import llama_cpp + +llm = llama_cpp.Llama( + model_path="models/model.gguf", + load_mode=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK, +) +``` + +The legacy loading arguments are retained only for call compatibility. They no +longer configure the underlying model parameters and may emit a deprecation +warning; set `load_mode` explicitly instead. Use the following migration +mapping: + +| Legacy configuration | Replacement | +| :--- | :--- | +| `use_mmap=False, use_mlock=False` | `load_mode=LLAMA_LOAD_MODE_NONE` | +| `use_mmap=True, use_mlock=False` | `load_mode=LLAMA_LOAD_MODE_MMAP` | +| `use_mmap=False, use_mlock=True` | `load_mode=LLAMA_LOAD_MODE_MLOCK` | +| `use_mmap=True, use_mlock=True` | `load_mode=LLAMA_LOAD_MODE_MMAP_MLOCK` | +| `use_direct_io=True` | `load_mode=LLAMA_LOAD_MODE_DIRECT_IO` | + ### Context & Batch Parameters | Parameter | Type | Default | Description | diff --git a/examples/notebooks/PerformanceTuning.ipynb b/examples/notebooks/PerformanceTuning.ipynb index ba74e4a41f..43772a5b7e 100644 --- a/examples/notebooks/PerformanceTuning.ipynb +++ b/examples/notebooks/PerformanceTuning.ipynb @@ -24,7 +24,13 @@ "# Hyperparameters\n", "space = [\n", " Categorical([True, False], name=\"f16_kv\"),\n", - " Categorical([True, False], name=\"use_mlock\"),\n", + " Categorical(\n", + " [\n", + " llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP,\n", + " llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK,\n", + " ],\n", + " name=\"load_mode\",\n", + " ),\n", " Integer(1, multiprocessing.cpu_count(), name=\"n_threads\"),\n", " Integer(1, 2048, name=\"n_batch\"),\n", "]\n", @@ -46,13 +52,13 @@ "@use_named_args(space)\n", "def objective(**params):\n", " f16_kv = params[\"f16_kv\"]\n", - " use_mlock = params[\"use_mlock\"]\n", + " load_mode = params[\"load_mode\"]\n", " n_threads = params[\"n_threads\"]\n", " n_batch = params[\"n_batch\"]\n", " llm = llama_cpp.Llama(\n", " model_path=MODEL_PATH,\n", " f16_kv=f16_kv,\n", - " use_mlock=use_mlock,\n", + " load_mode=load_mode,\n", " n_threads=n_threads,\n", " n_batch=n_batch,\n", " )\n", From f8bc6b05f8c384e675c6378e17e3c548c4f465e5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 30 Jul 2026 21:18:43 +0800 Subject: [PATCH 281/304] Update Submodule vendor/llama.cpp 7e1e28c..e1a1abb Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 20 ++++++++++++++++++++ vendor/llama.cpp | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2901c6e2e0..a956dda475 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -3544,6 +3544,26 @@ def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ... +# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) +# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens); +@ctypes_function( + "llama_vocab_get_suppress_tokens", + [ + llama_vocab_p_ctypes, + ctypes.POINTER(ctypes.c_int32), + ], + llama_token_p, +) +def llama_vocab_get_suppress_tokens( + vocab: llama_vocab_p, + n_suppress_tokens: ctypes.POINTER(ctypes.c_int32), # type: ignore +) -> llama_token_p: # type: ignore + """ + model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) + """ + ... + + # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_fim_pre", diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7e1e28cae3..e1a1abb787 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7e1e28cae36d41fe7bbe9dae7c9625de6565c063 +Subproject commit e1a1abb78746c025f5e9039f590e37ccdb758ae7 From 9ac3f545ac99048526a1dfcf6d3f30f1bc10df82 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 1 Aug 2026 00:14:19 +0800 Subject: [PATCH 282/304] Update Submodule vendor/llama.cpp e1a1abb..876a432 Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 6 +++++- vendor/llama.cpp | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a956dda475..0c26637092 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -798,6 +798,7 @@ class llama_model_tensor_buft_override(ctypes.Structure): # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used # bool no_alloc; // only load metadata and simulate memory allocations +# bool load_mtp; // whether to load MTP layers # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -817,7 +818,8 @@ class llama_model_params(ctypes.Structure): check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used - no_alloc (bool): only load metadata and simulate memory allocations""" + no_alloc (bool): only load metadata and simulate memory allocations + load_mtp (bool): whether to load MTP layers""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused @@ -835,6 +837,7 @@ class llama_model_params(ctypes.Structure): use_extra_bufts: bool no_host: bool no_alloc: bool + load_mtp: bool _fields_ = [ ("devices", ctypes.POINTER(ctypes.c_void_p)), # NOTE: unnused @@ -852,6 +855,7 @@ class llama_model_params(ctypes.Structure): ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), ("no_alloc", ctypes.c_bool), + ("load_mtp", ctypes.c_bool), ] llama_model_params_p = ctypes.POINTER(llama_model_params) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e1a1abb787..876a432116 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e1a1abb78746c025f5e9039f590e37ccdb758ae7 +Subproject commit 876a4321163249c43ca4e986818fab5ab081f282 From 6f60d0347bba9e2f987f36af51a9bf9865ae824b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 1 Aug 2026 00:18:03 +0800 Subject: [PATCH 283/304] feat(llama): expose additional model loading options - add `no_alloc` and `load_mtp` parameters - enable `extra buffer types` by default Signed-off-by: JamePeng --- llama_cpp/llama.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 3ce635b547..8092f86956 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -107,14 +107,16 @@ def __init__( load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_MMAP, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, - vocab_only: bool = False, + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, use_mmap: bool = False, use_direct_io: bool = False, use_mlock: bool = False, + vocab_only: bool = False, check_tensors: bool = False, - use_extra_bufts: bool = False, + use_extra_bufts: bool = True, no_host: bool = False, - kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, + no_alloc: bool = False, + load_mtp: bool = False, # Context Params seed: int = llama_cpp_lib.LLAMA_DEFAULT_SEED, n_ctx: int = 512, @@ -219,11 +221,13 @@ def __init__( load_mode: How to load the model. See llama_cpp.LLAMA_LOAD_MODE_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. + kv_overrides: Key-value overrides for the model. vocab_only: Only load the vocabulary no weights. check_tensors: validate model tensor data use_extra_bufts: use extra buffer types (used for weight repacking) no_host: bypass host buffer allowing extra buffers to be used - kv_overrides: Key-value overrides for the model. + no_alloc: only load metadata and simulate memory allocations + load_mtp: whether to load MTP layers seed: RNG seed, -1 for random n_ctx: Text context, 0 = from model n_keep: Number of tokens to keep from initial prompt @@ -383,6 +387,8 @@ def __init__( self.model_params.check_tensors = check_tensors self.model_params.use_extra_bufts = use_extra_bufts self.model_params.no_host = no_host + self.model_params.no_alloc = no_alloc + self.model_params.load_mtp = load_mtp # Logic of cpu_moe, n_cpu_moe # Reference from llama.cpp/tools/llama-bench/llama-bench.cpp @@ -3454,11 +3460,13 @@ def __getstate__(self): load_mode=self.model_params.load_mode, main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, + kv_overrides=self.kv_overrides, vocab_only=self.model_params.vocab_only, check_tensors=self.model_params.check_tensors, use_extra_bufts=self.model_params.use_extra_bufts, no_host=self.model_params.no_host, - kv_overrides=self.kv_overrides, + no_alloc=self.model_params.no_alloc, + load_mtp=self.model_params.load_mtp, # Context Params seed=self._seed, n_ctx=self.context_params.n_ctx, From aafc6fb74ebfba6a044510f80b5e9ad277109c12 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 1 Aug 2026 00:59:46 +0800 Subject: [PATCH 284/304] fix(ctypes): correct llama-ext binding signatures - use uint32_t for layer IDs - fix void return type for embedding extraction control - return target layer count as uint32_t Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 0c26637092..cc6a1a67fb 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -5318,15 +5318,15 @@ def llama_get_embeddings_nextn_ith( "__Z30llama_set_embeddings_layer_inpP13llama_contextjb", "_Z30llama_set_embeddings_layer_inpP13llama_contextjb", ], - [llama_context_p_ctypes, ctypes.c_int32, ctypes.c_bool], - ctypes.POINTER(ctypes.c_float), + [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_bool], + None, required=False, ) def llama_set_embeddings_layer_inp( ctx: llama_context_p, - lid: ctypes.c_int32, + lid: ctypes.c_uint32, value: bool, -) -> ctypes.POINTER(ctypes.c_float): # type: ignore +) -> None: # type: ignore """ Set whether the context outputs the input embeddings of a specific layer """ @@ -5342,13 +5342,13 @@ def llama_set_embeddings_layer_inp( "__Z30llama_get_embeddings_layer_inpP13llama_contextj", "_Z30llama_get_embeddings_layer_inpP13llama_contextj", ], - [llama_context_p_ctypes, ctypes.c_int32], + [llama_context_p_ctypes, ctypes.c_uint32], ctypes.POINTER(ctypes.c_float), required=False, ) def llama_get_embeddings_layer_inp( ctx: llama_context_p, - lid: ctypes.c_int32, + lid: ctypes.c_uint32, ) -> ctypes.POINTER(ctypes.c_float): # type: ignore ... @@ -5402,12 +5402,12 @@ def llama_model_target_layer_ids( "_Z30llama_model_target_layer_ids_nPK11llama_model", ], [llama_model_p_ctypes], - ctypes.POINTER(ctypes.c_uint32), + ctypes.c_uint32, required=False, ) def llama_model_target_layer_ids_n( model: llama_model_p -) -> ctypes.POINTER(ctypes.c_uint32): # type: ignore +) -> int: """ returns the number of extracted layers from target model """ From d9d27a7bdf1c27d50c1490ad6acd825f31804902 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 1 Aug 2026 04:17:19 +0800 Subject: [PATCH 285/304] Bump version to 0.3.45 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - This release focuses on reactivating and modernizing Llamaā€™s built-in embedding capabilities, aligning the Python bindings with the latest llama.cpp APIs, and improving reliability across platforms. Signed-off-by: JamePeng --- CHANGELOG.md | 152 ++++++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fc02b25c..075b978847 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,158 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.45] Reactivated Built-in Embeddings, Modern Model Loading, and Stronger Cross-Platform Reliability + +- fix(ctypes): correct llama-ext binding signatures + - use uint32_t for layer IDs + - fix void return type for embedding extraction control + - return target layer count as uint32_t + +- feat(llama): expose additional model loading options + - add `no_alloc` and `load_mtp` parameters + - enable `extra buffer types` by default + +- feat(llama): support llama_model_params `load_mode` + - Update model loading configuration to use the new `load_mode` field from + llama_model_params and align with the latest llama.cpp API changes. + - Remove deprecated internal handling of legacy loading flags and keep + backward compatibility by warning users when `use_mmap`, `use_direct_io`, + or `use_mlock` are still used. + - This prepares the Python bindings for the updated llama.cpp model loading + interface while providing a smoother migration path for existing users. + - docs: document `load_mode` migration + - Replace references to the legacy model loading flags with load_mode, document all supported loading modes for the Python API and server, and update the performance tuning example. + +- feat(tools): add cross-platform ABI inspection utility + - Inspect `PE`, `ELF`, and `Mach-O` exports and normalize platform-specific symbol names. + - Validate optional `llama_ext` ctypes aliases across Windows, Linux, and macOS builds. Keep artifacts and timestamped privacy-safe reports local to the repository. + - More information see here: [Cross-platform ABI inspection](https://github.com/JamePeng/llama-cpp-python/tree/main/tools/abi) + +- fix(ctypes): support GCC/Clang mangled symbols for optional llama_ext APIs + - Add missing `_Z` Itanium C++ ABI symbol variants to ctypes function + lookup lists. This improves compatibility with Linux and macOS builds + where C++ symbols are exported using GCC/Clang name mangling. + - Issue report from **@ckcfcc** (https://github.com/JamePeng/llama-cpp-python/issues/159) + +- fix(loader): guard `HIP_PATH` and `VULKAN_SDK` dirs with os.path.exists +os.add_dll_directory() raises FileNotFoundError [WinError 3] when the +directory does not exist, so a stale `HIP_PATH` or `VULKAN_SDK` left behind by +an uninstalled SDK makes "import llama_cpp" fail outright on Windows.(by **@emptyngton**) + + The CUDA_PATH branch above already guards each candidate directory with + os.path.exists(); this applies the same pattern to the HIP and Vulkan + branches. Valid directories are still added individually, so a partially + removed SDK contributes whichever of bin/lib remain instead of raising. + +- fix(_internals): clean up native resources on initialization failures + - Register native model and batch ownership immediately after allocation + so later validation failures cannot leak llama.cpp resources. + Free a loaded model when vocab lookup fails, and route mixed-batch setup + failures through idempotent cleanup. + - Initialize sampling-context resource fields before fallible setup and + make partial teardown safe to repeat. This prevents missing attributes + from interrupting cleanup when sampler-chain construction fails. + - Clear model, vocabulary, and sampling parameter references after native + context and sampler resources have been released. This prevents closed + wrapper objects from unnecessarily keeping models and related Python + objects alive. + - Add failure-injection tests that verify model and batch handles are freed + exactly once and partially initialized sampling contexts release their resources + idempotently.Extend lifecycle tests to verify that parent references are cleared + and that repeated close calls remain safe. + +- test(chat-format): modernize coverage with Qwen3.5-style templates + - Replace the legacy Mistral-focused chat format tests with self-contained + Qwen3.5-style Jinja template coverage: + - verify ChatML system, user, and assistant message rendering + - cover enabled and disabled thinking generation prompts + - test image and video placeholders with vision identifiers + - validate tool definitions, tool calls, and tool response history + - add clear error coverage for invalid message structures + - verify model-specific stop token criteria + - keep the tests independent of tokenizer files and model weights + +- docs(readme): replace the new logo with fork project branding + - Add the new llama-cpp-python logo asset under docs and update the README + header to reference the repository-local image. + - the new logo which combined llama, C++, and Project branding remains readable. + +- docs(embedding): add end-to-end embeddings and reranking guide + - Create a schema-compliant feature guide covering sentence embeddings, + token-level vectors, reranking workflows, pooling modes, normalization, + streaming batch configuration, return shapes, and output formats. + - Add complete examples for the standard Llama API, LlamaEmbedding, + pre-tokenized inputs, cosine-similarity output, and cross-encoder + reranking. + - Document common configuration problems, implementation limitations, and + the embedding and reranking model families currently listed as supported + by the project. + - Expose the new feature guide through the Wiki index. + +- docs(llama): expand embedding parameters and API guidance + - Add a role overview and reorganize constructor options into focused, + readable parameter groups. + - Document embedding, pooling, attention, KV cache, sequence capacity, and + recurrent-state settings with their defaults and runtime behavior. + - Expand the embed() and create_embedding() sections with normalization + modes, return shapes, batching semantics, pooling recommendations, + OpenAI compatibility notes, and resource-safe examples. + - Fix the YAML frontmatter and improve Markdown spacing for cleaner Wiki + rendering. + +- docs(embedding): document maintained APIs and sequence batch capacity + - Replace the deprecated Llama embedding guidance with current embed() and + create_embedding() usage. + - Document the roles of n_batch, n_ubatch, and n_seq_max, including + parallel batching examples, resource considerations, common sequence ID + errors, and the required configuration changes. + - Clarify that LlamaEmbedding remains a convenience interface for + embedding-oriented defaults and reranking workflows. + +- docs(example): refresh the built-in embedding usage example + - Fix the Llama constructor option from embedding=True to embeddings=True + and demonstrate L2-normalized output through create_embedding(). + +- test(embedding): cover built-in and streaming embedding workflows + - Add coverage for actionable LlamaBatch sequence-capacity errors and the + maintained embedding APIs on the standard Llama class. + - Verify pre-tokenized batches, normalization, separator-based inputs, + token accounting, OpenAI-compatible responses, and LlamaEmbedding + streaming behavior with n_seq_max=1. + - Explicitly close embedding models after integration tests to release + native context and model resources. + +- fix(embedding): respect n_seq_max when streaming embedding batches + - Use the configured sequence capacity instead of n_ubatch when deciding + when to decode the current LlamaEmbedding batch. + - This prevents invalid sequence IDs for multi-document inputs and allows + the default n_seq_max=1 configuration to process documents sequentially + without failing. + +- refactor(batch): improve sequence capacity validation guidance + - Make LlamaBatch sequence validation errors explain the configured + n_seq_max value, valid sequence ID range, and minimum capacity required + for parallel batching. + - Handle negative sequence IDs separately and provide actionable setup + guidance for Llama, LlamaEmbedding, and direct LlamaBatch users. + - Remove the unused normalize_embedding helper now that normalization is + handled by the embedding pipeline. + +- feat(embedding): modernize the built-in Llama embedding API + - Replace the legacy embedding path with sequence-aware streaming batch + processing based on the current LlamaBatch interface. + - Support string, batched string, and pre-tokenized inputs, token-level and + rank pooling outputs, separator-based splitting, token accounting, and + llama.cpp-compatible normalization modes. + - Restore embed() and create_embedding() as maintained Llama APIs while + preserving the existing boolean normalization behavior. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/876a4321163249c43ca4e986818fab5ab081f282](https://github.com/ggml-org/llama.cpp/commit/876a4321163249c43ca4e986818fab5ab081f282) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260801 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/ebf6099b81cf67cfb5eec569466367c9fa04e9d4...aafc6fb74ebfba6a044510f80b5e9ad277109c12 + ## [0.3.44] Improved Windows DLL(OpenMP) Loading Reliability for GGML Backends - fix(ggml): preload bundled OpenMP runtime before loading ggml-base diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 10e452d5f6..b359355f9e 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.44" +__version__ = "0.3.45" From 07c04257e3116b3575199748abb4d66f16168ae3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 2 Aug 2026 22:57:19 +0800 Subject: [PATCH 286/304] Update Submodule vendor/llama.cpp 876a432..221f0f6 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 876a432116..221f0f6356 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 876a4321163249c43ca4e986818fab5ab081f282 +Subproject commit 221f0f6356efe2260023208365705ec5d5a7c8f5 From 88fce160be1e6f21834c1b2ddcee6c4b72ccfed8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 3 Aug 2026 21:19:44 +0800 Subject: [PATCH 287/304] Update Submodule vendor/llama.cpp 221f0f6..563dec8 Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 9 ++++++--- vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index cc6a1a67fb..c3b38005d8 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -4130,6 +4130,7 @@ def llama_chat_builtin_templates( # struct ggml_tensor * probs; # struct ggml_tensor * sampled; # struct ggml_tensor * candidates; +# int64_t n_vocab; # }; class llama_sampler_data(ctypes.Structure): if TYPE_CHECKING: @@ -4137,12 +4138,14 @@ class llama_sampler_data(ctypes.Structure): probs: ctypes.c_void_p sampled: ctypes.c_void_p candidates: ctypes.c_void_p + n_vocab: ctypes.c_int64 _fields_ = [ ("logits", ctypes.c_void_p), ("probs", ctypes.c_void_p), ("sampled", ctypes.c_void_p), ("candidates", ctypes.c_void_p), + ("n_vocab", ctypes.c_int64), ] @@ -4654,9 +4657,9 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( # int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present); // 0.0 = disabled +# float penalty_repeat, // must be > 0.0, 1.0 = disabled +# float penalty_freq, // must be finite, 0.0 = disabled +# float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 221f0f6356..563dec81c1 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 221f0f6356efe2260023208365705ec5d5a7c8f5 +Subproject commit 563dec81c1c538aac0fad465ea933eb2a621a183 From 62d3ae5a20dea3a973bd06acd0abac9fdf694e70 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 3 Aug 2026 22:46:59 +0800 Subject: [PATCH 288/304] fix(windows): handle conflicting OpenMP and ggml libraries - Allow duplicate OpenMP runtimes in complex environments such as ComfyUI - Stop searching the deprecated /bin directory for ggml dynamic libraries Signed-off-by: JamePeng --- llama_cpp/_ggml.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 9a7dac517b..ee1a101870 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -35,6 +35,12 @@ def _preload_openmp_runtime(): if not _version_at_least("0.3.39"): return + # Some ComfyUI environments include complex software packages and may also contain + # additional OpenMP libraries (such as `libiomp5md.dll`); + # the best approach is to delete the conflicting libraries + # (i.e., OpenMP dynamic libraries that are not the VC143 version). + os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + libomp_path = (pathlib.Path(__file__).parent / "lib" / "libomp140.x86_64.dll") if not libomp_path.exists(): @@ -54,7 +60,7 @@ def _preload_openmp_runtime(): libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) libggml_base_paths = [ libggml_base_path / "lib", - libggml_base_path / "bin", + # libggml_base_path / "bin", # The `bin` path is no longer used as a search path for dynamic ggml libraries. ] # Load bundled OpenMP runtime before ggml-base on Windows. From 9af4ec35e7d25036a187c5b9fbdcbc44290d86b8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 3 Aug 2026 23:10:00 +0800 Subject: [PATCH 289/304] feat(internals): expose NextN embedding APIs on `LlamaContext` - Add accessors for NextN and layer input embeddings - Support selecting the NextN layer offset - Expose the auxiliary context handle - Validate layer IDs, offsets, and unavailable outputs Signed-off-by: JamePeng --- llama_cpp/_internals.py | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 9b37ebcc74..ebb785b3fe 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -868,6 +868,63 @@ def get_embeddings_seq(self, seq_id: int): self._assert_ctx() return llama_cpp.llama_get_embeddings_seq(self.ctx, seq_id) + def set_embeddings_nextn(self, enabled: bool, masked: bool) -> None: + """ + Set whether the context outputs nextn embeddings or not + If masked == true, output the embeddings only for the tokens with batch.logits != 0 + If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits + """ + self._assert_ctx() + llama_cpp.llama_set_embeddings_nextn(self.ctx, enabled, masked) + + def get_embeddings_nextn(self): + self._assert_ctx() + embeddings = llama_cpp.llama_get_embeddings_nextn(self.ctx) + if not embeddings: + raise RuntimeError("LlamaContext.get_embeddings_nextn: output is unavailable") + return embeddings + + def get_embeddings_nextn_ith(self, i: int): + self._assert_ctx() + embeddings = llama_cpp.llama_get_embeddings_nextn_ith(self.ctx, i) + if not embeddings: + raise RuntimeError( + f"LlamaContext.get_embeddings_nextn_ith: invalid output index {i}" + ) + return embeddings + + def set_embeddings_layer_inp(self, layer_id: int, enabled: bool) -> None: + self._assert_ctx() + if layer_id < 0: + raise ValueError("layer_id must be non-negative") + llama_cpp.llama_set_embeddings_layer_inp(self.ctx, layer_id, enabled) + + def get_embeddings_layer_inp(self, layer_id: int): + self._assert_ctx() + if layer_id < 0: + raise ValueError("layer_id must be non-negative") + embeddings = llama_cpp.llama_get_embeddings_layer_inp(self.ctx, layer_id) + if not embeddings: + raise RuntimeError( + f"LlamaContext.get_embeddings_layer_inp: layer {layer_id} output is unavailable" + ) + return embeddings + + def set_nextn_layer_offset(self, offset: int) -> None: + """ + Select which appended NextN block the DECODER_MTP graph runs (offset past + the trunk: il = n_layer() + offset). Used by the speculative NextN driver to + chain multiple trained NextN heads. Default 0 (first head). + """ + self._assert_ctx() + if offset < 0: + raise ValueError("NextN layer offset must be non-negative") + llama_cpp.llama_set_nextn_layer_offset(self.ctx, offset) + + def get_ctx_other(self): + self._assert_ctx() + return llama_cpp.llama_get_ctx_other(self.ctx) + def reset_timings(self): llama_cpp.llama_perf_context_reset(self.ctx) From 00a84126e0bd168b030a74c1abe5dbb6ffa85827 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 4 Aug 2026 21:08:51 +0800 Subject: [PATCH 290/304] Update Submodule vendor/llama.cpp 563dec8..1c3c967 Signed-off-by: JamePeng --- llama_cpp/_internals.py | 5 +- llama_cpp/llama_cpp.py | 127 ++++++++++++++++++++++------------------ vendor/llama.cpp | 2 +- 3 files changed, 73 insertions(+), 61 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index ebb785b3fe..7cbd87e4b5 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -2064,6 +2064,7 @@ def _build_sampler_chain(self): # Note: In some implementations, penalties come before other samplers if CommonSamplerType.PENALTIES in p.samplers: s.add_penalties( + self.n_vocab, p.penalty_last_n, p.penalty_repeat, p.penalty_freq, @@ -3176,8 +3177,8 @@ def add_grammar( c_trigger_tokens, len(trigger_tokens) )) - def add_penalties(self, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float): - self._add_sampler(llama_cpp.llama_sampler_init_penalties(penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) + def add_penalties(self, n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float): + self._add_sampler(llama_cpp.llama_sampler_init_penalties(n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) def add_dry(self, model: LlamaModel, multiplier: float, base: float, allowed_len: int, last_n: int, breakers: List[str]): """DRY (Don't Repeat Yourself) sampler.""" diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index c3b38005d8..609e0bb3b5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -146,60 +146,63 @@ class llama_vocab_type(enum.IntEnum): # https://github.com/ggml-org/llama.cpp/blob/master/src/llama-vocab.h#L10 # // pre-tokenization types # enum llama_vocab_pre_type { -# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, -# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, -# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, -# LLAMA_VOCAB_PRE_TYPE_MPT = 5, -# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, -# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, -# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, -# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, -# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, -# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, -# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, -# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, -# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, -# LLAMA_VOCAB_PRE_TYPE_PORO = 15, -# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, -# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, -# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, -# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, -# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, -# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, -# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, -# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, -# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, -# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, -# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, -# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, -# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, -# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, -# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, -# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, -# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, -# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, -# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, -# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36, -# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37, -# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38, -# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39, -# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40, -# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41, -# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42, -# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43, -# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44, -# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45, -# LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46, -# LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47, -# LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, -# LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, -# LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, -# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, -# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, -# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, +# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, +# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, +# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, +# LLAMA_VOCAB_PRE_TYPE_MPT = 5, +# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, +# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, +# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, +# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, +# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, +# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, +# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, +# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, +# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, +# LLAMA_VOCAB_PRE_TYPE_PORO = 15, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, +# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, +# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, +# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, +# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, +# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, +# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, +# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, +# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, +# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, +# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, +# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, +# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, +# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, +# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, +# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, +# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, +# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, +# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36, +# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37, +# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38, +# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39, +# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40, +# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41, +# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42, +# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43, +# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44, +# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45, +# LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46, +# LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47, +# LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, +# LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, +# LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, +# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, +# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, +# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, +# LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54, +# LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55, +# LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, # }; class llama_vocab_pre_type(enum.IntEnum): LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 @@ -256,6 +259,9 @@ class llama_vocab_pre_type(enum.IntEnum): LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51 LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52 LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53 + LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54 + LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55 + LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56 # // note: these values should be synchronized with ggml_rope @@ -4130,7 +4136,6 @@ def llama_chat_builtin_templates( # struct ggml_tensor * probs; # struct ggml_tensor * sampled; # struct ggml_tensor * candidates; -# int64_t n_vocab; # }; class llama_sampler_data(ctypes.Structure): if TYPE_CHECKING: @@ -4138,14 +4143,12 @@ class llama_sampler_data(ctypes.Structure): probs: ctypes.c_void_p sampled: ctypes.c_void_p candidates: ctypes.c_void_p - n_vocab: ctypes.c_int64 _fields_ = [ ("logits", ctypes.c_void_p), ("probs", ctypes.c_void_p), ("sampled", ctypes.c_void_p), ("candidates", ctypes.c_void_p), - ("n_vocab", ctypes.c_int64), ] @@ -4656,16 +4659,24 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( +# int32_t n_vocab, # int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) # float penalty_repeat, // must be > 0.0, 1.0 = disabled # float penalty_freq, // must be finite, 0.0 = disabled # float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], + [ + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_float, + ctypes.c_float, + ctypes.c_float, + ], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 563dec81c1..1c3c9674de 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 563dec81c1c538aac0fad465ea933eb2a621a183 +Subproject commit 1c3c9674de4d455f1e571bed808252af54932767 From 64e2114ed77d1155bd286d881144388062d9001b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 6 Aug 2026 01:44:47 +0800 Subject: [PATCH 291/304] Update Submodule vendor/llama.cpp 1c3c967..69bf643 Signed-off-by: JamePeng --- llama_cpp/_internals.py | 54 ++++++++++++++++++++++++++++++++++++++--- llama_cpp/llama_cpp.py | 37 +++++++++++++++++++++++----- vendor/llama.cpp | 2 +- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 7cbd87e4b5..226c8d8731 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -147,6 +147,55 @@ def n_head_kv(self) -> int: def n_swa(self) -> int: return llama_cpp.llama_model_n_swa(self.model) + def target_layer_ids_n(self) -> int: + """Return the number of target-model layers extracted by this model.""" + return llama_cpp.llama_model_target_layer_ids_n(self.model) + + def target_layer_ids(self) -> List[int]: + """Return the target-model layer indices extracted by this model.""" + count = self.target_layer_ids_n() + if count == 0: + return [] + + layer_ids = llama_cpp.llama_model_target_layer_ids(self.model) + if not layer_ids: + raise RuntimeError( + "LlamaModel.target_layer_ids: native API returned a null pointer " + f"for {count} layer IDs" + ) + return [int(layer_ids[i]) for i in range(count)] + + def get_tok_embd(self) -> npt.NDArray[np.float32]: + """Return a copy of the token embedding matrix as ``[n_vocab, n_embd]``.""" + element_count = llama_cpp.llama_model_get_tok_embd(self.model, None) + if element_count == 0: + raise RuntimeError( + "LlamaModel.get_tok_embd: token embedding matrix is unavailable" + ) + + n_vocab = self.n_vocab() + n_embd = self.n_embd() + expected_count = n_vocab * n_embd + if element_count != expected_count: + raise RuntimeError( + "LlamaModel.get_tok_embd: unexpected token embedding size: " + f"native API returned {element_count} elements, expected " + f"{expected_count} ({n_vocab} x {n_embd})" + ) + + out = np.empty(element_count, dtype=np.float32) + written = llama_cpp.llama_model_get_tok_embd( + self.model, + out.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + ) + if written != element_count: + raise RuntimeError( + "LlamaModel.get_tok_embd: failed to copy the complete token " + f"embedding matrix ({written}/{element_count} elements)" + ) + + return out.reshape(n_vocab, n_embd) + def rope_freq_scale_train(self) -> float: """ Get the model's RoPE frequency scaling factor @@ -1767,7 +1816,7 @@ class LlamaSamplingParams: dynatemp_range: float = 0.00 # 0.0 = disabled dynatemp_exponent: float = 1.00 # controls how entropy maps to temperature in dynamic temperature sampler - penalty_last_n: int = 64 # last n tokens to penalize (0 = disable penalty, -1 = context size) + penalty_last_n: int = 64 # last n tokens to penalize (0 = disable penalty) penalty_repeat: float = 1.0 # 1.0 = disabled penalty_freq: float = 0.00 # 0.0 = disabled penalty_present: float = 0.00 # 0.0 = disabled @@ -1775,7 +1824,7 @@ class LlamaSamplingParams: dry_multiplier: float = 0.0 # 0.0 = disabled; DRY repetition penalty for tokens extending repetition: dry_base: float = 1.75 # 0.0 = disabled; multiplier * base ^ (length of sequence before token - allowed length) dry_allowed_length: int = 2 # tokens extending repetitions beyond this receive penalty - dry_penalty_last_n: int = -1 # how many tokens to scan for repetitions (0 = disable penalty, -1 = context size) + dry_penalty_last_n: int = 64 # how many tokens to scan for repetitions (0 = disable penalty) adaptive_target: float = -1.0 # select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) adaptive_decay: float = 0.90 # EMA decay for adaptation; history ā‰ˆ 1/(1-decay) tokens (0.0 - 0.99) @@ -3188,7 +3237,6 @@ def add_dry(self, model: LlamaModel, multiplier: float, base: float, allowed_len self._add_sampler(llama_cpp.llama_sampler_init_dry( model.vocab, - model.n_ctx_train(), multiplier, base, allowed_len, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 609e0bb3b5..ed1179df28 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -4660,7 +4660,7 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( # int32_t n_vocab, -# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty) # float penalty_repeat, // must be > 0.0, 1.0 = disabled # float penalty_freq, // must be finite, 0.0 = disabled # float penalty_present); // must be finite, 0.0 = disabled @@ -4687,20 +4687,18 @@ def llama_sampler_init_penalties( # /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 -# LLAMA_API struct llama_sampler * llama_sampler_init_dry( +# LLAMA_API struct llama_sampler * llama_sampler_init_dry( # const struct llama_vocab * vocab, -# int32_t n_ctx_train, # float dry_multiplier, # float dry_base, # int32_t dry_allowed_length, -# int32_t dry_penalty_last_n, +# int32_t dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty) # const char ** seq_breakers, # size_t num_breakers); @ctypes_function( "llama_sampler_init_dry", [ llama_vocab_p_ctypes, - ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_int32, @@ -4712,7 +4710,6 @@ def llama_sampler_init_penalties( ) def llama_sampler_init_dry( vocab: llama_vocab_p, - n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, @@ -5426,3 +5423,31 @@ def llama_model_target_layer_ids_n( returns the number of extracted layers from target model """ ... + +# // retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) +# // returns total number of elements or 0 on error +# // if out is nullptr, returns the number of tokens without writing to out +# // caller must allocate enough memory for out before calling +# LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out); +@ctypes_function_llama_ext( + [ + "llama_model_get_tok_embd", + "?llama_model_get_tok_embd@@YAIPEBUllama_model@@PEAM@Z", + "__Z24llama_model_get_tok_embdPK11llama_modelPf", + "_Z24llama_model_get_tok_embdPK11llama_modelPf", + ], + [llama_model_p_ctypes, ctypes.POINTER(ctypes.c_float)], + ctypes.c_uint32, + required=False, +) +def llama_model_get_tok_embd( + model: llama_model_p, + out: Optional[ctypes.POINTER(ctypes.c_float)], # type: ignore +) -> int: + """ + retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) + returns total number of elements or 0 on error + if out is nullptr, returns the number of tokens without writing to out + caller must allocate enough memory for out before calling + """ + ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1c3c9674de..69bf643791 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1c3c9674de4d455f1e571bed808252af54932767 +Subproject commit 69bf6437914596fbbc4caf09a7ac16f2acdd1a94 From 3397ecb3d64a4f7ba21f0877baa34f6f6a852386 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 8 Aug 2026 17:15:44 +0800 Subject: [PATCH 292/304] feat(mtmd): update bindings for audio generation and chunk serialization - add input chunk save/load APIs - add experimental generated-audio types and processing APIs - support HunyuanVL decoder positions - sync enum values and correct ctypes signatures Signed-off-by: JamePeng --- llama_cpp/mtmd_cpp.py | 250 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 231 insertions(+), 19 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index fcfaa86ee7..7754b58b73 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -72,11 +72,26 @@ # MTMD_INPUT_CHUNK_TYPE_TEXT, # MTMD_INPUT_CHUNK_TYPE_IMAGE, # MTMD_INPUT_CHUNK_TYPE_AUDIO, +# MTMD_INPUT_CHUNK_TYPE_COUNT, // for validation # }; class mtmd_input_chunk_type(enum.IntEnum): - MTMD_INPUT_CHUNK_TYPE_TEXT = 0 + MTMD_INPUT_CHUNK_TYPE_TEXT = 0 MTMD_INPUT_CHUNK_TYPE_IMAGE = 1 MTMD_INPUT_CHUNK_TYPE_AUDIO = 2 + MTMD_INPUT_CHUNK_TYPE_COUNT = 3 + +# // position indexing for decoder model +# enum mtmd_pos_type { +# MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens +# MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes +# MTMD_POS_TYPE_HUNYUANVL, // HunyuanVL mrope + BOI/EOI/newline layout with XD-RoPE dim-3 +# MTMD_POS_TYPE_COUNT, // for validation +# }; +class mtmd_pos_type(enum.IntEnum): + MTMD_POS_TYPE_NORMAL = 0 # number of positions equals to number of tokens + MTMD_POS_TYPE_MROPE = 1 # qwen-vl mrope style, each image takes max(t,h,w) position indexes + MTMD_POS_TYPE_HUNYUANVL = 2 # HunyuanVL mrope + BOI/EOI/newline layout with XD-RoPE dim-3 + MTMD_POS_TYPE_COUNT = 3 # for validation # // opaque types @@ -96,15 +111,6 @@ class mtmd_input_chunk_type(enum.IntEnum): mtmd_bitmap_p = NewType("mtmd_bitmap_p", int) mtmd_bitmap_p_ctypes = c_void_p -# // position indexing for decoder model -# enum mtmd_pos_type { -# MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens -# MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes -# }; -class mtmd_pos_type(enum.IntEnum): - MTMD_POS_TYPE_NORMAL = 0 # number of positions equals to number of tokens - MTMD_POS_TYPE_MROPE = 1 # qwen-vl mrope style, each image takes max(t,h,w) position indexes - # struct mtmd_image_tokens { # uint32_t nx; // number of tokens in x direction # uint32_t ny; // number of tokens in y direction @@ -401,13 +407,13 @@ def mtmd_bitmap_init( # MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); @ctypes_function_mtmd( "mtmd_bitmap_init_from_audio", [ - c_uint, + c_size_t, POINTER(c_float) ], mtmd_bitmap_p_ctypes, ) def mtmd_bitmap_init_from_audio( - n_samples: c_uint, + n_samples: c_size_t, data: POINTER(c_float), # type: ignore /, ) -> mtmd_bitmap_p: @@ -635,6 +641,56 @@ def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p): """ ... +# // save/load an input chunk to/from a buffer (useful for KV save/load) +# // important: only chunk's metadata will be saved, the actual image/audio data will not be saved +# // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() +# // out_buf can be nullptr (to query expected_out_len) +# // returns 0 on success, non-zero on failure +# MTMD_API int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len); +@ctypes_function_mtmd("mtmd_input_chunk_save", + [ + mtmd_input_chunk_p_ctypes, + c_char_p, + c_size_t, + POINTER(c_size_t), + ], + c_int32, +) +def mtmd_input_chunk_save( + chunk: mtmd_input_chunk_p, + out_buf: bytes, + out_len: c_size_t, + expected_out_len: POINTER(c_size_t), # type: ignore +) -> int: + """ + save an input chunk to/from a buffer (useful for KV save) + important: only chunk's metadata will be saved, the actual image/audio data will not be saved + the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() + out_buf can be nullptr (to query expected_out_len) + returns 0 on success, non-zero on failure + """ + ... + +# // returns nullptr on failure +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len); +@ctypes_function_mtmd("mtmd_input_chunk_load", + [ + c_char_p, + c_size_t + ], + mtmd_input_chunk_p_ctypes, +) +def mtmd_input_chunk_load( + buf: bytes, + len: c_size_t, +) -> mtmd_input_chunk_p: + """ + load an input chunk from a buffer (useful for KV load) + important: only chunk's metadata will be saved, the actual image/audio data will not be saved + the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() + returns nullptr on failure + """ + ... # // mtmd_image_tokens # // @@ -697,8 +753,7 @@ class mtmd_decoder_pos(Structure): x: c_uint32 y: c_uint32 -mtmd_decoder_pos_p = POINTER(mtmd_decoder_pos) -mtmd_decoder_pos_p_ctypes = c_void_p +mtmd_decoder_pos_p_ctypes = POINTER(mtmd_decoder_pos) # // get position for decoder attention, to be used by M-RoPE models # // i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 @@ -955,14 +1010,171 @@ def mtmd_get_cap_from_file(mmproj_fname: c_char_p) -> mtmd_caps: ... +# // EXPERIMENTAL API for audio generation, subjected to breaking changes + +# // represent the pipeline type +# enum mtmd_gen_audio_type { +# MTMD_GEN_AUDIO_TYPE_NONE, // not supported +# MTMD_GEN_AUDIO_TYPE_QWEN3TTS, +# }; +class mtmd_gen_audio_type(enum.IntEnum): + """Generated audio pipeline type.""" + MTMD_GEN_AUDIO_TYPE_NONE = 0 + MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 + +# struct mtmd_gen_audio_info { +# enum mtmd_gen_audio_type type; +# int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +# }; +class mtmd_gen_audio_info(Structure): + """Audio generation pipeline information.""" + + _fields_ = [ + ("type", c_int), + ("sample_rate", c_int32), + ] + + if TYPE_CHECKING: + type: int + sample_rate: int + +# MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); +@ctypes_function_mtmd( + "mtmd_gen_audio_get_info", + [ + mtmd_context_p_ctypes, + ], + mtmd_gen_audio_info, +) +def mtmd_gen_audio_get_info( + ctx: mtmd_context_p, +) -> mtmd_gen_audio_info: + ... + +# enum mtmd_gen_process_type { +# MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.) +# MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert semantic to PCM audio +# // for qwen3tts, this is code2wav +# }; +class mtmd_gen_process_type(enum.IntEnum): + """Generated audio processing stage.""" + # hidden state -> semantic codes + MTMD_GEN_PROCESS_TYPE_GEN_CODE = 0 + # semantic codes -> PCM audio + MTMD_GEN_PROCESS_TYPE_GEN_WAV = 1 + +# struct mtmd_gen_inp { +# enum mtmd_gen_process_type type; + +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# int32_t code0; // the sampled codebook 0 entry from backbone +# float * embd; // the hidden state from backbone, must have n_text_embd elements +# int32_t top_k; +# float top_p; + +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# int32_t * codes; +# size_t n_codes; +# const char * state_data; +# size_t state_size; +# }; +class mtmd_gen_inp(Structure): + """Audio generation input.""" + + _fields_ = [ + ("type", c_int), + # GEN_CODE + ("code0", c_int32), + ("embd", POINTER(c_float)), + ("top_k", c_int32), + ("top_p", c_float), + # GEN_WAV + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("state_data", c_char_p), + ("state_size", c_size_t), + ] + + if TYPE_CHECKING: + type: int + code0: int + embd: POINTER[c_float] + top_k: int + top_p: float + codes: POINTER[c_int32] + n_codes: int + state_data: bytes + state_size: int + +# struct mtmd_gen_out { +# // note: output memory is allocated by the context, valid until next process() call +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# const int32_t * codes; +# size_t n_codes; +# const float * embd; // the generated hidden state, to be fed back to backbone +# // it must have n_text_embd elements +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# const float * audio; +# size_t n_samples; +# const char * state_data; +# size_t state_size; +# }; +class mtmd_gen_out(Structure): + """Audio generation output. + Memory is owned by mtmd_context and valid until + the next mtmd_gen_audio_process() call. + """ + + _fields_ = [ + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("embd", POINTER(c_float)), + ("audio", POINTER(c_float)), + ("n_samples", c_size_t), + ("state_data", c_char_p), + ("state_size", c_size_t), + ] + + if TYPE_CHECKING: + codes: POINTER[c_int32] + n_codes: int + embd: POINTER[c_float] + audio: POINTER[c_float] + n_samples: int + state_data: bytes + state_size: int + +# // note: this API is stateless, caller must handle state management and audio frame accumulation +# MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, +# const struct mtmd_gen_inp * inp, +# struct mtmd_gen_out * out); +@ctypes_function_mtmd( + "mtmd_gen_audio_process", + [ + mtmd_context_p_ctypes, + POINTER(mtmd_gen_inp), + POINTER(mtmd_gen_out), + ], + c_int32, +) +def mtmd_gen_audio_process( + ctx: mtmd_context_p, + inp: POINTER(mtmd_gen_inp), # type: ignore + out: POINTER(mtmd_gen_out), # type: ignore +) -> int: + """ + note: this API is stateless, caller must handle state management and audio frame accumulation + """ + ... + # // test function, to be used in test-mtmd-c-api.c # MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); @ctypes_function_mtmd( "mtmd_test_create_input_chunks", [], - mtmd_input_chunk_p_ctypes, + mtmd_input_chunks_p_ctypes, ) -def mtmd_test_create_input_chunks() -> mtmd_input_chunk_p: +def mtmd_test_create_input_chunks() -> mtmd_input_chunks_p: ... @@ -1111,14 +1323,14 @@ def mtmd_helper_get_n_pos(chunks: mtmd_input_chunks_p) -> c_int32: @ctypes_function_mtmd("mtmd_helper_image_get_decoder_pos", [ mtmd_image_tokens_p_ctypes, c_int32, - mtmd_decoder_pos_p_ctypes + mtmd_decoder_pos_p_ctypes, ], None) def mtmd_helper_image_get_decoder_pos( image: mtmd_image_tokens_p, pos_0: c_int32, - out_pos: mtmd_decoder_pos_p # type: ignore -) -> c_int32: + out_pos: POINTER(mtmd_decoder_pos) # type: ignore +): """ helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE out_pos must have length == mtmd_helper_get_n_tokens(image) From 81190b03f6d177988112dad5fc919491a77705d1 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 8 Aug 2026 19:53:24 +0800 Subject: [PATCH 293/304] Bump version to 0.3.46 - This release mainly focuses on API synchronization and binding improvements. Some of the newly exposed MTMD interfaces are experimental API adaptations at this stage; the corresponding higher-level features have not yet been integrated. Signed-off-by: JamePeng --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 075b978847..b3d5800c04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.46] Extended Model APIs, MTMD Binding Updates, and Improved Runtime Compatibility + +- feat(mtmd): update bindings for audio generation and chunk serialization + - add input chunk save/load APIs + - add experimental generated-audio types and processing APIs + - support HunyuanVL decoder positions + - sync enum values and correct ctypes signatures + +- feat(model): expose target layer ids and token embeddings + - Add LlamaModel helpers for accessing target layer metadata and extracting + the token embedding matrix from the native model. + - The new APIs provide: + - target_layer_ids() for retrieving target model layer indices + - get_tok_embd() for copying the token embedding matrix as a NumPy array + - Add validation for native return values, including null pointers, unexpected + embedding sizes, and incomplete copy operations to provide clearer runtime + errors. + - Also update sampling parameter comments to match the current llama.cpp + behavior for penalty window configuration. + +- feat(internals): expose NextN embedding APIs on LlamaContext + - Add accessors for NextN and layer input embeddings + - Support selecting the NextN layer offset + - Expose the auxiliary context handle + - Validate layer IDs, offsets, and unavailable outputs + +- fix(windows): handle conflicting OpenMP and ggml libraries + - Allow duplicate OpenMP runtimes in complex environments such as ComfyUI + - Some ComfyUI environments include complex software packages and may also contain additional OpenMP libraries (such as `libiomp5md.dll`); + - the best approach is to delete the **conflicting libraries** (i.e., OpenMP dynamic libraries that are not the VC143 version). + - Stop searching the deprecated /bin directory for ggml dynamic libraries + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/69bf6437914596fbbc4caf09a7ac16f2acdd1a94](https://github.com/ggml-org/llama.cpp/commit/69bf6437914596fbbc4caf09a7ac16f2acdd1a94) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260808 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/d9d27a7bdf1c27d50c1490ad6acd825f31804902...3397ecb3d64a4f7ba21f0877baa34f6f6a852386 + ## [0.3.45] Reactivated Built-in Embeddings, Modern Model Loading, and Stronger Cross-Platform Reliability - fix(ctypes): correct llama-ext binding signatures diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index b359355f9e..d3fec3b867 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.45" +__version__ = "0.3.46" From 1b46d6f2dfd1f36236f1055aeaf8fd1ca997a048 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 10 Aug 2026 23:41:19 +0800 Subject: [PATCH 294/304] Update Submodule vendor/llama.cpp 69bf643..dd1ea52 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 69bf643791..dd1ea52433 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 69bf6437914596fbbc4caf09a7ac16f2acdd1a94 +Subproject commit dd1ea524333b1e697489067d7a4c39c60d32beee From 6da23a3ef05eb5e74a2efe655441c03eeb0cd337 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 11 Aug 2026 00:02:14 +0800 Subject: [PATCH 295/304] feat(llama): add multi-output backend sampler API support - expose per-sequence output limits in context parameters - sync sampler reset and state-copy interfaces with llama.cpp - preserve advanced context settings when reconstructing Llama instances - document ordered multi-output sampling behavior Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 5 ++- llama_cpp/llama.py | 16 +++++++- llama_cpp/llama_cpp.py | 91 ++++++++++++++++++++++++++++++++++------- 3 files changed, 94 insertions(+), 18 deletions(-) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 00add6ea4c..af5a3ce510 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -3,7 +3,7 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-07-29 +last_updated: 2026-08-10 version_target: "latest" --- @@ -87,7 +87,8 @@ mapping: | `n_ubatch` | `int` | `512` | Maximum number of tokens in a physical micro-batch processed by llama.cpp. | | `n_seq_max` | `int` | `1` | Maximum independent sequence states in one decode batch. Embedding calls split automatically at this limit; larger values enable more parallel sequences. | | `n_rs_seq` | `int` | `0` | Experimental recurrent-state snapshots retained per sequence for rollback. `0` disables rollback snapshots. | -| `n_outputs_max` | `int` | `0` | Maximum outputs in a physical batch. `0` is converted to the effective `n_batch`. | +| `n_outputs_max` | `int` | `0` | Maximum outputs in a physical batch. `0` lets llama.cpp use the effective `n_batch`. | +| `n_outputs_max_per_seq` | `int` | `1` | Maximum outputs per sequence. `0` lets llama.cpp use the effective `n_outputs_max`. | | `n_threads` | `int` | `None` | Number of threads for generation (defaults to CPU count // 2). | | `n_threads_batch` | `int` | `None` | Number of threads for batch processing (defaults to CPU count). | | `ctx_type` | `int` | `LLAMA_CONTEXT_TYPE_DEFAULT` | Context implementation selected by llama.cpp. Keep the default unless a model or backend requires another context type. | diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 8092f86956..2ca91888a3 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -126,6 +126,7 @@ def __init__( n_seq_max: int = 1, n_rs_seq: int = 0, n_outputs_max: int = 0, + n_outputs_max_per_seq: int = 1, n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, ctx_type: Optional[ @@ -234,8 +235,12 @@ def __init__( n_batch: Prompt processing maximum batch size n_ubatch: Physical batch size n_seq_max: max number of sequences (i.e. distinct states for recurrent models) + n_rs_seq: Number of recurrent-state snapshots per sequence for rollback. 0 disables rollback snapshots. Experimental. + n_outputs_max: Maximum outputs in a physical batch. 0 lets llama.cpp use the effective n_batch. + n_outputs_max_per_seq: Maximum outputs per sequence. 0 lets llama.cpp use the effective n_outputs_max. n_threads: Number of threads to use for generation n_threads_batch: Number of threads to use for batch processing + ctx_type: Context implementation type, such as the MTP context type. rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggml-org/llama.cpp/pull/2054 pooling_type: Pooling type, from `enum llama_pooling_type`. attention_type: attention type to use for embeddings @@ -497,6 +502,7 @@ def __init__( self.n_seq_max = n_seq_max self.n_rs_seq = n_rs_seq self.n_outputs_max = n_outputs_max + self.n_outputs_max_per_seq = n_outputs_max_per_seq self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count() @@ -514,7 +520,8 @@ def __init__( raise RuntimeError(f"n_seq_max must be <= {llama_cpp_lib.LLAMA_MAX_SEQ}") self.context_params.n_rs_seq = self.n_rs_seq - self.context_params.n_outputs_max = self.n_batch if self.n_outputs_max == 0 else self.n_outputs_max + self.context_params.n_outputs_max = max(self.n_outputs_max, 0) + self.context_params.n_outputs_max_per_seq = max(self.n_outputs_max_per_seq, 0) self.context_params.n_threads = self.n_threads self.context_params.n_threads_batch = self.n_threads_batch @@ -3470,10 +3477,15 @@ def __getstate__(self): # Context Params seed=self._seed, n_ctx=self.context_params.n_ctx, - n_batch=self.n_batch, + n_batch=self.context_params.n_batch, n_ubatch=self.context_params.n_ubatch, + n_seq_max=self.context_params.n_seq_max, + n_rs_seq=self.context_params.n_rs_seq, + n_outputs_max=self.context_params.n_outputs_max, + n_outputs_max_per_seq=self.context_params.n_outputs_max_per_seq, n_threads=self.context_params.n_threads, n_threads_batch=self.context_params.n_threads_batch, + ctx_type=self.context_params.ctx_type, rope_scaling_type=self.context_params.rope_scaling_type, pooling_type=self.context_params.pooling_type, attention_type=self.context_params.attention_type, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index ed1179df28..efd58e3ba5 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -886,14 +886,15 @@ class llama_sampler_seq_config(ctypes.Structure): # // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations # // https://github.com/ggml-org/llama.cpp/pull/7544 # struct llama_context_params { -# uint32_t n_ctx; // text context, 0 = from model -# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode -# uint32_t n_ubatch; // physical maximum batch size -# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) -# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] -# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) -# int32_t n_threads; // number of threads to use for generation -# int32_t n_threads_batch; // number of threads to use for batch processing +# uint32_t n_ctx; // text context, 0 = from model +# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode +# uint32_t n_ubatch; // physical maximum batch size +# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) +# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] +# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) +# uint32_t n_outputs_max_per_seq; // max outputs per sequence (0 = n_outputs_max) +# int32_t n_threads; // number of threads to use for generation +# int32_t n_threads_batch; // number of threads to use for batch processing # enum llama_context_type ctx_type; // set the context type (e.g. MTP) # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` @@ -954,6 +955,7 @@ class llama_context_params(ctypes.Structure): n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_rs_seq (int): number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] n_outputs_max (int): max outputs in a ubatch (0 = n_batch) + n_outputs_max_per_seq (int): max outputs per sequence (0 = n_outputs_max) n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing @@ -1001,6 +1003,7 @@ class llama_context_params(ctypes.Structure): n_seq_max: int n_rs_seq: int n_outputs_max: int + n_outputs_max_per_seq: int n_threads: int n_threads_batch: int ctx_type: int @@ -1039,6 +1042,7 @@ class llama_context_params(ctypes.Structure): ("n_seq_max", ctypes.c_uint32), ("n_rs_seq", ctypes.c_uint32), ("n_outputs_max", ctypes.c_uint32), + ("n_outputs_max_per_seq", ctypes.c_uint32), ("n_threads", ctypes.c_int32), ("n_threads_batch", ctypes.c_int32), ("ctx_type", ctypes.c_int), @@ -3295,6 +3299,9 @@ def llama_get_embeddings_seq( # // # // Get the backend sampled token for the ith token. +# // With multiple outputs, sampler state advances when the token is accepted, +# // not when it is read through this function. +# // When accepting multiple outputs, accept a contiguous prefix in output order. # // Returns LLAMA_TOKEN_NULL if no token was sampled. # LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); @ctypes_function( @@ -3307,6 +3314,9 @@ def llama_get_sampled_token_ith( ) -> ctypes.c_int32: """ Get the backend sampled token for the ith token. + With multiple outputs, sampler state advances when the token is accepted, + not when it is read through this function. + When accepting multiple outputs, accept a contiguous prefix in output order. Returns LLAMA_TOKEN_NULL if no token was sampled. """ ... @@ -4164,9 +4174,12 @@ class llama_sampler_data(ctypes.Structure): # // [EXPERIMENTAL] # // backend sampling interface: -# // return true if the backend supports all ops needed by the sampler +# // return true if the backend supports all ops needed by the sampler and can handle up to n_outputs_max_per_seq outputs per sequence # // note: call once per sampler -# bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft); +# bool (*backend_init)( +# struct llama_sampler * smpl, +# ggml_backend_buffer_type_t buft, +# uint32_t n_outputs_max_per_seq); # // call after .backend_apply() # void (*backend_accept)( @@ -4184,6 +4197,13 @@ class llama_sampler_data(ctypes.Structure): # // called before graph execution to set inputs for the current ubatch # void (*backend_set_input)(struct llama_sampler * smpl); + +# // called before rebuilding a sampling graph to clear any internal sampler state +# void (*backend_reset)(struct llama_sampler * smpl); + +# // copy mutable state from src into dst while keeping dst's references to the current sampling graph +# // src and dst must have the same type and configuration +# void (*copy_state)(const struct llama_sampler * src, struct llama_sampler * dst); # }; # const char * (*name)(const struct llama_sampler * smpl); @@ -4226,13 +4246,20 @@ class llama_sampler_data(ctypes.Structure): # --- EXPERIMENTAL Backend Sampling Interface --- -# bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft); +# // return true if the backend supports all ops needed by the sampler and can handle up to n_outputs_max_per_seq outputs per sequence +# // note: call once per sampler +# bool (*backend_init)( +# struct llama_sampler * smpl, +# ggml_backend_buffer_type_t buft, +# uint32_t n_outputs_max_per_seq); llama_sampler_backend_init_fn = ctypes.CFUNCTYPE( ctypes.c_bool, # return bool ctypes.c_void_p, # smpl - ctypes.c_void_p # buft + ctypes.c_void_p, # buft + ctypes.c_uint32, # n_outputs_max_per_seq ) +# // call after .backend_apply() # void (*backend_accept)(struct llama_sampler * smpl, struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_tensor * selected_token); llama_sampler_backend_accept_fn = ctypes.CFUNCTYPE( None, # return void @@ -4242,6 +4269,7 @@ class llama_sampler_data(ctypes.Structure): ctypes.c_void_p # selected_token ) +# // call after .backend_init() # void (*backend_apply)(struct llama_sampler * smpl, struct ggml_context * ctx, struct ggml_cgraph * gf, struct llama_sampler_data * data); llama_sampler_backend_apply_fn = ctypes.CFUNCTYPE( None, # return void @@ -4251,12 +4279,29 @@ class llama_sampler_data(ctypes.Structure): ctypes.POINTER(llama_sampler_data) # data ) +# // called before graph execution to set inputs for the current ubatch # void (*backend_set_input)(struct llama_sampler * smpl); llama_sampler_backend_set_input_fn = ctypes.CFUNCTYPE( None, # return void ctypes.c_void_p # smpl ) +# // called before rebuilding a sampling graph to clear any internal sampler state +# void (*backend_reset)(struct llama_sampler * smpl); +llama_sampler_backend_reset_fn = ctypes.CFUNCTYPE( + None, # return void + ctypes.c_void_p # smpl +) + +# // copy mutable state from src into dst while keeping dst's references to the current sampling graph +# // src and dst must have the same type and configuration +# void (*copy_state)(const struct llama_sampler * src, struct llama_sampler * dst); +llama_sampler_copy_state_fn = ctypes.CFUNCTYPE( + None, # return void + ctypes.c_void_p, # src + ctypes.c_void_p, # dst +) + class llama_sampler_i(ctypes.Structure): _fields_ = [ ("name", llama_sampler_name_fn), @@ -4271,6 +4316,8 @@ class llama_sampler_i(ctypes.Structure): ("backend_accept", llama_sampler_backend_accept_fn), ("backend_apply", llama_sampler_backend_apply_fn), ("backend_set_input", llama_sampler_backend_set_input_fn), + ("backend_reset", llama_sampler_backend_reset_fn), + ("copy_state", llama_sampler_copy_state_fn), ] @@ -4352,8 +4399,7 @@ def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], None, ) def llama_sampler_apply( - smpl: llama_sampler_p, cur_p: CtypesPointer[llama_token_data_array], / -): + smpl: llama_sampler_p, cur_p: CtypesPointer[llama_token_data_array]): ... @@ -4377,6 +4423,22 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... +# // copy mutable sampler state without changing dst or its sampling graph bindings +# // src and dst must have the same type and configuration +# LLAMA_API void llama_sampler_copy (const struct llama_sampler * src, struct llama_sampler * dst); +@ctypes_function( + "llama_sampler_copy", + [llama_sampler_p_ctypes, llama_sampler_p_ctypes], + None, +) +def llama_sampler_copy(src: llama_sampler_p, dst: llama_sampler_p): + """ + copy mutable sampler state without changing dst or its sampling graph bindings + src and dst must have the same type and configuration + """ + ... + + # // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) # LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); @ctypes_function( @@ -4823,6 +4885,7 @@ def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: # /// @details Sample and accept a token from the idx-th output of the last evaluation +# // For multiple outputs from one sampler, call this function in output order without gaps. # // # // Shorthand for: # // const auto * logits = llama_get_logits_ith(ctx, idx); From a0c9c41033412fe46e19cdf613767f200b0ebe87 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 11 Aug 2026 02:05:33 +0800 Subject: [PATCH 296/304] fix(internals): disable new backend hooks for custom samplers - Explicitly set backend_reset and copy_state to NULL - Clarify CPU callback behavior for CustomSampler - Document inherited backend behavior in ReasoningBudgetSampler Signed-off-by: JamePeng --- llama_cpp/_internals.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 226c8d8731..64077e705d 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -2489,12 +2489,18 @@ def force_reasoning_budget(self) -> bool: class CustomSampler: """ - Base class for Python-backed custom samplers in the Llama sampler chain. + CPU sampler adapter backed by Python callbacks. Responsibilities: - - Provides apply, accept, reset, free and clone callbacks for the C sampler chain. - - Keeps Python references alive to prevent GC while C sampler still holds function pointers. - - Implements safe close to clear all callback references. + - Expose Python apply, accept, reset, free, and clone functions through + llama_sampler_i callbacks. + - Keep callback references alive while llama.cpp holds their function + pointers. + - Release the native sampler and break callback reference cycles on close. + + Backend sampling is intentionally unsupported. Every backend hook in + llama_sampler_i is explicitly initialized to NULL, including backend_reset + and copy_state, so llama.cpp keeps this sampler on the CPU callback path. """ def __init__( @@ -2545,7 +2551,7 @@ def _cb_clone(_): self._cb_free_ref = llama_cpp.llama_sampler_free_fn(_cb_free) self._cb_clone_ref = llama_cpp.llama_sampler_clone_fn(_cb_clone) - # Build llama_sampler_i + # Build the CPU-facing llama_sampler_i callback table. self.llama_sampler_i = llama_cpp.llama_sampler_i() self.llama_sampler_i.name = self._cb_name_ref @@ -2555,7 +2561,9 @@ def _cb_clone(_): self.llama_sampler_i.free = self._cb_free_ref self.llama_sampler_i.clone = self._cb_clone_ref - # Disable backend hooks + # Explicitly disable every backend hook instead of relying on ctypes + # zero-initialization. Python-backed samplers operate through the CPU + # callbacks above and do not own backend sampling graph state. self.llama_sampler_i.backend_init = ctypes.cast( 0, llama_cpp.llama_sampler_backend_init_fn ) @@ -2568,6 +2576,12 @@ def _cb_clone(_): self.llama_sampler_i.backend_set_input = ctypes.cast( 0, llama_cpp.llama_sampler_backend_set_input_fn ) + self.llama_sampler_i.backend_reset = ctypes.cast( + 0, llama_cpp.llama_sampler_backend_reset_fn + ) + self.llama_sampler_i.copy_state = ctypes.cast( + 0, llama_cpp.llama_sampler_copy_state_fn + ) self.sampler_p = llama_cpp.llama_sampler_init( ctypes.pointer(self.llama_sampler_i), @@ -2625,6 +2639,11 @@ class ReasoningBudgetSampler(CustomSampler): This mirrors the core idea of llama.cpp's reasoning-budget sampler while keeping the Python API small and explicit. + + As a CustomSampler subclass, this remains CPU/Python-backed. Its backend + hooks, including backend_reset and copy_state, stay disabled; runtime state + is managed by the regular _accept(), _apply(), _reset(), and _clone() + callbacks instead. """ def __init__( From 8a3bcbfb5e9ed628327ad94b2587cd278507ffa9 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 11 Aug 2026 02:43:31 +0800 Subject: [PATCH 297/304] fix(types): use size_t for sampler count Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index efd58e3ba5..6ae590ea44 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1071,7 +1071,7 @@ class llama_context_params(ctypes.Structure): ("swa_full", ctypes.c_bool), ("kv_unified", ctypes.c_bool), ("samplers", llama_sampler_seq_config_p), - ("n_samplers", ctypes.c_int), + ("n_samplers", ctypes.c_size_t), ("ctx_other", ctypes.c_void_p), ] From d5a108d726aff2c0ee9b440ed7d33e55e27b689f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 14 Aug 2026 01:59:35 +0800 Subject: [PATCH 298/304] Update Submodule vendor/llama.cpp dd1ea52..9c5531e Signed-off-by: JamePeng --- llama_cpp/llama.py | 2 +- llama_cpp/llama_cpp.py | 44 ++++++++++++++++++++++-------------- llama_cpp/server/settings.py | 2 +- vendor/llama.cpp | 2 +- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2ca91888a3..410eee3a98 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -104,7 +104,7 @@ def __init__( cpu_moe: bool = False, n_cpu_moe: int = 0, split_mode: int = llama_cpp_lib.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, - load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_MMAP, + load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_AUTO, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 6ae590ea44..41c6de4ddf 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -511,18 +511,20 @@ class llama_split_mode(enum.IntEnum): LLAMA_SPLIT_MODE_TENSOR = 3 # enum llama_load_mode { -# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode -# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model -# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing -# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing -# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# LLAMA_LOAD_MODE_AUTO = -1, // auto-detect based on device capabilities +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available # }; class llama_load_mode(enum.IntEnum): - LLAMA_LOAD_MODE_NONE = 0 # no special loading mode - LLAMA_LOAD_MODE_MMAP = 1 # memory map the model - LLAMA_LOAD_MODE_MLOCK = 2 # force system to keep model in RAM rather than swapping or compressing - LLAMA_LOAD_MODE_MMAP_MLOCK = 3 # mmap + force system to keep model in RAM rather than swapping or compressing - LLAMA_LOAD_MODE_DIRECT_IO = 4 # use direct I/O if available + LLAMA_LOAD_MODE_AUTO = -1 # auto-detect based on device capabilities + LLAMA_LOAD_MODE_NONE = 0 # no special loading mode + LLAMA_LOAD_MODE_MMAP = 1 # memory map the model + LLAMA_LOAD_MODE_MLOCK = 2 # force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_MMAP_MLOCK = 3 # mmap + force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_DIRECT_IO = 4 # use direct I/O if available # LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); @ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) @@ -1298,6 +1300,17 @@ class llama_chat_message(ctypes.Structure): llama_adapter_cvec_p_ctypes = ctypes.POINTER(ctypes.c_void_p) +# LLAMA_API const char * llama_version(void); +@ctypes_function( + "llama_version", + [], + ctypes.c_char_p, +) +def llama_version() -> bytes: + """Get libllama version""" + ... + + # // Helpers for getting default parameters # LLAMA_API struct llama_model_params llama_model_default_params(void); @ctypes_function( @@ -2853,7 +2866,7 @@ def llama_state_seq_save_file( ) -> int: ... - +# If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded # LLAMA_API size_t llama_state_seq_load_file( # struct llama_context * ctx, # const char * filepath, @@ -2882,6 +2895,9 @@ def llama_state_seq_load_file( n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, ) -> int: + """ + If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded + """ ... # define LLAMA_STATE_SEQ_FLAGS_NONE 0 @@ -4423,8 +4439,6 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... -# // copy mutable sampler state without changing dst or its sampling graph bindings -# // src and dst must have the same type and configuration # LLAMA_API void llama_sampler_copy (const struct llama_sampler * src, struct llama_sampler * dst); @ctypes_function( "llama_sampler_copy", @@ -4432,10 +4446,6 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: None, ) def llama_sampler_copy(src: llama_sampler_p, dst: llama_sampler_p): - """ - copy mutable sampler state without changing dst or its sampling graph bindings - src and dst must have the same type and configuration - """ ... diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 62ce3b5044..8652d6d942 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -35,7 +35,7 @@ class ModelSettings(BaseSettings): description="how to split the model across multiple GPUs", ) load_mode: int = Field( - default=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP, + default=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_AUTO, description="how to load the model", ) main_gpu: int = Field( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index dd1ea52433..9c5531e2bf 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit dd1ea524333b1e697489067d7a4c39c60d32beee +Subproject commit 9c5531e2bf2c95e86eeac807d7109de29266ca7b From 962eef15b5375a84345d8d82150595d63084b91e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 14 Aug 2026 02:21:01 +0800 Subject: [PATCH 299/304] fix(llama): fully clear model state on reset - Clear native context memory and invalidate hybrid checkpoints to keep Python state synchronized across standard, recurrent, and hybrid models. Signed-off-by: JamePeng --- llama_cpp/llama.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 410eee3a98..7da01140a4 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1089,9 +1089,20 @@ def set_seed(self, seed: int): self._seed = seed def reset(self): - """Reset the model state.""" + """Reset all Python and native model state.""" + # Use a full memory clear rather than sequence removal: recurrent state + # cannot always be partially truncated, and hybrid memory must clear + # both its attention KV cache and recurrent state. + self._ctx.memory_clear(True) + + # Keep the Python-side token cursor in sync with the empty native state. self.n_tokens = 0 + # Hybrid checkpoints contain snapshots of the state cleared above and + # must not be reused after a reset. + if self.is_hybrid and self._hybrid_cache_mgr is not None: + self._hybrid_cache_mgr.clear() + def abort(self) -> None: """ Safely aborts any ongoing text generation. @@ -1737,10 +1748,7 @@ def generate( ) if reset: # No prefix matched at all. Completely clear the KV cache to prevent context poisoning. - self.n_tokens = 0 - self._ctx.memory_clear(True) - if self.is_hybrid and self._hybrid_cache_mgr is not None: - self._hybrid_cache_mgr.clear() + self.reset() if self.verbose: print("Llama.generate: Context reset requested or no prefix match. Cleared KV cache.", file=sys.stderr) From 403771f9979505db02bdd5071dfce1505d15cf30 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 14 Aug 2026 05:39:42 +0800 Subject: [PATCH 300/304] feat(mtmd): sync Pocket TTS and audio helper API bindings - add Pocket TTS audio generation types and fields - update generated-audio structures for the latest MTMD ABI - expose default generation parameters and audio helper APIs - add multimodal chat capability detection Signed-off-by: JamePeng --- llama_cpp/mtmd_cpp.py | 296 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 293 insertions(+), 3 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 7754b58b73..ec99b7ac6a 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -1016,15 +1016,18 @@ def mtmd_get_cap_from_file(mmproj_fname: c_char_p) -> mtmd_caps: # enum mtmd_gen_audio_type { # MTMD_GEN_AUDIO_TYPE_NONE, // not supported # MTMD_GEN_AUDIO_TYPE_QWEN3TTS, +# MTMD_GEN_AUDIO_TYPE_POCKETTTS, # }; class mtmd_gen_audio_type(enum.IntEnum): """Generated audio pipeline type.""" - MTMD_GEN_AUDIO_TYPE_NONE = 0 - MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 + MTMD_GEN_AUDIO_TYPE_NONE = 0 + MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 + MTMD_GEN_AUDIO_TYPE_POCKETTTS = 2 # struct mtmd_gen_audio_info { # enum mtmd_gen_audio_type type; # int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +# const char * model_variant; // name of the weight variant, can be nullptr if not applicable # }; class mtmd_gen_audio_info(Structure): """Audio generation pipeline information.""" @@ -1032,11 +1035,13 @@ class mtmd_gen_audio_info(Structure): _fields_ = [ ("type", c_int), ("sample_rate", c_int32), + ("model_variant", c_char_p), ] if TYPE_CHECKING: type: int sample_rate: int + model_variant: Optional[bytes] # MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); @ctypes_function_mtmd( @@ -1055,6 +1060,7 @@ def mtmd_gen_audio_get_info( # MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.) # MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert semantic to PCM audio # // for qwen3tts, this is code2wav +# // for pocket-tts, this is mimi decoder # }; class mtmd_gen_process_type(enum.IntEnum): """Generated audio processing stage.""" @@ -1071,10 +1077,15 @@ class mtmd_gen_process_type(enum.IntEnum): # float * embd; // the hidden state from backbone, must have n_text_embd elements # int32_t top_k; # float top_p; +# uint32_t seed; // UINT32_MAX for random +# float temp; // sampling temperature, or noise scale for flow-matching decoders # // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# // pass either codes (discrete) or feats (continuous), depending on the pipeline # int32_t * codes; # size_t n_codes; +# const float * feats; +# size_t n_feats; # const char * state_data; # size_t state_size; # }; @@ -1088,21 +1099,31 @@ class mtmd_gen_inp(Structure): ("embd", POINTER(c_float)), ("top_k", c_int32), ("top_p", c_float), + ("seed", c_uint32), + ("temp", c_float), # GEN_WAV ("codes", POINTER(c_int32)), ("n_codes", c_size_t), + ("feats", POINTER(c_float)), + ("n_feats", c_size_t), ("state_data", c_char_p), ("state_size", c_size_t), ] if TYPE_CHECKING: + # GEN_CODE type: int code0: int embd: POINTER[c_float] top_k: int top_p: float + seed: int + temp: float + # GEN_WAV codes: POINTER[c_int32] n_codes: int + feats: POINTER[c_float] + n_feats: int state_data: bytes state_size: int @@ -1110,9 +1131,12 @@ class mtmd_gen_inp(Structure): # // note: output memory is allocated by the context, valid until next process() call # // for MTMD_GEN_PROCESS_TYPE_GEN_CODE # const int32_t * codes; -# size_t n_codes; +# size_t n_codes; +# const float * feats; // continuous counterpart of codes +# size_t n_feats; # const float * embd; // the generated hidden state, to be fed back to backbone # // it must have n_text_embd elements +# bool is_eos; // only set by pipelines having the EOS head inside mmproj # // for MTMD_GEN_PROCESS_TYPE_GEN_WAV # const float * audio; # size_t n_samples; @@ -1128,7 +1152,10 @@ class mtmd_gen_out(Structure): _fields_ = [ ("codes", POINTER(c_int32)), ("n_codes", c_size_t), + ("feats", POINTER(c_float)), + ("n_feats", c_size_t), ("embd", POINTER(c_float)), + ("is_eos", c_bool), ("audio", POINTER(c_float)), ("n_samples", c_size_t), ("state_data", c_char_p), @@ -1138,12 +1165,32 @@ class mtmd_gen_out(Structure): if TYPE_CHECKING: codes: POINTER[c_int32] n_codes: int + feats: POINTER[c_float] + n_feats: int embd: POINTER[c_float] + is_eos: bool audio: POINTER[c_float] n_samples: int state_data: bytes state_size: int +# // defaults tuned for the loaded pipeline, callers override only what they care about +# MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx); +@ctypes_function_mtmd( + "mtmd_gen_inp_default", + [ + mtmd_context_p_ctypes, + ], + mtmd_gen_inp, +) +def mtmd_gen_inp_default( + ctx: mtmd_context_p, +) -> mtmd_gen_inp: + """ + defaults tuned for the loaded pipeline, callers override only what they care about + """ + ... + # // note: this API is stateless, caller must handle state management and audio frame accumulation # MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, # const struct mtmd_gen_inp * inp, @@ -1653,3 +1700,246 @@ def mtmd_helper_video_read_next( -2 on error """ ... + +# // return true if model can be used for chat +# MTMD_API bool mtmd_helper_model_can_chat(struct llama_context * lctx, struct mtmd_context * mctx); +@ctypes_function_mtmd( + "mtmd_helper_model_can_chat", [ + llama_cpp_lib.llama_context_p_ctypes, + mtmd_context_p_ctypes, + ], + c_bool, +) +def mtmd_helper_model_can_chat( + lctx: llama_cpp_lib.llama_context_p, + mctx: mtmd_context_p, + /, +) -> bool: + """ + return true if model can be used for chat + """ + ... + +# // +# // Audio generation helpers +# // (early-stage experimental, subjected to breaking changes) +# // + +# // audio generation helper context +# // contains accumulator for generated audio features and PCM audio +# struct mtmd_helper_gen_audio { +# std::unique_ptr pipeline; +# }; +# typedef struct mtmd_helper_gen_audio mtmd_helper_gen_audio; +mtmd_helper_gen_audio_p = NewType("mtmd_helper_gen_audio_p", int) +mtmd_helper_gen_audio_p_ctypes = c_void_p + +# enum mtmd_helper_gen_audio_outtype { +# MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM, // raw PCM +# MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono +# }; +class mtmd_helper_gen_audio_outtype(enum.IntEnum): + MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM = 0 # raw PCM + MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV = 1 # WAV PCM 16-bit LE, mono + +# struct mtmd_helper_gen_audio_inp { +# llama_seq_id seq_id; +# const char * prompt; +# size_t prompt_len; +# mtmd_bitmap * speaker_ref; // optional, can be NULL +# const char * lang; // optional, can be NULL +# int32_t top_k; +# float top_p; +# uint32_t seed; // UINT32_MAX for random (default: random) +# enum mtmd_helper_gen_audio_outtype out_type; +# }; +class mtmd_helper_gen_audio_inp(Structure): + _fields_ = [ + ("seq_id", c_int32), + ("prompt", c_char_p), + ("prompt_len", c_size_t), + ("speaker_ref", mtmd_bitmap_p_ctypes), + ("lang", c_char_p), + ("top_k", c_int32), + ("top_p", c_float), + ("seed", c_uint32), + ("out_type", c_int), + ] + + if TYPE_CHECKING: + seq_id: int + prompt: bytes + prompt_len: int + speaker_ref: mtmd_bitmap_p + lang: Optional[bytes] + top_k: int + top_p: float + seed: int + out_type: int + +# MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( +# struct llama_context * lctx, +# struct mtmd_context * mctx); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_init", + [ + llama_cpp_lib.llama_context_p_ctypes, + mtmd_context_p_ctypes, + ], + mtmd_helper_gen_audio_p_ctypes, +) +def mtmd_helper_gen_audio_init( + lctx: llama_cpp_lib.llama_context_p, + mctx: mtmd_context_p, + /, +) -> mtmd_helper_gen_audio_p: + """ + Initialize the experimental audio generation helper context. + """ + ... + +# MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_free", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_free( + ctx: mtmd_helper_gen_audio_p, + /, +): + ... + +# MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_reset", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_reset( + ctx: mtmd_helper_gen_audio_p, + /, +): + ... + +# MTMD_API int32_t mtmd_helper_gen_audio_set_input( +# mtmd_helper_gen_audio * ctx, +# const struct mtmd_helper_gen_audio_inp * inp); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_set_input", + [ + mtmd_helper_gen_audio_p_ctypes, + POINTER(mtmd_helper_gen_audio_inp), + ], + c_int32, +) +def mtmd_helper_gen_audio_set_input( + ctx: mtmd_helper_gen_audio_p, + inp: POINTER(mtmd_helper_gen_audio_inp), # type: ignore + /, +) -> c_int32: + ... + +# // processes at most n_batch prompt tokens per call +# // returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error +# MTMD_API int32_t mtmd_helper_gen_audio_step_prompt( +# mtmd_helper_gen_audio * ctx, +# int32_t n_batch); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_step_prompt", + [ + mtmd_helper_gen_audio_p_ctypes, + c_int32, + ], + c_int32, +) +def mtmd_helper_gen_audio_step_prompt( + ctx: mtmd_helper_gen_audio_p, + n_batch: c_int32, + /, +) -> c_int32: + """ + Process at most n_batch prompt tokens per call. + Returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error + """ + ... + +# // generates one frame; must only be called after step_prompt() has returned 0 +# // sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token +# // out_stop (optional) is set on end-of-speech, the caller must then stop the loop +# // h_state_out is valid until next step_gen() or reset() call, null if no frame is generated +# MTMD_API int32_t mtmd_helper_gen_audio_step_gen( +# mtmd_helper_gen_audio * ctx, +# llama_token sampled, +# const float * h_state_in, +# const float ** h_state_out, +# bool * out_stop); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_step_gen", + [ + mtmd_helper_gen_audio_p_ctypes, + llama_cpp_lib.llama_token, + POINTER(c_float), + POINTER(POINTER(c_float)), + POINTER(c_bool), + ], + c_int32, +) +def mtmd_helper_gen_audio_step_gen( + ctx: mtmd_helper_gen_audio_p, + sampled: llama_cpp_lib.llama_token, + h_state_in: POINTER(c_float), # type: ignore + h_state_out: POINTER(POINTER(c_float)), # type: ignore + out_stop: POINTER(c_bool), # type: ignore + /, +) -> c_int32: + """ + Generate one audio frame. + + Must only be called after mtmd_helper_gen_audio_step_prompt() returns 0. + + sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token. + + out_stop (optional) is set on end-of-speech, the caller must then stop the loop. + + h_state_out is owned by the helper context and remains valid until the + next step_gen() or reset() call, null if no frame is generated. + """ + ... + +# // out_data valid until next get_output() or reset() call +# // out_n_samples (optional, can be NULL) receives the number of generated PCM samples +# MTMD_API int32_t mtmd_helper_gen_audio_get_output( +# mtmd_helper_gen_audio * ctx, +# int32_t * out_sample_rate, +# const char ** out_data, +# size_t * out_data_len, +# int64_t * out_n_samples); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_get_output", + [ + mtmd_helper_gen_audio_p_ctypes, + POINTER(c_int32), + POINTER(c_char_p), + POINTER(c_size_t), + POINTER(c_int64), + ], + c_int32, +) +def mtmd_helper_gen_audio_get_output( + ctx: mtmd_helper_gen_audio_p, + out_sample_rate: POINTER(c_int32), # type: ignore + out_data: POINTER(c_char_p), # type: ignore + out_data_len: POINTER(c_size_t), # type: ignore + out_n_samples: POINTER(c_int64), # type: ignore + /, +) -> c_int32: + """ + Get accumulated generated audio output. + + out_data is owned by the helper context and remains valid until the next + get_output() or reset() call. + + out_n_samples (optional, can be NULL) receives the number of generated PCM samples. + """ + ... From 9acda8b4b35482d9b2dac9e191bbb9880ddf094e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 15 Aug 2026 08:23:17 +0800 Subject: [PATCH 301/304] Update Submodule vendor/llama.cpp 9c5531e..ad1de39 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 9c5531e2bf..ad1de39e07 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 9c5531e2bf2c95e86eeac807d7109de29266ca7b +Subproject commit ad1de39e0708e3ced9c71bb3c82d93a2c046a73f From 4854c7d305650b6bc9cf2dc805931a5bf2e40dd0 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 16 Aug 2026 00:10:26 +0800 Subject: [PATCH 302/304] Bump version to 0.3.47 - Version 0.3.47 is a small iterative update focused on keeping the Python bindings synchronized with recent llama.cpp changes, especially around MTMD audio generation, sampler backends, and model state management. Signed-off-by: JamePeng --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d5800c04..fe70cdd6ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.47] Multi-Output Sampling, Pocket TTS and audio helper API Bindings, and Llama State Reset Improvements + +- feat(mtmd): sync Pocket TTS and audio helper API bindings + - add Pocket TTS audio generation types and fields + - update generated-audio structures for the latest MTMD ABI + - expose default generation parameters and audio helper APIs + - add multimodal chat capability detection + +- fix(llama): fully clear model state on reset + - Clear native context memory and invalidate hybrid checkpoints to keep + Python state synchronized across standard, recurrent, and hybrid models. + +- fix(internals): disable new backend hooks for custom samplers + - Explicitly set backend_reset and copy_state to NULL + - Clarify CPU callback behavior for CustomSampler + - Document inherited backend behavior in ReasoningBudgetSampler + +- feat(llama): add multi-output backend sampler API support + - expose per-sequence output limits in context parameters + - sync sampler reset and state-copy interfaces with llama.cpp + - preserve advanced context settings when reconstructing Llama instances + - document ordered multi-output sampling behavior + - fix(types): use size_t for sampler count + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/ad1de39e0708e3ced9c71bb3c82d93a2c046a73f](https://github.com/ggml-org/llama.cpp/commit/ad1de39e0708e3ced9c71bb3c82d93a2c046a73f) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260813 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/81190b03f6d177988112dad5fc919491a77705d1...9acda8b4b35482d9b2dac9e191bbb9880ddf094e + ## [0.3.46] Extended Model APIs, MTMD Binding Updates, and Improved Runtime Compatibility - feat(mtmd): update bindings for audio generation and chunk serialization diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index d3fec3b867..89e056542c 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.46" +__version__ = "0.3.47" From 46e3036573370751322bd489d23a865303d0481c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 17 Aug 2026 01:08:11 +0800 Subject: [PATCH 303/304] Update Submodule vendor/llama.cpp ad1de39..4df29be Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index ad1de39e07..4df29be4f4 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit ad1de39e0708e3ced9c71bb3c82d93a2c046a73f +Subproject commit 4df29be4f4c3673f428170fda944a5b19f743bb8 From 389aaa67b70be379c5163520b863facd55e1f856 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 18 Aug 2026 09:04:18 +0800 Subject: [PATCH 304/304] Update Submodule vendor/llama.cpp 4df29be..9d77fa1 Signed-off-by: JamePeng --- llama_cpp/mtmd_cpp.py | 11 ++++++++++- vendor/llama.cpp | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index ec99b7ac6a..4d9ca48bff 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -641,6 +641,15 @@ def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p): """ ... +# // similar to mtmd_input_chunk_copy, but returns a placeholder chunk +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk); +@ctypes_function_mtmd("mtmd_input_chunk_get_placeholder", [mtmd_input_chunk_p_ctypes], mtmd_input_chunk_p_ctypes) +def mtmd_input_chunk_get_placeholder(chunk: mtmd_input_chunk_p) -> mtmd_input_chunk_p: + """ + similar to mtmd_input_chunk_copy, but returns a placeholder chunk + """ + ... + # // save/load an input chunk to/from a buffer (useful for KV save/load) # // important: only chunk's metadata will be saved, the actual image/audio data will not be saved # // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() @@ -1306,7 +1315,7 @@ def mtmd_helper_bitmap_init_from_file( # // note: # // - for now, video input is only supported via C++ helper functions # // - audio files will be auto-detected based on magic bytes -# // - output bitmap will have FNV hash as the ID +# // - output bitmap will have SHA-256 hash (hex string) as the ID # // returns nullptr on failure # // this function is thread-safe # MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 4df29be4f4..9d77fa1725 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 4df29be4f4c3673f428170fda944a5b19f743bb8 +Subproject commit 9d77fa17254e1dee4b9e92504c91611a60b1359f