From 866bed9565c1be3fd3c2985d71a62c05c9f1eea3 Mon Sep 17 00:00:00 2001 From: Emptyngton <40150265+emptyngton@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:56:43 -0400 Subject: [PATCH 01/29] fix(loader): guard HIP_PATH and VULKAN_SDK dirs with os.path.exists os.add_dll_directory() raises FileNotFoundError [WinError 3] when the directory does not exist, so a stale HIP_PATH or VULKAN_SDK left behind by an uninstalled SDK makes "import llama_cpp" fail outright on Windows. The CUDA_PATH branch above already guards each candidate directory with os.path.exists(); this applies the same pattern to the HIP and Vulkan branches. Valid directories are still added individually, so a partially removed SDK contributes whichever of bin/lib remain instead of raising. --- llama_cpp/_ctypes_extensions.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/llama_cpp/_ctypes_extensions.py b/llama_cpp/_ctypes_extensions.py index a9a2c02e5..363472068 100644 --- a/llama_cpp/_ctypes_extensions.py +++ b/llama_cpp/_ctypes_extensions.py @@ -118,13 +118,19 @@ def load_shared_library(lib_base_name: str, base_paths: Union[pathlib.Path, list # Add HIP runtime DLL directories when HIP backend is available. if "HIP_PATH" in os.environ: - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "bin")) - os.add_dll_directory(os.path.join(os.environ["HIP_PATH"], "lib")) + hip_path = os.environ["HIP_PATH"] + for sub_dir in ["bin", "lib"]: + full_path = os.path.join(hip_path, sub_dir) + if os.path.exists(full_path): + os.add_dll_directory(full_path) # Add Vulkan SDK DLL directories when Vulkan backend is enabled. if "VULKAN_SDK" in os.environ: - os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Bin")) - os.add_dll_directory(os.path.join(os.environ["VULKAN_SDK"], "Lib")) + vulkan_sdk = os.environ["VULKAN_SDK"] + for sub_dir in ["Bin", "Lib"]: + full_path = os.path.join(vulkan_sdk, sub_dir) + if os.path.exists(full_path): + os.add_dll_directory(full_path) # Add package-provided library directories. # From 8e1ea5ef1b91a88bb26e9bc809b8aa645785c479 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 27 Jul 2026 22:43:32 +0800 Subject: [PATCH 02/29] Update Submodule vendor/llama.cpp 8bb9093..b77d646 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 8bb909374..b77d64675 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 8bb909374d04d40621340aee5ba2245860027fdc +Subproject commit b77d646751d01c0962bc203b6809e9d94f7d50b7 From 194dfb29e9dc504f942949328a52c7d4f372d445 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 28 Jul 2026 00:17:11 +0800 Subject: [PATCH 03/29] fix(ctypes): support GCC/Clang mangled symbols for optional llama_ext APIs - Add missing `_Z` Itanium C++ ABI symbol variants to ctypes function lookup lists. This improves compatibility with Linux and macOS builds where C++ symbols are exported using GCC/Clang name mangling. Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2f402cd74..317a43548 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -5092,6 +5092,7 @@ def llama_opt_epoch( "llama_graph_reserve", "?llama_graph_reserve@@YAPEAUggml_cgraph@@PEAUllama_context@@III@Z", "__Z19llama_graph_reserveP13llama_contextjjj", + "_Z19llama_graph_reserveP13llama_contextjjj", ], [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ctypes.c_uint32], ctypes.POINTER(ggml_cgraph), @@ -5115,6 +5116,7 @@ def llama_graph_reserve( "llama_ftype_get_default_type", "?llama_ftype_get_default_type@@YA?AW4ggml_type@@W4llama_ftype@@@Z", "__Z28llama_ftype_get_default_type11llama_ftype", + "_Z28llama_ftype_get_default_type11llama_ftype", ], [ctypes.c_int], int, @@ -5134,6 +5136,7 @@ def llama_ftype_get_default_type( "llama_model_n_expert", "?llama_model_n_expert@@YAHPEBUllama_model@@@Z", "__Z20llama_model_n_expertPK11llama_model", + "_Z20llama_model_n_expertPK11llama_model", ], [llama_model_p_ctypes], ctypes.c_int32, @@ -5150,6 +5153,7 @@ def llama_model_n_expert( "llama_model_n_devices", "?llama_model_n_devices@@YAHPEBUllama_model@@@Z", "__Z21llama_model_n_devicesPK11llama_model", + "_Z21llama_model_n_devicesPK11llama_model", ], [llama_model_p_ctypes], ctypes.c_int32, @@ -5166,6 +5170,7 @@ def llama_model_n_devices( "llama_model_get_device", "?llama_model_get_device@@YAPEAUggml_backend_device@@PEBUllama_model@@H@Z", "__Z22llama_model_get_devicePK11llama_modeli", + "_Z22llama_model_get_devicePK11llama_modeli", ], [llama_model_p_ctypes, ctypes.c_int], ctypes.c_void_p, @@ -5186,6 +5191,7 @@ def llama_model_get_device( "llama_set_embeddings_nextn", "?llama_set_embeddings_nextn@@YAXPEAUllama_context@@_N1@Z", "__Z26llama_set_embeddings_nextnP13llama_contextbb", + "_Z26llama_set_embeddings_nextnP13llama_contextbb", ], [llama_context_p_ctypes, ctypes.c_bool, ctypes.c_bool], None, @@ -5212,6 +5218,7 @@ def llama_set_embeddings_nextn( "llama_set_nextn_layer_offset", "?llama_set_nextn_layer_offset@@YAXPEAUllama_context@@H@Z", "__Z28llama_set_nextn_layer_offsetP13llama_contexti", + "_Z28llama_set_nextn_layer_offsetP13llama_contexti", ], [llama_context_p_ctypes, ctypes.c_int32], None, @@ -5236,6 +5243,7 @@ def llama_set_nextn_layer_offset( "llama_get_embeddings_nextn", "?llama_get_embeddings_nextn@@YAPEAMPEAUllama_context@@@Z", "__Z26llama_get_embeddings_nextnP13llama_context", + "_Z26llama_get_embeddings_nextnP13llama_context", ], [llama_context_p_ctypes], ctypes.POINTER(ctypes.c_float), @@ -5253,6 +5261,7 @@ def llama_get_embeddings_nextn( "llama_get_embeddings_nextn_ith", "?llama_get_embeddings_nextn_ith@@YAPEAMPEAUllama_context@@H@Z", "__Z30llama_get_embeddings_nextn_ithP13llama_contexti", + "_Z30llama_get_embeddings_nextn_ithP13llama_contexti", ], [llama_context_p_ctypes, ctypes.c_int32], ctypes.POINTER(ctypes.c_float), @@ -5271,6 +5280,7 @@ def llama_get_embeddings_nextn_ith( "llama_set_embeddings_layer_inp", "?llama_set_embeddings_layer_inp@@YAXPEAUllama_context@@I_N@Z", "__Z30llama_set_embeddings_layer_inpP13llama_contextjb", + "_Z30llama_set_embeddings_layer_inpP13llama_contextjb", ], [llama_context_p_ctypes, ctypes.c_int32, ctypes.c_bool], ctypes.POINTER(ctypes.c_float), @@ -5294,6 +5304,7 @@ def llama_set_embeddings_layer_inp( "llama_get_embeddings_layer_inp", "?llama_get_embeddings_layer_inp@@YAPEAMPEAUllama_context@@I@Z", "__Z30llama_get_embeddings_layer_inpP13llama_contextj", + "_Z30llama_get_embeddings_layer_inpP13llama_contextj", ], [llama_context_p_ctypes, ctypes.c_int32], ctypes.POINTER(ctypes.c_float), @@ -5311,6 +5322,7 @@ def llama_get_embeddings_layer_inp( "llama_get_ctx_other", "?llama_get_ctx_other@@YAPEAUllama_context@@PEAU1@@Z", "__Z19llama_get_ctx_otherP13llama_context", + "_Z19llama_get_ctx_otherP13llama_context", ], [llama_context_p_ctypes], llama_context_p_ctypes, @@ -5330,6 +5342,7 @@ def llama_get_ctx_other( "llama_model_target_layer_ids", "?llama_model_target_layer_ids@@YAPEBHPEBUllama_model@@@Z", "__Z28llama_model_target_layer_idsPK11llama_model", + "_Z28llama_model_target_layer_idsPK11llama_model", ], [llama_model_p_ctypes], ctypes.POINTER(ctypes.c_int32), @@ -5349,7 +5362,8 @@ def llama_model_target_layer_ids( [ "llama_model_target_layer_ids_n", "?llama_model_target_layer_ids_n@@YAIPEBUllama_model@@@Z", - "__Z30llama_model_target_layer_ids_nPK11llama_model" + "__Z30llama_model_target_layer_ids_nPK11llama_model", + "_Z30llama_model_target_layer_ids_nPK11llama_model", ], [llama_model_p_ctypes], ctypes.POINTER(ctypes.c_uint32), From 1e49f9da22b2125ed9d032ad88a9fd9afa8e4f2c Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 28 Jul 2026 04:04:19 +0800 Subject: [PATCH 04/29] feat(tools): add cross-platform ABI inspection utility Inspect PE, ELF, and Mach-O exports and normalize platform-specific symbol names. Validate optional llama_ext ctypes aliases across Windows, Linux, and macOS builds. Keep artifacts and timestamped privacy-safe reports local to the repository. Signed-off-by: JamePeng --- .gitignore | 11 +- tools/abi/README.md | 164 ++++ tools/abi/__init__.py | 3 + tools/abi/__main__.py | 5 + tools/abi/artifacts/.gitignore | 4 + tools/abi/artifacts/README.md | 20 + tools/abi/output/.gitignore | 4 + tools/abi/output/README.md | 7 + tools/abi/scan_dynamic.py | 863 +++++++++++++++++++++ tools/abi/tests/test_platform_artifacts.py | 85 ++ tools/abi/tests/test_scan_dynamic.py | 194 +++++ 11 files changed, 1359 insertions(+), 1 deletion(-) create mode 100644 tools/abi/README.md create mode 100644 tools/abi/__init__.py create mode 100644 tools/abi/__main__.py create mode 100644 tools/abi/artifacts/.gitignore create mode 100644 tools/abi/artifacts/README.md create mode 100644 tools/abi/output/.gitignore create mode 100644 tools/abi/output/README.md create mode 100644 tools/abi/scan_dynamic.py create mode 100644 tools/abi/tests/test_platform_artifacts.py create mode 100644 tools/abi/tests/test_scan_dynamic.py diff --git a/.gitignore b/.gitignore index fad7f4331..b5d60bf89 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,15 @@ local_settings.py models/ docker/open_llama/*.bin +# Repository-only ABI tool inputs and generated reports. +# Keep only the directory instructions and local ignore rules tracked. +/tools/abi/artifacts/* +!/tools/abi/artifacts/.gitignore +!/tools/abi/artifacts/README.md +/tools/abi/output/* +!/tools/abi/output/.gitignore +!/tools/abi/output/README.md + # C extensions (llama_cpp bindings) llama_cpp/*.so llama_cpp/*.dylib @@ -208,4 +217,4 @@ docs/_build/ # Installer logs pip-log.txt -pip-delete-this-directory.txt \ No newline at end of file +pip-delete-this-directory.txt diff --git a/tools/abi/README.md b/tools/abi/README.md new file mode 100644 index 000000000..ea509f2f7 --- /dev/null +++ b/tools/abi/README.md @@ -0,0 +1,164 @@ +# Cross-platform ABI inspection + +Author: **JamePeng** + +This repository-only tool inspects PE (`.dll`), ELF (`.so` and `.so.*`), and +Mach-O (`.dylib`) exports. Its primary purpose is to collect ctypes symbol +candidates and verify optional `llama_ext` bindings across MSVC, GCC/Clang, +and macOS builds. + +## Boundary and safety + +The tool is intentionally excluded from wheels: + +```toml +wheel.packages = ["llama_cpp"] +``` + +It is not imported by `llama_cpp`, has no installed command, and keeps LIEF +out of project dependencies. Run it only from a trusted source checkout. +LIEF parses native binaries, so do not scan untrusted artifacts. + +Install the maintainer-only dependency: + +```bash +python -m pip install lief +``` + +The tool and its documentation use the same MIT License as this repository. + +## Artifact layout + +Run commands from the repository root. Put builds under +`tools/abi/artifacts`, or replace that argument with an external absolute +directory: + +```text +tools/abi/artifacts/ +├── windows-x86_64/ +│ └── +├── linux-x86_64/ +│ └── +└── macos-arm64/ + └── +``` + +Names are not significant. `--select-symbol llama_decode` identifies the +llama library by content when dependency and backend libraries share the same +directory. + +Artifacts may come from local builds, an installed or extracted wheel, +[project releases](https://github.com/JamePeng/llama-cpp-python/releases), or +[upstream releases](https://github.com/ggml-org/llama.cpp/releases). Record +the source revision, compiler, architecture, and build options. Upstream +artifacts may not contain fork-only `llama_ext` APIs. + +## Scan exports + +```bash +python -m tools.abi scan tools/abi/artifacts --recursive +``` + +The default output is one same-named JSONL file per library: + +```text +tools/abi/output/ +└── 20260728T153012.123456Z/ + ├── llama.dll.jsonl + ├── libllama.so.jsonl + └── libllama.dylib.jsonl +``` + +Useful options: + +```bash +# Select only binaries that export llama_decode. +python -m tools.abi scan tools/abi/artifacts --recursive \ + --select-symbol llama_decode + +# Print instead of writing per-library JSONL. +python -m tools.abi scan tools/abi/artifacts --recursive --format text + +# Write one aggregate file; its filename receives a UTC timestamp. +python -m tools.abi scan tools/abi/artifacts --recursive \ + --format jsonl --output all-symbols.jsonl +``` + +`--prefix` is optional. By default all exports are retained: + +```bash +python -m tools.abi scan tools/abi/artifacts --recursive \ + --prefix llama_ --prefix ggml_ +``` + +## Check optional llama_ext bindings + +This is the primary ABI validation command: + +```bash +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive \ + --source llama_cpp/llama_cpp.py +``` + +It statically reads ctypes decorators without importing `llama_cpp`. The +default `--scope optional` checks declarations marked `required=False` and +returns exit code 1 if any candidate is missing. Other scopes are available: + +```bash +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive --scope required +python -m tools.abi check-bindings tools/abi/artifacts \ + --recursive --scope all +``` + +## Compare and create a manifest + +```bash +python -m tools.abi compare tools/abi/artifacts \ + --recursive --select-symbol llama_decode + +python -m tools.abi manifest tools/abi/artifacts \ + --recursive --select-symbol llama_decode \ + --output llama-exports.json +``` + +Cross-platform comparison uses `canonical_name`: + +```text +?llama_graph_reserve@@... MSVC +_Z19llama_graph_reserve... Linux Itanium ABI +__Z19llama_graph_reserve... Mach-O symbol table + ↓ +llama_graph_reserve canonical name +``` + +Records retain `raw_name`, ctypes `lookup_name`, `canonical_name`, ABI, +address, ordinal, library filename, format, architecture, SHA-256, and UTC +generation time. They never contain the artifact's absolute source path. + +Every run receives a timestamp, preventing normal output from overwriting +previous results. Generated artifacts and reports are ignored by Git. + +## Verification + +Unit tests are independent from the project's default test suite: + +```bash +python -m pytest tools/abi/tests/test_scan_dynamic.py -q +``` + +The opt-in integration test requires Windows, Linux, and macOS artifacts: + +```powershell +$env:LLAMA_ABI_ARTIFACTS = "tools/abi/artifacts" +python -m pytest tools/abi/tests/test_platform_artifacts.py -q +``` + +```bash +LLAMA_ABI_ARTIFACTS=tools/abi/artifacts \ +python -m pytest tools/abi/tests/test_platform_artifacts.py -q +``` + +Without configured artifacts, integration tests skip. With +`LLAMA_ABI_ARTIFACTS` set, a missing platform or optional ABI alias fails. diff --git a/tools/abi/__init__.py b/tools/abi/__init__.py new file mode 100644 index 000000000..0a387e675 --- /dev/null +++ b/tools/abi/__init__.py @@ -0,0 +1,3 @@ +"""Cross-platform shared-library ABI inspection tools.""" + +__author__ = "JamePeng" diff --git a/tools/abi/__main__.py b/tools/abi/__main__.py new file mode 100644 index 000000000..acfe21acf --- /dev/null +++ b/tools/abi/__main__.py @@ -0,0 +1,5 @@ +from .scan_dynamic import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/abi/artifacts/.gitignore b/tools/abi/artifacts/.gitignore new file mode 100644 index 000000000..dbea0aa37 --- /dev/null +++ b/tools/abi/artifacts/.gitignore @@ -0,0 +1,4 @@ +# Keep downloaded or locally built native libraries out of Git. +* +!.gitignore +!README.md diff --git a/tools/abi/artifacts/README.md b/tools/abi/artifacts/README.md new file mode 100644 index 000000000..30863ea8c --- /dev/null +++ b/tools/abi/artifacts/README.md @@ -0,0 +1,20 @@ +# ABI artifacts + +Maintainer: **JamePeng** + +Place trusted Windows, Linux, and macOS build artifacts here for local ABI +inspection. Filenames do not need to follow a fixed convention. + +```text +tools/abi/artifacts/ +├── windows-x86_64/ +├── linux-x86_64/ +├── macos-arm64/ +└── macos-x86_64/ +``` + +Downloaded and copied content is ignored by both the local and repository +`.gitignore`; only this README and `.gitignore` are tracked. `git add -f` can +still deliberately override ignore rules. + +See `tools/abi/README.md` for commands and artifact provenance requirements. diff --git a/tools/abi/output/.gitignore b/tools/abi/output/.gitignore new file mode 100644 index 000000000..f12e9f061 --- /dev/null +++ b/tools/abi/output/.gitignore @@ -0,0 +1,4 @@ +# Keep generated ABI reports local. +* +!.gitignore +!README.md diff --git a/tools/abi/output/README.md b/tools/abi/output/README.md new file mode 100644 index 000000000..3601ac132 --- /dev/null +++ b/tools/abi/output/README.md @@ -0,0 +1,7 @@ +# Local ABI reports + +Each run is stored in a UTC timestamp directory. Reports omit artifact source +paths but may contain binary hashes and non-public symbols. + +Generated content is ignored by both the local and repository `.gitignore`; +only this README and `.gitignore` are tracked. Review reports before sharing. diff --git a/tools/abi/scan_dynamic.py b/tools/abi/scan_dynamic.py new file mode 100644 index 000000000..8ee11fd8c --- /dev/null +++ b/tools/abi/scan_dynamic.py @@ -0,0 +1,863 @@ +"""Inspect and compare exported symbols in PE, ELF, and Mach-O libraries. + +This repository-only maintainer utility supports collection and verification +of cross-platform ctypes symbol candidates, with particular focus on optional +llama_ext APIs. + +LIEF is imported lazily so that ``--help`` remains available when the optional +dependency is not installed. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import re +import sys +from collections import defaultdict +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Sequence + +LIBRARY_SUFFIXES = {".dll", ".dylib", ".so"} +__author__ = "JamePeng" + + +class ScanError(RuntimeError): + """Raised when a shared library cannot be inspected.""" + + +@dataclass(frozen=True) +class SymbolRecord: + """One exported symbol and its cross-platform names.""" + + raw_name: str + lookup_name: str + canonical_name: str + abi: str + address: str + ordinal: int | None = None + + +@dataclass(frozen=True) +class BindingDeclaration: + """One ctypes decorator declaration extracted without importing llama_cpp.""" + + python_name: str + candidates: tuple[str, ...] + required: bool + line: int + + +@dataclass(frozen=True) +class LibraryScan: + """Metadata and exported symbols for one binary architecture.""" + + library: str + format: str + platform: str + architecture: str + sha256: str + symbols: tuple[SymbolRecord, ...] + + +def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def generation_timestamp() -> str: + """Return a sortable, collision-resistant UTC generation timestamp.""" + + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + + +def _enum_name(value: Any) -> str: + text = str(value) + return text.rsplit(".", 1)[-1] + + +def get_format(binary: Any) -> str: + value = str(binary.format).upper() + if "MACHO" in value: + return "Mach-O" + if "ELF" in value: + return "ELF" + if "PE" in value: + return "PE" + return str(binary.format) + + +def get_platform(binary_format: str) -> str: + return { + "PE": "windows", + "ELF": "linux", + "Mach-O": "darwin", + }.get(binary_format, "unknown") + + +def get_architecture(binary: Any, binary_format: str) -> str: + header = binary.header + if binary_format == "PE": + return _enum_name(header.machine) + if binary_format == "ELF": + return _enum_name(header.machine_type) + if binary_format == "Mach-O": + return _enum_name(header.cpu_type) + return "unknown" + + +def normalize_symbol_name(raw_name: str, binary_format: str) -> str: + """Return the name used by ctypes/dlsym and cross-platform comparison. + + Mach-O symbol tables prefix external C names with an underscore. dlsym and + ctypes callers use the source-level name without that platform prefix. + """ + + if binary_format == "Mach-O" and raw_name.startswith("_"): + return raw_name[1:] + return raw_name + + +def detect_abi(normalized_name: str) -> str: + if normalized_name.startswith("?"): + return "msvc-cxxabi" + if normalized_name.startswith("_Z"): + return "itanium-cxxabi" + return "unmangled" + + +def canonicalize_symbol_name(normalized_name: str, abi: str) -> str: + """Recover a source-level name from simple global C++ mangling. + + llama_ext functions are global functions, so their MSVC and Itanium + spellings can be mapped without a full ABI demangler. Namespaced, + overloaded, and templated symbols remain mangled to avoid false matches. + """ + + if abi == "msvc-cxxabi": + match = re.match(r"^\?([^@?$]+)@@", normalized_name) + if match: + return match.group(1) + + if abi == "itanium-cxxabi": + match = re.match(r"^_Z(\d+)", normalized_name) + if match: + length = int(match.group(1)) + start = match.end() + candidate = normalized_name[start : start + length] + if len(candidate) == length: + return candidate + + return normalized_name + + +def _symbol_address(symbol: Any) -> str: + value = getattr(symbol, "address", None) + if value is None: + value = getattr(symbol, "value", 0) + return hex(int(value)) + + +def _exported_symbols(binary: Any, binary_format: str) -> Iterable[Any]: + if binary_format == "PE": + if not binary.has_exports: + return () + return binary.get_export().entries + + # LIEF's exported_symbols filters undefined ELF imports and non-exported + # Mach-O symbols, unlike dynamic_symbols/symbols. + return binary.exported_symbols + + +def _iter_binaries(parsed: Any) -> list[Any]: + # A universal Mach-O may contain several architecture slices. + if type(parsed).__name__ == "FatBinary": + return list(parsed) + return [parsed] + + +def scan_library( + path: str | Path, +) -> list[LibraryScan]: + """Inspect one library, returning one result per architecture slice.""" + + try: + import lief + except ImportError as exc: + raise ScanError( + "LIEF is required for ABI inspection. Install it with: pip install lief" + ) from exc + + library_path = Path(path).expanduser().resolve() + if not library_path.is_file(): + raise ScanError(f"Not a file: {library_path.name}") + + try: + parsed = lief.parse(str(library_path)) + except Exception as exc: + detail = str(exc).replace(str(library_path), library_path.name) + raise ScanError(f"Failed to parse {library_path.name}: {detail}") from exc + + if parsed is None: + raise ScanError(f"LIEF did not recognize {library_path.name}") + + digest = sha256_file(library_path) + results: list[LibraryScan] = [] + + for binary in _iter_binaries(parsed): + binary_format = get_format(binary) + records: list[SymbolRecord] = [] + + for symbol in _exported_symbols(binary, binary_format): + raw_name = getattr(symbol, "name", None) + if not raw_name: + # PE supports ordinal-only exports. They cannot be matched to + # Python bindings by name, so keep a stable synthetic label. + ordinal = getattr(symbol, "ordinal", None) + if ordinal is None: + continue + raw_name = f"#{ordinal}" + + lookup_name = normalize_symbol_name(raw_name, binary_format) + abi = detect_abi(lookup_name) + canonical_name = canonicalize_symbol_name(lookup_name, abi) + + records.append( + SymbolRecord( + raw_name=raw_name, + lookup_name=lookup_name, + canonical_name=canonical_name, + abi=abi, + address=_symbol_address(symbol), + ordinal=getattr(symbol, "ordinal", None), + ) + ) + + records.sort(key=lambda item: (item.canonical_name, item.raw_name)) + results.append( + LibraryScan( + library=library_path.name, + format=binary_format, + platform=get_platform(binary_format), + architecture=get_architecture(binary, binary_format), + sha256=digest, + symbols=tuple(records), + ) + ) + + return results + + +def select_scans_by_symbols( + scans: Sequence[LibraryScan], + required_symbols: Sequence[str], +) -> list[LibraryScan]: + """Select binaries by exported canonical names, independent of filenames.""" + + if not required_symbols: + return list(scans) + selected = [] + for scan in scans: + exported = {symbol.canonical_name for symbol in scan.symbols} + if all(name in exported for name in required_symbols): + selected.append(scan) + return selected + + +def filter_scan_symbols( + scans: Sequence[LibraryScan], + prefixes: Sequence[str], +) -> list[LibraryScan]: + if not prefixes: + return list(scans) + return [ + replace( + scan, + symbols=tuple( + symbol + for symbol in scan.symbols + if any(symbol.canonical_name.startswith(prefix) for prefix in prefixes) + ), + ) + for scan in scans + ] + + +def extract_ctypes_bindings(source: str | Path) -> list[BindingDeclaration]: + """Extract literal ctypes decorator candidates without importing the module.""" + + source_path = Path(source) + try: + tree = ast.parse( + source_path.read_text(encoding="utf-8"), + filename=str(source_path), + ) + except (OSError, SyntaxError) as exc: + raise ScanError(f"Failed to parse binding source {source_path}: {exc}") from exc + + declarations: list[BindingDeclaration] = [] + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Call) or not decorator.args: + continue + decorator_name = "" + if isinstance(decorator.func, ast.Name): + decorator_name = decorator.func.id + elif isinstance(decorator.func, ast.Attribute): + decorator_name = decorator.func.attr + if not decorator_name.startswith("ctypes_function"): + continue + + try: + names = ast.literal_eval(decorator.args[0]) + except (ValueError, TypeError): + continue + if isinstance(names, str): + candidates = (names,) + elif isinstance(names, (list, tuple)) and all( + isinstance(name, str) for name in names + ): + candidates = tuple(names) + else: + continue + + required = True + for keyword in decorator.keywords: + if keyword.arg == "required": + try: + required = bool(ast.literal_eval(keyword.value)) + except (ValueError, TypeError): + pass + + declarations.append( + BindingDeclaration( + python_name=node.name, + candidates=candidates, + required=required, + line=node.lineno, + ) + ) + + return sorted(declarations, key=lambda item: item.line) + + +def check_bindings( + scan: LibraryScan, + declarations: Sequence[BindingDeclaration], +) -> dict[str, Any]: + """Check which ctypes candidate would be selected for one library.""" + + exported = {symbol.lookup_name for symbol in scan.symbols} + available = [] + missing_required = [] + missing_optional = [] + + for declaration in declarations: + selected = next( + (name for name in declaration.candidates if name in exported), + None, + ) + item = { + "python_name": declaration.python_name, + "required": declaration.required, + "line": declaration.line, + "candidates": list(declaration.candidates), + "selected": selected, + } + if selected is not None: + available.append(item) + elif declaration.required: + missing_required.append(item) + else: + missing_optional.append(item) + + return { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "declaration_count": len(declarations), + "available_count": len(available), + "available": available, + "missing_required": missing_required, + "missing_optional": missing_optional, + } + + +def compare_scans(scans: Sequence[LibraryScan]) -> dict[str, Any]: + if len(scans) < 2: + raise ValueError("At least two library scans are required for comparison") + + symbol_sets = [{symbol.canonical_name for symbol in scan.symbols} for scan in scans] + common = set.intersection(*symbol_sets) + libraries = [] + + for index, scan in enumerate(scans): + others = set.union(*(symbol_sets[i] for i in range(len(scans)) if i != index)) + libraries.append( + { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "symbol_count": len(symbol_sets[index]), + "only_here": sorted(symbol_sets[index] - others), + "missing_here": sorted(others - symbol_sets[index]), + } + ) + + return { + "common_count": len(common), + "common": sorted(common), + "libraries": libraries, + } + + +def build_manifest( + scans: Sequence[LibraryScan], + *, + generated_at: str | None = None, +) -> dict[str, Any]: + generated_at = generated_at or generation_timestamp() + symbols: dict[str, list[dict[str, Any]]] = defaultdict(list) + libraries = [] + + for scan in scans: + libraries.append(_scan_metadata(scan)) + for symbol in scan.symbols: + symbols[symbol.canonical_name].append( + { + "library": scan.library, + "platform": scan.platform, + "architecture": scan.architecture, + "raw_name": symbol.raw_name, + "lookup_name": symbol.lookup_name, + "abi": symbol.abi, + "address": symbol.address, + "ordinal": symbol.ordinal, + } + ) + + return { + "schema_version": 1, + "generated_at": generated_at, + "libraries": libraries, + "symbols": dict(sorted(symbols.items())), + } + + +def collect_library_paths( + inputs: Sequence[str], + *, + recursive: bool = False, +) -> list[Path]: + def is_shared_library(path: Path) -> bool: + name = path.name.lower() + return path.suffix.lower() in LIBRARY_SUFFIXES or ".so." in name + + paths: list[Path] = [] + for value in inputs: + path = Path(value).expanduser() + if path.is_dir(): + candidates = path.rglob("*") if recursive else path.iterdir() + paths.extend( + candidate + for candidate in candidates + if candidate.is_file() and is_shared_library(candidate) + ) + else: + paths.append(path) + return sorted(set(paths), key=lambda item: str(item).lower()) + + +def _scan_paths( + paths: Sequence[Path], +) -> tuple[list[LibraryScan], list[str]]: + scans: list[LibraryScan] = [] + errors: list[str] = [] + for path in paths: + try: + scans.extend(scan_library(path)) + except ScanError as exc: + errors.append(str(exc)) + except Exception as exc: + detail = str(exc).replace(str(path.resolve()), path.name) + errors.append(f"{path.name}: {detail}") + return scans, errors + + +def _timestamped_output_path(output: str | Path, timestamp: str) -> Path: + path = Path(output) + return path.with_name(f"{path.stem}.{timestamp}{path.suffix}") + + +def _write_output( + text: str, + output: str | None, + *, + timestamp: str, +) -> None: + if output: + output_path = _timestamped_output_path(output, timestamp) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(text + "\n", encoding="utf-8") + print(f"saved: {output_path}") + else: + print(text) + + +def _scan_metadata(scan: LibraryScan) -> dict[str, Any]: + return {key: value for key, value in asdict(scan).items() if key != "symbols"} + + +def _jsonl_rows(scan: LibraryScan, generated_at: str) -> list[str]: + metadata = _scan_metadata(scan) + return [ + json.dumps( + { + "generated_at": generated_at, + **metadata, + **asdict(symbol), + }, + ensure_ascii=False, + ) + for symbol in scan.symbols + ] + + +def write_library_jsonl( + scans: Sequence[LibraryScan], + output_dir: str | Path = "tools/abi/output", + *, + timestamp: str | None = None, +) -> list[Path]: + """Write one same-named JSONL per library under a timestamped run directory.""" + + timestamp = timestamp or generation_timestamp() + destination = Path(output_dir) / timestamp + destination.mkdir(parents=True, exist_ok=True) + grouped: dict[str, list[LibraryScan]] = defaultdict(list) + for scan in scans: + grouped[scan.library].append(scan) + + written = [] + for library, library_scans in sorted(grouped.items()): + output_path = destination / f"{library}.jsonl" + rows = [ + row + for library_scan in library_scans + for row in _jsonl_rows(library_scan, timestamp) + ] + output_path.write_text( + "\n".join(rows) + ("\n" if rows else ""), + encoding="utf-8", + ) + written.append(output_path) + return written + + +def _scan_text(scans: Sequence[LibraryScan], errors: Sequence[str]) -> str: + lines: list[str] = [] + for scan in scans: + lines.append( + f"{scan.library} [{scan.format}/{scan.architecture}]: " + f"{len(scan.symbols)} exported symbol(s)" + ) + for symbol in scan.symbols: + raw_suffix = ( + f" (raw: {symbol.raw_name})" + if symbol.raw_name != symbol.canonical_name + else "" + ) + lines.append( + f" {symbol.canonical_name} [{symbol.abi}]" + f" @ {symbol.address}{raw_suffix}" + ) + for error in errors: + lines.append(f"ERROR: {error}") + return "\n".join(lines) + + +def _compare_text(comparison: dict[str, Any]) -> str: + lines = [f"Common canonical symbols: {comparison['common_count']}"] + for library in comparison["libraries"]: + lines.extend( + [ + "", + ( + f"{library['library']} " + f"[{library['platform']}/{library['architecture']}]: " + f"{library['symbol_count']} symbol(s)" + ), + f" Only here: {len(library['only_here'])}", + ] + ) + lines.extend(f" {name}" for name in library["only_here"]) + lines.append(f" Missing here: {len(library['missing_here'])}") + lines.extend(f" {name}" for name in library["missing_here"]) + return "\n".join(lines) + + +def _bindings_text( + results: Sequence[dict[str, Any]], + scope: str, +) -> str: + lines: list[str] = [] + for result in results: + lines.append( + f"{result['library']} " + f"[{result['platform']}/{result['architecture']}]: " + f"{result['available_count']}/{result['declaration_count']} " + "binding(s) available in selected scope" + ) + if scope in {"required", "all"}: + lines.append(f" Missing required: {len(result['missing_required'])}") + lines.extend( + f" {item['python_name']} (line {item['line']})" + for item in result["missing_required"] + ) + if scope in {"optional", "all"}: + lines.append(f" Missing optional: {len(result['missing_optional'])}") + lines.extend( + f" {item['python_name']} (line {item['line']})" + for item in result["missing_optional"] + ) + return "\n".join(lines) + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Inspect and compare PE, ELF, and Mach-O exported symbols." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + def add_common(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument("paths", nargs="+", help="Library files or directories") + subparser.add_argument( + "--prefix", + action="append", + default=[], + help=( + "Optional canonical-name filter; may be repeated " + "(default: keep all exports)" + ), + ) + subparser.add_argument( + "--select-symbol", + action="append", + default=[], + help=( + "Select libraries exporting this canonical symbol; may be " + "repeated and does not depend on the library filename" + ), + ) + subparser.add_argument( + "--recursive", + action="store_true", + help="Recursively search directory inputs", + ) + subparser.add_argument("-o", "--output", help="Write output to this file") + + scan_parser = subparsers.add_parser("scan", help="List exported symbols") + add_common(scan_parser) + scan_parser.add_argument( + "--format", + choices=("text", "json", "jsonl"), + default="jsonl", + help="Output format", + ) + scan_parser.add_argument( + "--output-dir", + default="tools/abi/output", + help=( + "Directory for default per-library JSONL files " + "(default: tools/abi/output)" + ), + ) + + compare_parser = subparsers.add_parser( + "compare", help="Compare canonical symbol names across libraries" + ) + add_common(compare_parser) + compare_parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format", + ) + + manifest_parser = subparsers.add_parser( + "manifest", help="Create a cross-platform symbol manifest" + ) + add_common(manifest_parser) + + bindings_parser = subparsers.add_parser( + "check-bindings", + help="Check literal ctypes decorator candidates against libraries", + ) + add_common(bindings_parser) + bindings_parser.add_argument( + "--source", + default="llama_cpp/llama_cpp.py", + help="Python binding source to inspect without importing it", + ) + bindings_parser.add_argument( + "--format", + choices=("text", "json"), + default="text", + help="Output format", + ) + bindings_parser.add_argument( + "--scope", + choices=("optional", "required", "all"), + default="optional", + help=("Binding declarations to check " "(default: optional llama_ext APIs)"), + ) + + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = create_parser().parse_args(argv) + paths = collect_library_paths(args.paths, recursive=args.recursive) + if not paths: + print("No shared libraries found.", file=sys.stderr) + return 2 + + # Scan all exports first. Selection must not depend on --prefix, because a + # caller may use an anchor outside the displayed prefix set. + scans, errors = _scan_paths(paths) + if not scans: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + if args.select_symbol: + scans = select_scans_by_symbols(scans, args.select_symbol) + if not scans: + print( + "No library exports all requested selection symbols: " + + ", ".join(args.select_symbol), + file=sys.stderr, + ) + return 2 + + scans = filter_scan_symbols(scans, args.prefix) + if args.command in {"compare", "manifest"} and args.prefix: + # A package lib directory normally contains ggml and accelerator + # backends. Empty prefix matches are not comparison targets. + scans = [scan for scan in scans if scan.symbols] + timestamp = generation_timestamp() + + validation_failed = False + + if args.command == "scan": + if args.format == "text": + output = _scan_text(scans, errors) + elif args.format == "json": + output = json.dumps( + { + "generated_at": timestamp, + "libraries": [asdict(scan) for scan in scans], + "errors": errors, + }, + ensure_ascii=False, + indent=2, + ) + else: + output = "\n".join( + row for scan in scans for row in _jsonl_rows(scan, timestamp) + ) + + if args.format == "jsonl" and args.output is None: + try: + written = write_library_jsonl( + scans, + args.output_dir, + timestamp=timestamp, + ) + except OSError as exc: + print(f"ERROR: failed to write JSONL output: {exc}", file=sys.stderr) + return 1 + for path in written: + print(f"saved: {path}") + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors else 0 + elif args.command == "compare": + if len(scans) < 2: + print("Comparison requires at least two libraries.", file=sys.stderr) + return 2 + comparison = compare_scans(scans) + output = ( + _compare_text(comparison) + if args.format == "text" + else json.dumps(comparison, ensure_ascii=False, indent=2) + ) + elif args.command == "manifest": + output = json.dumps( + build_manifest(scans, generated_at=timestamp), + ensure_ascii=False, + indent=2, + ) + else: + try: + declarations = extract_ctypes_bindings(args.source) + except ScanError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + if args.scope == "optional": + declarations = [ + declaration for declaration in declarations if not declaration.required + ] + elif args.scope == "required": + declarations = [ + declaration for declaration in declarations if declaration.required + ] + if not declarations: + print( + f"No {args.scope} ctypes binding declarations found in " + f"{args.source}.", + file=sys.stderr, + ) + return 2 + binding_results = [check_bindings(scan, declarations) for scan in scans] + if not args.select_symbol and binding_results: + # A package directory may contain arbitrarily named dependency and + # backend libraries. The library ctypes would want is the one with + # the greatest declaration coverage, regardless of filename. + best_count = max(result["available_count"] for result in binding_results) + binding_results = [ + result + for result in binding_results + if result["available_count"] == best_count + ] + output = ( + _bindings_text(binding_results, args.scope) + if args.format == "text" + else json.dumps(binding_results, ensure_ascii=False, indent=2) + ) + validation_failed = any( + result["missing_required"] or result["missing_optional"] + for result in binding_results + ) + + try: + _write_output(output, args.output, timestamp=timestamp) + except OSError as exc: + print(f"ERROR: failed to write output: {exc}", file=sys.stderr) + return 1 + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 if errors or validation_failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/abi/tests/test_platform_artifacts.py b/tools/abi/tests/test_platform_artifacts.py new file mode 100644 index 000000000..b5f88dcc3 --- /dev/null +++ b/tools/abi/tests/test_platform_artifacts.py @@ -0,0 +1,85 @@ +"""Opt-in integration tests for real Windows, Linux, and macOS artifacts.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from tools.abi.scan_dynamic import ( + check_bindings, + collect_library_paths, + extract_ctypes_bindings, + scan_library, + select_scans_by_symbols, +) + +ARTIFACTS_ENV = "LLAMA_ABI_ARTIFACTS" +DEFAULT_ARTIFACTS = Path("tools/abi/artifacts") +REQUIRED_PLATFORMS = {"windows", "linux", "darwin"} +STABLE_LLAMA_SYMBOLS = { + "llama_decode", + "llama_model_load_from_file", +} +BINDING_SOURCE = Path("llama_cpp/llama_cpp.py") + + +@pytest.fixture(scope="module") +def platform_scans(): + configured = os.environ.get(ARTIFACTS_ENV) + artifacts = Path(configured) if configured else DEFAULT_ARTIFACTS + paths = collect_library_paths([str(artifacts)], recursive=True) + + if not paths and not configured: + pytest.skip( + "No ABI artifacts installed. Set LLAMA_ABI_ARTIFACTS to run " + "the Windows/Linux/macOS integration test." + ) + + assert paths, f"No shared libraries found under {artifacts}" + scans = [scan for path in paths for scan in scan_library(path)] + scans = select_scans_by_symbols(scans, ["llama_decode"]) + by_platform = {scan.platform: scan for scan in scans} + assert REQUIRED_PLATFORMS <= set(by_platform), ( + "The ABI artifact set must contain llama libraries for Windows, " + f"Linux, and macOS. Found: {sorted(by_platform)}" + ) + return by_platform + + +def test_windows_linux_and_macos_llama_exports(platform_scans): + common = set.intersection( + *( + {symbol.canonical_name for symbol in platform_scans[platform].symbols} + for platform in sorted(REQUIRED_PLATFORMS) + ) + ) + assert STABLE_LLAMA_SYMBOLS <= common + + +def test_macos_macho_lookup_name_removes_symbol_table_prefix(platform_scans): + decode = next( + symbol + for symbol in platform_scans["darwin"].symbols + if symbol.canonical_name == "llama_decode" + ) + assert decode.raw_name == "_llama_decode" + assert decode.lookup_name == "llama_decode" + + +def test_optional_llama_ext_abi_aliases_on_all_platforms(platform_scans): + optional = [ + declaration + for declaration in extract_ctypes_bindings(BINDING_SOURCE) + if not declaration.required + ] + assert optional, "No optional llama_ext ctypes bindings were found" + + for platform in sorted(REQUIRED_PLATFORMS): + result = check_bindings(platform_scans[platform], optional) + assert ( + result["missing_optional"] == [] + ), f"{platform} is missing optional llama_ext ABI aliases: " + ", ".join( + item["python_name"] for item in result["missing_optional"] + ) diff --git a/tools/abi/tests/test_scan_dynamic.py b/tools/abi/tests/test_scan_dynamic.py new file mode 100644 index 000000000..ebcf75fd4 --- /dev/null +++ b/tools/abi/tests/test_scan_dynamic.py @@ -0,0 +1,194 @@ +import json + +import tools.abi.scan_dynamic as abi_tool + +from tools.abi.scan_dynamic import ( + BindingDeclaration, + SymbolRecord, + LibraryScan, + canonicalize_symbol_name, + check_bindings, + collect_library_paths, + compare_scans, + detect_abi, + extract_ctypes_bindings, + normalize_symbol_name, + select_scans_by_symbols, + write_library_jsonl, +) + + +def _scan(library: str, platform: str, names: list[str]) -> LibraryScan: + records = tuple( + SymbolRecord( + raw_name=name, + lookup_name=name, + canonical_name=name, + abi="unmangled", + address="0x0", + ) + for name in names + ) + return LibraryScan( + library=library, + format="test", + platform=platform, + architecture="test", + sha256="test", + symbols=records, + ) + + +def test_normalizes_macho_external_prefix(): + assert normalize_symbol_name("_llama_decode", "Mach-O") == "llama_decode" + assert normalize_symbol_name("__ZN5llama", "Mach-O") == "_ZN5llama" + assert normalize_symbol_name("llama_decode", "ELF") == "llama_decode" + assert normalize_symbol_name("llama_decode", "PE") == "llama_decode" + + +def test_detects_abi_after_platform_normalization(): + assert detect_abi("?function@@YAXXZ") == "msvc-cxxabi" + assert detect_abi("_ZN5llama") == "itanium-cxxabi" + assert detect_abi("llama_decode") == "unmangled" + + +def test_canonicalizes_simple_global_cpp_names(): + assert ( + canonicalize_symbol_name( + "?llama_graph_reserve@@YAXXZ", + "msvc-cxxabi", + ) + == "llama_graph_reserve" + ) + assert ( + canonicalize_symbol_name( + "_Z19llama_graph_reserveP13llama_contextjjj", + "itanium-cxxabi", + ) + == "llama_graph_reserve" + ) + nested = "_ZN5llama6detail3fooEv" + assert canonicalize_symbol_name(nested, "itanium-cxxabi") == nested + + +def test_compares_canonical_names(): + comparison = compare_scans( + [ + _scan("libllama.so", "linux", ["llama_decode"]), + _scan( + "llama.dll", + "windows", + ["llama_decode", "llama_windows_only"], + ), + ] + ) + + assert comparison["common"] == ["llama_decode"] + assert comparison["libraries"][0]["missing_here"] == ["llama_windows_only"] + assert comparison["libraries"][1]["only_here"] == ["llama_windows_only"] + + +def test_collects_versioned_elf_library(tmp_path): + library = tmp_path / "libllama.so.1" + library.touch() + + assert collect_library_paths([str(tmp_path)]) == [library] + + +def test_extracts_and_checks_literal_binding_aliases(tmp_path): + source = tmp_path / "bindings.py" + source.write_text( + """ +@ctypes_function( + ["llama_ext", "?llama_ext@@YAXXZ", "_Z9llama_extv"], + [], + None, + required=False, +) +def llama_ext(): + pass +""", + encoding="utf-8", + ) + declarations = extract_ctypes_bindings(source) + scan = _scan("libllama.so", "linux", ["_Z9llama_extv"]) + result = check_bindings(scan, declarations) + + assert declarations == [ + BindingDeclaration( + python_name="llama_ext", + candidates=( + "llama_ext", + "?llama_ext@@YAXXZ", + "_Z9llama_extv", + ), + required=False, + line=8, + ) + ] + assert result["available"][0]["selected"] == "_Z9llama_extv" + assert result["missing_optional"] == [] + + +def test_selects_library_by_symbol_not_filename(): + scans = [ + _scan("custom-backend-name.dll", "windows", ["ggml_backend_init"]), + _scan("renamed-native-output.bin", "windows", ["llama_decode"]), + ] + + selected = select_scans_by_symbols(scans, ["llama_decode"]) + + assert [scan.library for scan in selected] == ["renamed-native-output.bin"] + + +def test_writes_jsonl_named_after_dynamic_library(tmp_path): + output_dir = tmp_path / "output" + scans = [ + _scan("libllama.so", "linux", ["llama_decode"]), + _scan("llama.dll", "windows", ["llama_decode"]), + ] + + timestamp = "20260728T120000.123456Z" + written = write_library_jsonl( + scans, + output_dir, + timestamp=timestamp, + ) + + assert [path.name for path in written] == [ + "libllama.so.jsonl", + "llama.dll.jsonl", + ] + run_dir = output_dir / timestamp + row = json.loads((run_dir / "llama.dll.jsonl").read_text("utf-8")) + assert row["library"] == "llama.dll" + assert row["canonical_name"] == "llama_decode" + assert row["generated_at"] == timestamp + assert "path" not in row + + +def test_check_bindings_cli_fails_when_optional_api_is_missing( + tmp_path, + monkeypatch, +): + library = tmp_path / "renamed.dll" + library.touch() + source = tmp_path / "bindings.py" + source.write_text( + """ +@ctypes_function(["llama_ext", "_Z9llama_extv"], [], None, required=False) +def llama_ext(): + pass +""", + encoding="utf-8", + ) + scan = _scan("renamed.dll", "windows", ["llama_decode"]) + monkeypatch.setattr( + abi_tool, + "_scan_paths", + lambda paths: ([scan], []), + ) + + exit_code = abi_tool.main(["check-bindings", str(library), "--source", str(source)]) + + assert exit_code == 1 From 7708b3de2596fb3262df565ce46c14a4db9ab254 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 28 Jul 2026 23:55:47 +0800 Subject: [PATCH 05/29] Update Submodule vendor/llama.cpp b77d646..7e1e28c Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 52 ++++++++++++++++++++++++++---------------- vendor/llama.cpp | 2 +- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 317a43548..2901c6e2e 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -504,6 +504,30 @@ class llama_split_mode(enum.IntEnum): LLAMA_SPLIT_MODE_ROW = 2 LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_load_mode { +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# }; +class llama_load_mode(enum.IntEnum): + LLAMA_LOAD_MODE_NONE = 0 # no special loading mode + LLAMA_LOAD_MODE_MMAP = 1 # memory map the model + LLAMA_LOAD_MODE_MLOCK = 2 # force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_MMAP_MLOCK = 3 # mmap + force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_DIRECT_IO = 4 # use direct I/O if available + +# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); +@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) +def llama_load_mode_name(load_mode: int) -> bytes: + ... + +# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); +@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int) +def llama_load_mode_from_str(str: ctypes.c_char_p) -> int: + ... + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -743,17 +767,15 @@ class llama_model_tensor_buft_override(ctypes.Structure): # struct llama_model_params { # // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) # ggml_backend_dev_t * devices; -# + # // NULL-terminated list of buffer types to use for tensors that match a pattern # const struct llama_model_tensor_buft_override * tensor_buft_overrides; -# + # int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers # enum llama_split_mode split_mode; // how to split the model across multiple GPUs +# enum llama_load_mode load_mode; // how to load the model -# // main_gpu interpretation depends on split_mode: -# // LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model -# // LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results -# // LLAMA_SPLIT_MODE_LAYER: ignored +# // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; # // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() @@ -770,12 +792,8 @@ class llama_model_tensor_buft_override(ctypes.Structure): # // override key-value pairs of the model meta data # const struct llama_model_kv_override * kv_overrides; - # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported -# bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used @@ -789,15 +807,13 @@ class llama_model_params(ctypes.Structure): tensor_buft_overrides(llama_model_tensor_buft_override): NULL-terminated list of buffer types to use for tensors that match a pattern n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs + load_mode (int): how to load the model main_gpu (int): the GPU that is used for the entire model. main_gpu interpretation depends on split_mode: LLAMA_SPLIT_NONE: the GPU that is used for the entire model LLAMA_SPLIT_ROW: the GPU that is used for small tensors and intermediate results LLAMA_SPLIT_LAYER: ignored tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights - use_mmap (bool): use mmap if possible - use_direct_io(bool): use direct io, takes precedence over use_mmap when supported - use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used @@ -808,34 +824,30 @@ class llama_model_params(ctypes.Structure): tensor_buft_overrides: CtypesPointer[llama_model_tensor_buft_override] n_gpu_layers: int split_mode: int + load_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool - use_mmap: bool - use_direct_io: bool - use_mlock: bool check_tensors: bool use_extra_bufts: bool no_host: bool no_alloc: bool _fields_ = [ - ("devices", ctypes.c_void_p), # NOTE: unnused + ("devices", ctypes.POINTER(ctypes.c_void_p)), # NOTE: unnused ("tensor_buft_overrides", ctypes.POINTER(llama_model_tensor_buft_override)), ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), + ("load_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), - ("use_mmap", ctypes.c_bool), - ("use_direct_io", ctypes.c_bool), - ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b77d64675..7e1e28cae 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b77d646751d01c0962bc203b6809e9d94f7d50b7 +Subproject commit 7e1e28cae36d41fe7bbe9dae7c9625de6565c063 From 00591a5b6a635682914e410224e3736d21ecb088 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 29 Jul 2026 00:03:14 +0800 Subject: [PATCH 06/29] feat(llama): support llama_model_params load_mode - Update model loading configuration to use the new `load_mode` field from llama_model_params and align with the latest llama.cpp API changes. - Remove deprecated internal handling of legacy loading flags and keep backward compatibility by warning users when `use_mmap`, `use_direct_io`, or `use_mlock` are still used. - This prepares the Python bindings for the updated llama.cpp model loading interface while providing a smoother migration path for existing users. Signed-off-by: JamePeng --- examples/low_level_api/common.py | 3 --- .../low_level_api/low_level_api_chat_cpp.py | 3 --- llama_cpp/llama.py | 22 +++++++++++-------- llama_cpp/server/model.py | 4 +--- llama_cpp/server/settings.py | 18 +++++---------- tests/test_llama.py | 3 --- 6 files changed, 19 insertions(+), 34 deletions(-) diff --git a/examples/low_level_api/common.py b/examples/low_level_api/common.py index 8adb2923c..601f5cebd 100644 --- a/examples/low_level_api/common.py +++ b/examples/low_level_api/common.py @@ -60,9 +60,6 @@ class GptParams: instruct: bool = False perplexity: bool = False - use_mmap: bool = True - use_direct_io: bool = False - use_mlock: bool = False mem_test: bool = False verbose_prompt: bool = False diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 1f4f5b3e7..96c4121f4 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -76,9 +76,6 @@ def __init__(self, params: GptParams) -> None: self.lparams.n_parts = self.params.n_parts self.lparams.seed = self.params.seed self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap - self.lparams.use_direct_io = self.params.use_direct_io self.model = llama_cpp.llama_load_model_from_file( self.params.model.encode("utf8"), self.lparams diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index f733d7afb..3ce635b54 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -104,10 +104,11 @@ def __init__( cpu_moe: bool = False, n_cpu_moe: int = 0, split_mode: int = llama_cpp_lib.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, + load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_MMAP, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, vocab_only: bool = False, - use_mmap: bool = True, + use_mmap: bool = False, use_direct_io: bool = False, use_mlock: bool = False, check_tensors: bool = False, @@ -215,11 +216,10 @@ def __init__( n_cpu_moe: Keep the MoE expert weights of the first N layers on CPU. Useful when VRAM is insufficient for MoE models. split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options. + load_mode: How to load the model. See llama_cpp.LLAMA_LOAD_MODE_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. vocab_only: Only load the vocabulary no weights. - use_mmap: Use mmap if possible. - use_mlock: Force the system to keep the model in RAM. check_tensors: validate model tensor data use_extra_bufts: use extra buffer types (used for weight repacking) no_host: bypass host buffer allowing extra buffers to be used @@ -352,10 +352,19 @@ def __init__( self.model_path = model_path + if (use_mmap or use_direct_io or use_mlock) and verbose: + print( + "Llama.__init__: WARNING: " + "Legacy load options (`use_mmap`, `use_direct_io`, `use_mlock`) " + "are deprecated. Use `load_mode` instead.", + file=sys.stderr, + ) + # Model Params self.model_params = llama_cpp_lib.llama_model_default_params() self.model_params.n_gpu_layers = self._parse_n_gpu_layers(n_gpu_layers) self.model_params.split_mode = split_mode + self.model_params.load_mode = load_mode self.model_params.main_gpu = main_gpu self.tensor_split = tensor_split self._c_tensor_split = None @@ -371,9 +380,6 @@ def __init__( ) # keep a reference to the array so it is not gc'd self.model_params.tensor_split = self._c_tensor_split self.model_params.vocab_only = vocab_only - self.model_params.use_mmap = use_mmap - self.model_params.use_direct_io = use_direct_io - self.model_params.use_mlock = use_mlock self.model_params.check_tensors = check_tensors self.model_params.use_extra_bufts = use_extra_bufts self.model_params.no_host = no_host @@ -3445,12 +3451,10 @@ def __getstate__(self): cpu_moe=self.cpu_moe, n_cpu_moe=self.n_cpu_moe, split_mode=self.model_params.split_mode, + load_mode=self.model_params.load_mode, main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, vocab_only=self.model_params.vocab_only, - use_mmap=self.model_params.use_mmap, - use_direct_io=self.model_params.use_direct_io, - use_mlock=self.model_params.use_mlock, check_tensors=self.model_params.check_tensors, use_extra_bufts=self.model_params.use_extra_bufts, no_host=self.model_params.no_host, diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index 6b3fd1dd1..0d509bbcf 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -294,12 +294,10 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: # Model Params n_gpu_layers=settings.n_gpu_layers, split_mode=settings.split_mode, + load_mode=settings.load_mode, main_gpu=settings.main_gpu, tensor_split=settings.tensor_split, vocab_only=settings.vocab_only, - use_mmap=settings.use_mmap, - use_direct_io=settings.use_direct_io, - use_mlock=settings.use_mlock, check_tensors=settings.check_tensors, use_extra_bufts=settings.use_extra_bufts, no_host=settings.no_host, diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 350ccc232..62ce3b504 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -32,8 +32,12 @@ class ModelSettings(BaseSettings): ) split_mode: int = Field( default=llama_cpp.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, - description="The split mode to use.", + description="how to split the model across multiple GPUs", ) + load_mode: int = Field( + default=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP, + description="how to load the model", + ) main_gpu: int = Field( default=0, ge=0, @@ -46,18 +50,6 @@ class ModelSettings(BaseSettings): vocab_only: bool = Field( default=False, description="Whether to only return the vocabulary." ) - use_mmap: bool = Field( - default=True, - description="Enable mmap to use filesystem cache.", - ) - use_direct_io: bool = Field( - default=False, - description="Use direct io, takes precedence over use_mmap.", - ) - use_mlock: bool = Field( - default=False, - description="Use mlock for force system to keep model in RAM", - ) check_tensors: bool = Field( default=False, description="Validate model tensor data.", diff --git a/tests/test_llama.py b/tests/test_llama.py index b233ea526..d0053feab 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -260,9 +260,6 @@ def test_real_model(llama_cpp_model_path): # 1. Setup Model Parameters params = llama_cpp.llama_model_default_params() - params.use_mmap = llama_cpp.llama_supports_mmap() - params.use_direct_io = False - params.use_mlock = llama_cpp.llama_supports_mlock() params.check_tensors = False # 2. Load the Model From bab30611b4035bd69765d4856f907c763a6a69fb Mon Sep 17 00:00:00 2001 From: JamePeng Date: Wed, 29 Jul 2026 00:11:27 +0800 Subject: [PATCH 07/29] docs: document `load_mode` migration - Replace references to the legacy model loading flags with load_mode, document all supported loading modes for the Python API and server, and update the performance tuning example. Signed-off-by: JamePeng --- docs/server.md | 20 +++++++++++ docs/wiki/core/Llama.md | 40 ++++++++++++++++++++-- examples/notebooks/PerformanceTuning.ipynb | 12 +++++-- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/docs/server.md b/docs/server.md index cd6f86c51..3bdc0c7e6 100644 --- a/docs/server.md +++ b/docs/server.md @@ -37,6 +37,26 @@ CLI arguments and environment variables are available for all of the fields defi Additionally the server supports configuration check out the [configuration section](#configuration-and-multi-model-support) for more information and examples. +#### Model loading mode + +Use `load_mode` to select how the server loads model data. The corresponding +CLI option is `--load_mode`, the environment variable is `LOAD_MODE`, and a +multi-model JSON configuration can set `"load_mode"` for each model. +`use_mmap`, `use_direct_io`, and `use_mlock` are no longer server settings. + +| Value | Mode | Description | +|---:|---|---| +| `0` | `LLAMA_LOAD_MODE_NONE` | Use no special model-loading mode. | +| `1` | `LLAMA_LOAD_MODE_MMAP` | Memory-map the model. This is the default. | +| `2` | `LLAMA_LOAD_MODE_MLOCK` | Keep the loaded model in RAM rather than allowing it to be swapped or compressed. | +| `3` | `LLAMA_LOAD_MODE_MMAP_MLOCK` | Memory-map the model and keep its mapped pages in RAM. | +| `4` | `LLAMA_LOAD_MODE_DIRECT_IO` | Use direct I/O when it is available. | + +For example, start the server with memory mapping plus memory locking: + +```bash +python3 -m llama_cpp.server --model --load_mode 3 +``` ## Guides diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 305624c8f..00add6ea4 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -3,7 +3,7 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-07-26 +last_updated: 2026-07-29 version_target: "latest" --- @@ -36,13 +36,47 @@ Initialize the model and context. Note that model loading will immediately alloc | `cpu_moe` | `bool` | `False` | Whether to keep all MoE weights on CPU | | `n_cpu_moe` | `int` | `0` | Number of first N MoE layers to keep on CPU (compatible with `cpu_moe`) | | `split_mode` | `int` | `LLAMA_SPLIT_MODE_LAYER` | Model GPU split mode:
• `LLAMA_SPLIT_MODE_NONE`: single GPU
• `LLAMA_SPLIT_MODE_ROW`: row-level split
• `LLAMA_SPLIT_MODE_LAYER`: layer-level split | +| `load_mode` | `int` (`llama_load_mode`) | `LLAMA_LOAD_MODE_MMAP` | How model data is loaded. Select one of the `LLAMA_LOAD_MODE_*` values described below. | | `main_gpu` | `int` | `0` | The primary GPU to use for intermediate results or the entire model. | | `tensor_split` | `List[float]` | `None` | Proportional split of tensors across GPUs (max `LLAMA_MAX_DEVICES`). | -| `use_mmap` | `bool` | `True` | Whether to use memory mapping (mmap) if possible. | -| `use_mlock` | `bool` | `False` | Force the system to keep the model in RAM, preventing swapping. | | `kv_overrides` | `Dict` | `None` | Key-value overrides for the model metadata (supports bool, int, float, str). | | `numa` | `Union[bool, int]` | `False` | NUMA strategy (e.g., `GGML_NUMA_STRATEGY_DISTRIBUTE`). | +#### Model Load Modes + +`load_mode` replaces the legacy `use_mmap`, `use_direct_io`, and `use_mlock` +arguments. It accepts a member of `llama_cpp.llama_load_mode`: + +| Value | Integer | Description | +| :--- | :---: | :--- | +| `LLAMA_LOAD_MODE_NONE` | `0` | Use no special model-loading mode. | +| `LLAMA_LOAD_MODE_MMAP` | `1` | Memory-map the model. This is the default. | +| `LLAMA_LOAD_MODE_MLOCK` | `2` | Keep the loaded model in RAM rather than allowing it to be swapped or compressed. | +| `LLAMA_LOAD_MODE_MMAP_MLOCK` | `3` | Memory-map the model and keep its mapped pages in RAM. | +| `LLAMA_LOAD_MODE_DIRECT_IO` | `4` | Use direct I/O when it is available. | + +```python +import llama_cpp + +llm = llama_cpp.Llama( + model_path="models/model.gguf", + load_mode=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK, +) +``` + +The legacy loading arguments are retained only for call compatibility. They no +longer configure the underlying model parameters and may emit a deprecation +warning; set `load_mode` explicitly instead. Use the following migration +mapping: + +| Legacy configuration | Replacement | +| :--- | :--- | +| `use_mmap=False, use_mlock=False` | `load_mode=LLAMA_LOAD_MODE_NONE` | +| `use_mmap=True, use_mlock=False` | `load_mode=LLAMA_LOAD_MODE_MMAP` | +| `use_mmap=False, use_mlock=True` | `load_mode=LLAMA_LOAD_MODE_MLOCK` | +| `use_mmap=True, use_mlock=True` | `load_mode=LLAMA_LOAD_MODE_MMAP_MLOCK` | +| `use_direct_io=True` | `load_mode=LLAMA_LOAD_MODE_DIRECT_IO` | + ### Context & Batch Parameters | Parameter | Type | Default | Description | diff --git a/examples/notebooks/PerformanceTuning.ipynb b/examples/notebooks/PerformanceTuning.ipynb index ba74e4a41..43772a5b7 100644 --- a/examples/notebooks/PerformanceTuning.ipynb +++ b/examples/notebooks/PerformanceTuning.ipynb @@ -24,7 +24,13 @@ "# Hyperparameters\n", "space = [\n", " Categorical([True, False], name=\"f16_kv\"),\n", - " Categorical([True, False], name=\"use_mlock\"),\n", + " Categorical(\n", + " [\n", + " llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP,\n", + " llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP_MLOCK,\n", + " ],\n", + " name=\"load_mode\",\n", + " ),\n", " Integer(1, multiprocessing.cpu_count(), name=\"n_threads\"),\n", " Integer(1, 2048, name=\"n_batch\"),\n", "]\n", @@ -46,13 +52,13 @@ "@use_named_args(space)\n", "def objective(**params):\n", " f16_kv = params[\"f16_kv\"]\n", - " use_mlock = params[\"use_mlock\"]\n", + " load_mode = params[\"load_mode\"]\n", " n_threads = params[\"n_threads\"]\n", " n_batch = params[\"n_batch\"]\n", " llm = llama_cpp.Llama(\n", " model_path=MODEL_PATH,\n", " f16_kv=f16_kv,\n", - " use_mlock=use_mlock,\n", + " load_mode=load_mode,\n", " n_threads=n_threads,\n", " n_batch=n_batch,\n", " )\n", From f8bc6b05f8c384e675c6378e17e3c548c4f465e5 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 30 Jul 2026 21:18:43 +0800 Subject: [PATCH 08/29] Update Submodule vendor/llama.cpp 7e1e28c..e1a1abb Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 20 ++++++++++++++++++++ vendor/llama.cpp | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 2901c6e2e..a956dda47 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -3544,6 +3544,26 @@ def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ... +# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) +# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens); +@ctypes_function( + "llama_vocab_get_suppress_tokens", + [ + llama_vocab_p_ctypes, + ctypes.POINTER(ctypes.c_int32), + ], + llama_token_p, +) +def llama_vocab_get_suppress_tokens( + vocab: llama_vocab_p, + n_suppress_tokens: ctypes.POINTER(ctypes.c_int32), # type: ignore +) -> llama_token_p: # type: ignore + """ + model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) + """ + ... + + # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_fim_pre", diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 7e1e28cae..e1a1abb78 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 7e1e28cae36d41fe7bbe9dae7c9625de6565c063 +Subproject commit e1a1abb78746c025f5e9039f590e37ccdb758ae7 From 9ac3f545ac99048526a1dfcf6d3f30f1bc10df82 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 1 Aug 2026 00:14:19 +0800 Subject: [PATCH 09/29] Update Submodule vendor/llama.cpp e1a1abb..876a432 Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 6 +++++- vendor/llama.cpp | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index a956dda47..0c2663709 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -798,6 +798,7 @@ class llama_model_tensor_buft_override(ctypes.Structure): # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used # bool no_alloc; // only load metadata and simulate memory allocations +# bool load_mtp; // whether to load MTP layers # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -817,7 +818,8 @@ class llama_model_params(ctypes.Structure): check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used - no_alloc (bool): only load metadata and simulate memory allocations""" + no_alloc (bool): only load metadata and simulate memory allocations + load_mtp (bool): whether to load MTP layers""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused @@ -835,6 +837,7 @@ class llama_model_params(ctypes.Structure): use_extra_bufts: bool no_host: bool no_alloc: bool + load_mtp: bool _fields_ = [ ("devices", ctypes.POINTER(ctypes.c_void_p)), # NOTE: unnused @@ -852,6 +855,7 @@ class llama_model_params(ctypes.Structure): ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), ("no_alloc", ctypes.c_bool), + ("load_mtp", ctypes.c_bool), ] llama_model_params_p = ctypes.POINTER(llama_model_params) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e1a1abb78..876a43211 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e1a1abb78746c025f5e9039f590e37ccdb758ae7 +Subproject commit 876a4321163249c43ca4e986818fab5ab081f282 From 6f60d0347bba9e2f987f36af51a9bf9865ae824b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 1 Aug 2026 00:18:03 +0800 Subject: [PATCH 10/29] feat(llama): expose additional model loading options - add `no_alloc` and `load_mtp` parameters - enable `extra buffer types` by default Signed-off-by: JamePeng --- llama_cpp/llama.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 3ce635b54..8092f8695 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -107,14 +107,16 @@ def __init__( load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_MMAP, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, - vocab_only: bool = False, + kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, use_mmap: bool = False, use_direct_io: bool = False, use_mlock: bool = False, + vocab_only: bool = False, check_tensors: bool = False, - use_extra_bufts: bool = False, + use_extra_bufts: bool = True, no_host: bool = False, - kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, + no_alloc: bool = False, + load_mtp: bool = False, # Context Params seed: int = llama_cpp_lib.LLAMA_DEFAULT_SEED, n_ctx: int = 512, @@ -219,11 +221,13 @@ def __init__( load_mode: How to load the model. See llama_cpp.LLAMA_LOAD_MODE_* for options. main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split. + kv_overrides: Key-value overrides for the model. vocab_only: Only load the vocabulary no weights. check_tensors: validate model tensor data use_extra_bufts: use extra buffer types (used for weight repacking) no_host: bypass host buffer allowing extra buffers to be used - kv_overrides: Key-value overrides for the model. + no_alloc: only load metadata and simulate memory allocations + load_mtp: whether to load MTP layers seed: RNG seed, -1 for random n_ctx: Text context, 0 = from model n_keep: Number of tokens to keep from initial prompt @@ -383,6 +387,8 @@ def __init__( self.model_params.check_tensors = check_tensors self.model_params.use_extra_bufts = use_extra_bufts self.model_params.no_host = no_host + self.model_params.no_alloc = no_alloc + self.model_params.load_mtp = load_mtp # Logic of cpu_moe, n_cpu_moe # Reference from llama.cpp/tools/llama-bench/llama-bench.cpp @@ -3454,11 +3460,13 @@ def __getstate__(self): load_mode=self.model_params.load_mode, main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, + kv_overrides=self.kv_overrides, vocab_only=self.model_params.vocab_only, check_tensors=self.model_params.check_tensors, use_extra_bufts=self.model_params.use_extra_bufts, no_host=self.model_params.no_host, - kv_overrides=self.kv_overrides, + no_alloc=self.model_params.no_alloc, + load_mtp=self.model_params.load_mtp, # Context Params seed=self._seed, n_ctx=self.context_params.n_ctx, From aafc6fb74ebfba6a044510f80b5e9ad277109c12 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 1 Aug 2026 00:59:46 +0800 Subject: [PATCH 11/29] fix(ctypes): correct llama-ext binding signatures - use uint32_t for layer IDs - fix void return type for embedding extraction control - return target layer count as uint32_t Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 0c2663709..cc6a1a67f 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -5318,15 +5318,15 @@ def llama_get_embeddings_nextn_ith( "__Z30llama_set_embeddings_layer_inpP13llama_contextjb", "_Z30llama_set_embeddings_layer_inpP13llama_contextjb", ], - [llama_context_p_ctypes, ctypes.c_int32, ctypes.c_bool], - ctypes.POINTER(ctypes.c_float), + [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_bool], + None, required=False, ) def llama_set_embeddings_layer_inp( ctx: llama_context_p, - lid: ctypes.c_int32, + lid: ctypes.c_uint32, value: bool, -) -> ctypes.POINTER(ctypes.c_float): # type: ignore +) -> None: # type: ignore """ Set whether the context outputs the input embeddings of a specific layer """ @@ -5342,13 +5342,13 @@ def llama_set_embeddings_layer_inp( "__Z30llama_get_embeddings_layer_inpP13llama_contextj", "_Z30llama_get_embeddings_layer_inpP13llama_contextj", ], - [llama_context_p_ctypes, ctypes.c_int32], + [llama_context_p_ctypes, ctypes.c_uint32], ctypes.POINTER(ctypes.c_float), required=False, ) def llama_get_embeddings_layer_inp( ctx: llama_context_p, - lid: ctypes.c_int32, + lid: ctypes.c_uint32, ) -> ctypes.POINTER(ctypes.c_float): # type: ignore ... @@ -5402,12 +5402,12 @@ def llama_model_target_layer_ids( "_Z30llama_model_target_layer_ids_nPK11llama_model", ], [llama_model_p_ctypes], - ctypes.POINTER(ctypes.c_uint32), + ctypes.c_uint32, required=False, ) def llama_model_target_layer_ids_n( model: llama_model_p -) -> ctypes.POINTER(ctypes.c_uint32): # type: ignore +) -> int: """ returns the number of extracted layers from target model """ From d9d27a7bdf1c27d50c1490ad6acd825f31804902 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 1 Aug 2026 04:17:19 +0800 Subject: [PATCH 12/29] Bump version to 0.3.45 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - This release focuses on reactivating and modernizing Llama’s built-in embedding capabilities, aligning the Python bindings with the latest llama.cpp APIs, and improving reliability across platforms. Signed-off-by: JamePeng --- CHANGELOG.md | 152 ++++++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fc02b25..075b97884 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,158 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.45] Reactivated Built-in Embeddings, Modern Model Loading, and Stronger Cross-Platform Reliability + +- fix(ctypes): correct llama-ext binding signatures + - use uint32_t for layer IDs + - fix void return type for embedding extraction control + - return target layer count as uint32_t + +- feat(llama): expose additional model loading options + - add `no_alloc` and `load_mtp` parameters + - enable `extra buffer types` by default + +- feat(llama): support llama_model_params `load_mode` + - Update model loading configuration to use the new `load_mode` field from + llama_model_params and align with the latest llama.cpp API changes. + - Remove deprecated internal handling of legacy loading flags and keep + backward compatibility by warning users when `use_mmap`, `use_direct_io`, + or `use_mlock` are still used. + - This prepares the Python bindings for the updated llama.cpp model loading + interface while providing a smoother migration path for existing users. + - docs: document `load_mode` migration + - Replace references to the legacy model loading flags with load_mode, document all supported loading modes for the Python API and server, and update the performance tuning example. + +- feat(tools): add cross-platform ABI inspection utility + - Inspect `PE`, `ELF`, and `Mach-O` exports and normalize platform-specific symbol names. + - Validate optional `llama_ext` ctypes aliases across Windows, Linux, and macOS builds. Keep artifacts and timestamped privacy-safe reports local to the repository. + - More information see here: [Cross-platform ABI inspection](https://github.com/JamePeng/llama-cpp-python/tree/main/tools/abi) + +- fix(ctypes): support GCC/Clang mangled symbols for optional llama_ext APIs + - Add missing `_Z` Itanium C++ ABI symbol variants to ctypes function + lookup lists. This improves compatibility with Linux and macOS builds + where C++ symbols are exported using GCC/Clang name mangling. + - Issue report from **@ckcfcc** (https://github.com/JamePeng/llama-cpp-python/issues/159) + +- fix(loader): guard `HIP_PATH` and `VULKAN_SDK` dirs with os.path.exists +os.add_dll_directory() raises FileNotFoundError [WinError 3] when the +directory does not exist, so a stale `HIP_PATH` or `VULKAN_SDK` left behind by +an uninstalled SDK makes "import llama_cpp" fail outright on Windows.(by **@emptyngton**) + + The CUDA_PATH branch above already guards each candidate directory with + os.path.exists(); this applies the same pattern to the HIP and Vulkan + branches. Valid directories are still added individually, so a partially + removed SDK contributes whichever of bin/lib remain instead of raising. + +- fix(_internals): clean up native resources on initialization failures + - Register native model and batch ownership immediately after allocation + so later validation failures cannot leak llama.cpp resources. + Free a loaded model when vocab lookup fails, and route mixed-batch setup + failures through idempotent cleanup. + - Initialize sampling-context resource fields before fallible setup and + make partial teardown safe to repeat. This prevents missing attributes + from interrupting cleanup when sampler-chain construction fails. + - Clear model, vocabulary, and sampling parameter references after native + context and sampler resources have been released. This prevents closed + wrapper objects from unnecessarily keeping models and related Python + objects alive. + - Add failure-injection tests that verify model and batch handles are freed + exactly once and partially initialized sampling contexts release their resources + idempotently.Extend lifecycle tests to verify that parent references are cleared + and that repeated close calls remain safe. + +- test(chat-format): modernize coverage with Qwen3.5-style templates + - Replace the legacy Mistral-focused chat format tests with self-contained + Qwen3.5-style Jinja template coverage: + - verify ChatML system, user, and assistant message rendering + - cover enabled and disabled thinking generation prompts + - test image and video placeholders with vision identifiers + - validate tool definitions, tool calls, and tool response history + - add clear error coverage for invalid message structures + - verify model-specific stop token criteria + - keep the tests independent of tokenizer files and model weights + +- docs(readme): replace the new logo with fork project branding + - Add the new llama-cpp-python logo asset under docs and update the README + header to reference the repository-local image. + - the new logo which combined llama, C++, and Project branding remains readable. + +- docs(embedding): add end-to-end embeddings and reranking guide + - Create a schema-compliant feature guide covering sentence embeddings, + token-level vectors, reranking workflows, pooling modes, normalization, + streaming batch configuration, return shapes, and output formats. + - Add complete examples for the standard Llama API, LlamaEmbedding, + pre-tokenized inputs, cosine-similarity output, and cross-encoder + reranking. + - Document common configuration problems, implementation limitations, and + the embedding and reranking model families currently listed as supported + by the project. + - Expose the new feature guide through the Wiki index. + +- docs(llama): expand embedding parameters and API guidance + - Add a role overview and reorganize constructor options into focused, + readable parameter groups. + - Document embedding, pooling, attention, KV cache, sequence capacity, and + recurrent-state settings with their defaults and runtime behavior. + - Expand the embed() and create_embedding() sections with normalization + modes, return shapes, batching semantics, pooling recommendations, + OpenAI compatibility notes, and resource-safe examples. + - Fix the YAML frontmatter and improve Markdown spacing for cleaner Wiki + rendering. + +- docs(embedding): document maintained APIs and sequence batch capacity + - Replace the deprecated Llama embedding guidance with current embed() and + create_embedding() usage. + - Document the roles of n_batch, n_ubatch, and n_seq_max, including + parallel batching examples, resource considerations, common sequence ID + errors, and the required configuration changes. + - Clarify that LlamaEmbedding remains a convenience interface for + embedding-oriented defaults and reranking workflows. + +- docs(example): refresh the built-in embedding usage example + - Fix the Llama constructor option from embedding=True to embeddings=True + and demonstrate L2-normalized output through create_embedding(). + +- test(embedding): cover built-in and streaming embedding workflows + - Add coverage for actionable LlamaBatch sequence-capacity errors and the + maintained embedding APIs on the standard Llama class. + - Verify pre-tokenized batches, normalization, separator-based inputs, + token accounting, OpenAI-compatible responses, and LlamaEmbedding + streaming behavior with n_seq_max=1. + - Explicitly close embedding models after integration tests to release + native context and model resources. + +- fix(embedding): respect n_seq_max when streaming embedding batches + - Use the configured sequence capacity instead of n_ubatch when deciding + when to decode the current LlamaEmbedding batch. + - This prevents invalid sequence IDs for multi-document inputs and allows + the default n_seq_max=1 configuration to process documents sequentially + without failing. + +- refactor(batch): improve sequence capacity validation guidance + - Make LlamaBatch sequence validation errors explain the configured + n_seq_max value, valid sequence ID range, and minimum capacity required + for parallel batching. + - Handle negative sequence IDs separately and provide actionable setup + guidance for Llama, LlamaEmbedding, and direct LlamaBatch users. + - Remove the unused normalize_embedding helper now that normalization is + handled by the embedding pipeline. + +- feat(embedding): modernize the built-in Llama embedding API + - Replace the legacy embedding path with sequence-aware streaming batch + processing based on the current LlamaBatch interface. + - Support string, batched string, and pre-tokenized inputs, token-level and + rank pooling outputs, separator-based splitting, token accounting, and + llama.cpp-compatible normalization modes. + - Restore embed() and create_embedding() as maintained Llama APIs while + preserving the existing boolean normalization behavior. + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/876a4321163249c43ca4e986818fab5ab081f282](https://github.com/ggml-org/llama.cpp/commit/876a4321163249c43ca4e986818fab5ab081f282) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260801 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/ebf6099b81cf67cfb5eec569466367c9fa04e9d4...aafc6fb74ebfba6a044510f80b5e9ad277109c12 + ## [0.3.44] Improved Windows DLL(OpenMP) Loading Reliability for GGML Backends - fix(ggml): preload bundled OpenMP runtime before loading ggml-base diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 10e452d5f..b359355f9 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.44" +__version__ = "0.3.45" From 07c04257e3116b3575199748abb4d66f16168ae3 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 2 Aug 2026 22:57:19 +0800 Subject: [PATCH 13/29] Update Submodule vendor/llama.cpp 876a432..221f0f6 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 876a43211..221f0f635 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 876a4321163249c43ca4e986818fab5ab081f282 +Subproject commit 221f0f6356efe2260023208365705ec5d5a7c8f5 From 88fce160be1e6f21834c1b2ddcee6c4b72ccfed8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 3 Aug 2026 21:19:44 +0800 Subject: [PATCH 14/29] Update Submodule vendor/llama.cpp 221f0f6..563dec8 Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 9 ++++++--- vendor/llama.cpp | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index cc6a1a67f..c3b38005d 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -4130,6 +4130,7 @@ def llama_chat_builtin_templates( # struct ggml_tensor * probs; # struct ggml_tensor * sampled; # struct ggml_tensor * candidates; +# int64_t n_vocab; # }; class llama_sampler_data(ctypes.Structure): if TYPE_CHECKING: @@ -4137,12 +4138,14 @@ class llama_sampler_data(ctypes.Structure): probs: ctypes.c_void_p sampled: ctypes.c_void_p candidates: ctypes.c_void_p + n_vocab: ctypes.c_int64 _fields_ = [ ("logits", ctypes.c_void_p), ("probs", ctypes.c_void_p), ("sampled", ctypes.c_void_p), ("candidates", ctypes.c_void_p), + ("n_vocab", ctypes.c_int64), ] @@ -4654,9 +4657,9 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( # int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present); // 0.0 = disabled +# float penalty_repeat, // must be > 0.0, 1.0 = disabled +# float penalty_freq, // must be finite, 0.0 = disabled +# float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 221f0f635..563dec81c 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 221f0f6356efe2260023208365705ec5d5a7c8f5 +Subproject commit 563dec81c1c538aac0fad465ea933eb2a621a183 From 62d3ae5a20dea3a973bd06acd0abac9fdf694e70 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 3 Aug 2026 22:46:59 +0800 Subject: [PATCH 15/29] fix(windows): handle conflicting OpenMP and ggml libraries - Allow duplicate OpenMP runtimes in complex environments such as ComfyUI - Stop searching the deprecated /bin directory for ggml dynamic libraries Signed-off-by: JamePeng --- llama_cpp/_ggml.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/llama_cpp/_ggml.py b/llama_cpp/_ggml.py index 9a7dac517..ee1a10187 100644 --- a/llama_cpp/_ggml.py +++ b/llama_cpp/_ggml.py @@ -35,6 +35,12 @@ def _preload_openmp_runtime(): if not _version_at_least("0.3.39"): return + # Some ComfyUI environments include complex software packages and may also contain + # additional OpenMP libraries (such as `libiomp5md.dll`); + # the best approach is to delete the conflicting libraries + # (i.e., OpenMP dynamic libraries that are not the VC143 version). + os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + libomp_path = (pathlib.Path(__file__).parent / "lib" / "libomp140.x86_64.dll") if not libomp_path.exists(): @@ -54,7 +60,7 @@ def _preload_openmp_runtime(): libggml_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) libggml_base_paths = [ libggml_base_path / "lib", - libggml_base_path / "bin", + # libggml_base_path / "bin", # The `bin` path is no longer used as a search path for dynamic ggml libraries. ] # Load bundled OpenMP runtime before ggml-base on Windows. From 9af4ec35e7d25036a187c5b9fbdcbc44290d86b8 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 3 Aug 2026 23:10:00 +0800 Subject: [PATCH 16/29] feat(internals): expose NextN embedding APIs on `LlamaContext` - Add accessors for NextN and layer input embeddings - Support selecting the NextN layer offset - Expose the auxiliary context handle - Validate layer IDs, offsets, and unavailable outputs Signed-off-by: JamePeng --- llama_cpp/_internals.py | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 9b37ebcc7..ebb785b3f 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -868,6 +868,63 @@ def get_embeddings_seq(self, seq_id: int): self._assert_ctx() return llama_cpp.llama_get_embeddings_seq(self.ctx, seq_id) + def set_embeddings_nextn(self, enabled: bool, masked: bool) -> None: + """ + Set whether the context outputs nextn embeddings or not + If masked == true, output the embeddings only for the tokens with batch.logits != 0 + If masked == false, output the embeddings for all tokens in the batch regardless of batch.logits + """ + self._assert_ctx() + llama_cpp.llama_set_embeddings_nextn(self.ctx, enabled, masked) + + def get_embeddings_nextn(self): + self._assert_ctx() + embeddings = llama_cpp.llama_get_embeddings_nextn(self.ctx) + if not embeddings: + raise RuntimeError("LlamaContext.get_embeddings_nextn: output is unavailable") + return embeddings + + def get_embeddings_nextn_ith(self, i: int): + self._assert_ctx() + embeddings = llama_cpp.llama_get_embeddings_nextn_ith(self.ctx, i) + if not embeddings: + raise RuntimeError( + f"LlamaContext.get_embeddings_nextn_ith: invalid output index {i}" + ) + return embeddings + + def set_embeddings_layer_inp(self, layer_id: int, enabled: bool) -> None: + self._assert_ctx() + if layer_id < 0: + raise ValueError("layer_id must be non-negative") + llama_cpp.llama_set_embeddings_layer_inp(self.ctx, layer_id, enabled) + + def get_embeddings_layer_inp(self, layer_id: int): + self._assert_ctx() + if layer_id < 0: + raise ValueError("layer_id must be non-negative") + embeddings = llama_cpp.llama_get_embeddings_layer_inp(self.ctx, layer_id) + if not embeddings: + raise RuntimeError( + f"LlamaContext.get_embeddings_layer_inp: layer {layer_id} output is unavailable" + ) + return embeddings + + def set_nextn_layer_offset(self, offset: int) -> None: + """ + Select which appended NextN block the DECODER_MTP graph runs (offset past + the trunk: il = n_layer() + offset). Used by the speculative NextN driver to + chain multiple trained NextN heads. Default 0 (first head). + """ + self._assert_ctx() + if offset < 0: + raise ValueError("NextN layer offset must be non-negative") + llama_cpp.llama_set_nextn_layer_offset(self.ctx, offset) + + def get_ctx_other(self): + self._assert_ctx() + return llama_cpp.llama_get_ctx_other(self.ctx) + def reset_timings(self): llama_cpp.llama_perf_context_reset(self.ctx) From 00a84126e0bd168b030a74c1abe5dbb6ffa85827 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 4 Aug 2026 21:08:51 +0800 Subject: [PATCH 17/29] Update Submodule vendor/llama.cpp 563dec8..1c3c967 Signed-off-by: JamePeng --- llama_cpp/_internals.py | 5 +- llama_cpp/llama_cpp.py | 127 ++++++++++++++++++++++------------------ vendor/llama.cpp | 2 +- 3 files changed, 73 insertions(+), 61 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index ebb785b3f..7cbd87e4b 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -2064,6 +2064,7 @@ def _build_sampler_chain(self): # Note: In some implementations, penalties come before other samplers if CommonSamplerType.PENALTIES in p.samplers: s.add_penalties( + self.n_vocab, p.penalty_last_n, p.penalty_repeat, p.penalty_freq, @@ -3176,8 +3177,8 @@ def add_grammar( c_trigger_tokens, len(trigger_tokens) )) - def add_penalties(self, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float): - self._add_sampler(llama_cpp.llama_sampler_init_penalties(penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) + def add_penalties(self, n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float): + self._add_sampler(llama_cpp.llama_sampler_init_penalties(n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) def add_dry(self, model: LlamaModel, multiplier: float, base: float, allowed_len: int, last_n: int, breakers: List[str]): """DRY (Don't Repeat Yourself) sampler.""" diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index c3b38005d..609e0bb3b 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -146,60 +146,63 @@ class llama_vocab_type(enum.IntEnum): # https://github.com/ggml-org/llama.cpp/blob/master/src/llama-vocab.h#L10 # // pre-tokenization types # enum llama_vocab_pre_type { -# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, -# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, -# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, -# LLAMA_VOCAB_PRE_TYPE_MPT = 5, -# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, -# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, -# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, -# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, -# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, -# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, -# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, -# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, -# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, -# LLAMA_VOCAB_PRE_TYPE_PORO = 15, -# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, -# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, -# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, -# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, -# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, -# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, -# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, -# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, -# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, -# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, -# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, -# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, -# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, -# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, -# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, -# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, -# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, -# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, -# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, -# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, -# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36, -# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37, -# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38, -# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39, -# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40, -# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41, -# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42, -# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43, -# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44, -# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45, -# LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46, -# LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47, -# LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, -# LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, -# LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, -# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, -# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, -# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, +# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0, +# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3, +# LLAMA_VOCAB_PRE_TYPE_FALCON = 4, +# LLAMA_VOCAB_PRE_TYPE_MPT = 5, +# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6, +# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7, +# LLAMA_VOCAB_PRE_TYPE_REFACT = 8, +# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9, +# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10, +# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11, +# LLAMA_VOCAB_PRE_TYPE_OLMO = 12, +# LLAMA_VOCAB_PRE_TYPE_DBRX = 13, +# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14, +# LLAMA_VOCAB_PRE_TYPE_PORO = 15, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16, +# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17, +# LLAMA_VOCAB_PRE_TYPE_VIKING = 18, +# LLAMA_VOCAB_PRE_TYPE_JAIS = 19, +# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20, +# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21, +# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22, +# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23, +# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24, +# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25, +# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, +# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, +# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, +# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29, +# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30, +# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31, +# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32, +# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33, +# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34, +# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35, +# LLAMA_VOCAB_PRE_TYPE_HUNYUAN = 36, +# LLAMA_VOCAB_PRE_TYPE_KIMI_K2 = 37, +# LLAMA_VOCAB_PRE_TYPE_HUNYUAN_DENSE = 38, +# LLAMA_VOCAB_PRE_TYPE_GROK_2 = 39, +# LLAMA_VOCAB_PRE_TYPE_GRANITE_DOCLING = 40, +# LLAMA_VOCAB_PRE_TYPE_MINIMAX_M2 = 41, +# LLAMA_VOCAB_PRE_TYPE_AFMOE = 42, +# LLAMA_VOCAB_PRE_TYPE_SOLAR_OPEN = 43, +# LLAMA_VOCAB_PRE_TYPE_YOUTU = 44, +# LLAMA_VOCAB_PRE_TYPE_EXAONE_MOE = 45, +# LLAMA_VOCAB_PRE_TYPE_QWEN35 = 46, +# LLAMA_VOCAB_PRE_TYPE_TINY_AYA = 47, +# LLAMA_VOCAB_PRE_TYPE_JOYAI_LLM = 48, +# LLAMA_VOCAB_PRE_TYPE_JAIS2 = 49, +# LLAMA_VOCAB_PRE_TYPE_GEMMA4 = 50, +# LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51, +# LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52, +# LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53, +# LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54, +# LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55, +# LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, # }; class llama_vocab_pre_type(enum.IntEnum): LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0 @@ -256,6 +259,9 @@ class llama_vocab_pre_type(enum.IntEnum): LLAMA_VOCAB_PRE_TYPE_SARVAM_MOE = 51 LLAMA_VOCAB_PRE_TYPE_MINICPM5 = 52 LLAMA_VOCAB_PRE_TYPE_WHITESPACE = 53 + LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI = 54 + LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55 + LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56 # // note: these values should be synchronized with ggml_rope @@ -4130,7 +4136,6 @@ def llama_chat_builtin_templates( # struct ggml_tensor * probs; # struct ggml_tensor * sampled; # struct ggml_tensor * candidates; -# int64_t n_vocab; # }; class llama_sampler_data(ctypes.Structure): if TYPE_CHECKING: @@ -4138,14 +4143,12 @@ class llama_sampler_data(ctypes.Structure): probs: ctypes.c_void_p sampled: ctypes.c_void_p candidates: ctypes.c_void_p - n_vocab: ctypes.c_int64 _fields_ = [ ("logits", ctypes.c_void_p), ("probs", ctypes.c_void_p), ("sampled", ctypes.c_void_p), ("candidates", ctypes.c_void_p), - ("n_vocab", ctypes.c_int64), ] @@ -4656,16 +4659,24 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( +# int32_t n_vocab, # int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) # float penalty_repeat, // must be > 0.0, 1.0 = disabled # float penalty_freq, // must be finite, 0.0 = disabled # float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], + [ + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_float, + ctypes.c_float, + ctypes.c_float, + ], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 563dec81c..1c3c9674d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 563dec81c1c538aac0fad465ea933eb2a621a183 +Subproject commit 1c3c9674de4d455f1e571bed808252af54932767 From 64e2114ed77d1155bd286d881144388062d9001b Mon Sep 17 00:00:00 2001 From: JamePeng Date: Thu, 6 Aug 2026 01:44:47 +0800 Subject: [PATCH 18/29] Update Submodule vendor/llama.cpp 1c3c967..69bf643 Signed-off-by: JamePeng --- llama_cpp/_internals.py | 54 ++++++++++++++++++++++++++++++++++++++--- llama_cpp/llama_cpp.py | 37 +++++++++++++++++++++++----- vendor/llama.cpp | 2 +- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 7cbd87e4b..226c8d873 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -147,6 +147,55 @@ def n_head_kv(self) -> int: def n_swa(self) -> int: return llama_cpp.llama_model_n_swa(self.model) + def target_layer_ids_n(self) -> int: + """Return the number of target-model layers extracted by this model.""" + return llama_cpp.llama_model_target_layer_ids_n(self.model) + + def target_layer_ids(self) -> List[int]: + """Return the target-model layer indices extracted by this model.""" + count = self.target_layer_ids_n() + if count == 0: + return [] + + layer_ids = llama_cpp.llama_model_target_layer_ids(self.model) + if not layer_ids: + raise RuntimeError( + "LlamaModel.target_layer_ids: native API returned a null pointer " + f"for {count} layer IDs" + ) + return [int(layer_ids[i]) for i in range(count)] + + def get_tok_embd(self) -> npt.NDArray[np.float32]: + """Return a copy of the token embedding matrix as ``[n_vocab, n_embd]``.""" + element_count = llama_cpp.llama_model_get_tok_embd(self.model, None) + if element_count == 0: + raise RuntimeError( + "LlamaModel.get_tok_embd: token embedding matrix is unavailable" + ) + + n_vocab = self.n_vocab() + n_embd = self.n_embd() + expected_count = n_vocab * n_embd + if element_count != expected_count: + raise RuntimeError( + "LlamaModel.get_tok_embd: unexpected token embedding size: " + f"native API returned {element_count} elements, expected " + f"{expected_count} ({n_vocab} x {n_embd})" + ) + + out = np.empty(element_count, dtype=np.float32) + written = llama_cpp.llama_model_get_tok_embd( + self.model, + out.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + ) + if written != element_count: + raise RuntimeError( + "LlamaModel.get_tok_embd: failed to copy the complete token " + f"embedding matrix ({written}/{element_count} elements)" + ) + + return out.reshape(n_vocab, n_embd) + def rope_freq_scale_train(self) -> float: """ Get the model's RoPE frequency scaling factor @@ -1767,7 +1816,7 @@ class LlamaSamplingParams: dynatemp_range: float = 0.00 # 0.0 = disabled dynatemp_exponent: float = 1.00 # controls how entropy maps to temperature in dynamic temperature sampler - penalty_last_n: int = 64 # last n tokens to penalize (0 = disable penalty, -1 = context size) + penalty_last_n: int = 64 # last n tokens to penalize (0 = disable penalty) penalty_repeat: float = 1.0 # 1.0 = disabled penalty_freq: float = 0.00 # 0.0 = disabled penalty_present: float = 0.00 # 0.0 = disabled @@ -1775,7 +1824,7 @@ class LlamaSamplingParams: dry_multiplier: float = 0.0 # 0.0 = disabled; DRY repetition penalty for tokens extending repetition: dry_base: float = 1.75 # 0.0 = disabled; multiplier * base ^ (length of sequence before token - allowed length) dry_allowed_length: int = 2 # tokens extending repetitions beyond this receive penalty - dry_penalty_last_n: int = -1 # how many tokens to scan for repetitions (0 = disable penalty, -1 = context size) + dry_penalty_last_n: int = 64 # how many tokens to scan for repetitions (0 = disable penalty) adaptive_target: float = -1.0 # select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) adaptive_decay: float = 0.90 # EMA decay for adaptation; history ≈ 1/(1-decay) tokens (0.0 - 0.99) @@ -3188,7 +3237,6 @@ def add_dry(self, model: LlamaModel, multiplier: float, base: float, allowed_len self._add_sampler(llama_cpp.llama_sampler_init_dry( model.vocab, - model.n_ctx_train(), multiplier, base, allowed_len, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 609e0bb3b..ed1179df2 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -4660,7 +4660,7 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( # int32_t n_vocab, -# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty) # float penalty_repeat, // must be > 0.0, 1.0 = disabled # float penalty_freq, // must be finite, 0.0 = disabled # float penalty_present); // must be finite, 0.0 = disabled @@ -4687,20 +4687,18 @@ def llama_sampler_init_penalties( # /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 -# LLAMA_API struct llama_sampler * llama_sampler_init_dry( +# LLAMA_API struct llama_sampler * llama_sampler_init_dry( # const struct llama_vocab * vocab, -# int32_t n_ctx_train, # float dry_multiplier, # float dry_base, # int32_t dry_allowed_length, -# int32_t dry_penalty_last_n, +# int32_t dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty) # const char ** seq_breakers, # size_t num_breakers); @ctypes_function( "llama_sampler_init_dry", [ llama_vocab_p_ctypes, - ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_int32, @@ -4712,7 +4710,6 @@ def llama_sampler_init_penalties( ) def llama_sampler_init_dry( vocab: llama_vocab_p, - n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, @@ -5426,3 +5423,31 @@ def llama_model_target_layer_ids_n( returns the number of extracted layers from target model """ ... + +# // retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) +# // returns total number of elements or 0 on error +# // if out is nullptr, returns the number of tokens without writing to out +# // caller must allocate enough memory for out before calling +# LLAMA_API uint32_t llama_model_get_tok_embd(const struct llama_model * model, float * out); +@ctypes_function_llama_ext( + [ + "llama_model_get_tok_embd", + "?llama_model_get_tok_embd@@YAIPEBUllama_model@@PEAM@Z", + "__Z24llama_model_get_tok_embdPK11llama_modelPf", + "_Z24llama_model_get_tok_embdPK11llama_modelPf", + ], + [llama_model_p_ctypes, ctypes.POINTER(ctypes.c_float)], + ctypes.c_uint32, + required=False, +) +def llama_model_get_tok_embd( + model: llama_model_p, + out: Optional[ctypes.POINTER(ctypes.c_float)], # type: ignore +) -> int: + """ + retrieves the whole token embedding matrix in F32 format (n_embd * n_vocab) + returns total number of elements or 0 on error + if out is nullptr, returns the number of tokens without writing to out + caller must allocate enough memory for out before calling + """ + ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 1c3c9674d..69bf64379 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 1c3c9674de4d455f1e571bed808252af54932767 +Subproject commit 69bf6437914596fbbc4caf09a7ac16f2acdd1a94 From 3397ecb3d64a4f7ba21f0877baa34f6f6a852386 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 8 Aug 2026 17:15:44 +0800 Subject: [PATCH 19/29] feat(mtmd): update bindings for audio generation and chunk serialization - add input chunk save/load APIs - add experimental generated-audio types and processing APIs - support HunyuanVL decoder positions - sync enum values and correct ctypes signatures Signed-off-by: JamePeng --- llama_cpp/mtmd_cpp.py | 250 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 231 insertions(+), 19 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index fcfaa86ee..7754b58b7 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -72,11 +72,26 @@ # MTMD_INPUT_CHUNK_TYPE_TEXT, # MTMD_INPUT_CHUNK_TYPE_IMAGE, # MTMD_INPUT_CHUNK_TYPE_AUDIO, +# MTMD_INPUT_CHUNK_TYPE_COUNT, // for validation # }; class mtmd_input_chunk_type(enum.IntEnum): - MTMD_INPUT_CHUNK_TYPE_TEXT = 0 + MTMD_INPUT_CHUNK_TYPE_TEXT = 0 MTMD_INPUT_CHUNK_TYPE_IMAGE = 1 MTMD_INPUT_CHUNK_TYPE_AUDIO = 2 + MTMD_INPUT_CHUNK_TYPE_COUNT = 3 + +# // position indexing for decoder model +# enum mtmd_pos_type { +# MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens +# MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes +# MTMD_POS_TYPE_HUNYUANVL, // HunyuanVL mrope + BOI/EOI/newline layout with XD-RoPE dim-3 +# MTMD_POS_TYPE_COUNT, // for validation +# }; +class mtmd_pos_type(enum.IntEnum): + MTMD_POS_TYPE_NORMAL = 0 # number of positions equals to number of tokens + MTMD_POS_TYPE_MROPE = 1 # qwen-vl mrope style, each image takes max(t,h,w) position indexes + MTMD_POS_TYPE_HUNYUANVL = 2 # HunyuanVL mrope + BOI/EOI/newline layout with XD-RoPE dim-3 + MTMD_POS_TYPE_COUNT = 3 # for validation # // opaque types @@ -96,15 +111,6 @@ class mtmd_input_chunk_type(enum.IntEnum): mtmd_bitmap_p = NewType("mtmd_bitmap_p", int) mtmd_bitmap_p_ctypes = c_void_p -# // position indexing for decoder model -# enum mtmd_pos_type { -# MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens -# MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes -# }; -class mtmd_pos_type(enum.IntEnum): - MTMD_POS_TYPE_NORMAL = 0 # number of positions equals to number of tokens - MTMD_POS_TYPE_MROPE = 1 # qwen-vl mrope style, each image takes max(t,h,w) position indexes - # struct mtmd_image_tokens { # uint32_t nx; // number of tokens in x direction # uint32_t ny; // number of tokens in y direction @@ -401,13 +407,13 @@ def mtmd_bitmap_init( # MTMD_API mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples, const float * data); @ctypes_function_mtmd( "mtmd_bitmap_init_from_audio", [ - c_uint, + c_size_t, POINTER(c_float) ], mtmd_bitmap_p_ctypes, ) def mtmd_bitmap_init_from_audio( - n_samples: c_uint, + n_samples: c_size_t, data: POINTER(c_float), # type: ignore /, ) -> mtmd_bitmap_p: @@ -635,6 +641,56 @@ def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p): """ ... +# // save/load an input chunk to/from a buffer (useful for KV save/load) +# // important: only chunk's metadata will be saved, the actual image/audio data will not be saved +# // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() +# // out_buf can be nullptr (to query expected_out_len) +# // returns 0 on success, non-zero on failure +# MTMD_API int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len); +@ctypes_function_mtmd("mtmd_input_chunk_save", + [ + mtmd_input_chunk_p_ctypes, + c_char_p, + c_size_t, + POINTER(c_size_t), + ], + c_int32, +) +def mtmd_input_chunk_save( + chunk: mtmd_input_chunk_p, + out_buf: bytes, + out_len: c_size_t, + expected_out_len: POINTER(c_size_t), # type: ignore +) -> int: + """ + save an input chunk to/from a buffer (useful for KV save) + important: only chunk's metadata will be saved, the actual image/audio data will not be saved + the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() + out_buf can be nullptr (to query expected_out_len) + returns 0 on success, non-zero on failure + """ + ... + +# // returns nullptr on failure +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len); +@ctypes_function_mtmd("mtmd_input_chunk_load", + [ + c_char_p, + c_size_t + ], + mtmd_input_chunk_p_ctypes, +) +def mtmd_input_chunk_load( + buf: bytes, + len: c_size_t, +) -> mtmd_input_chunk_p: + """ + load an input chunk from a buffer (useful for KV load) + important: only chunk's metadata will be saved, the actual image/audio data will not be saved + the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() + returns nullptr on failure + """ + ... # // mtmd_image_tokens # // @@ -697,8 +753,7 @@ class mtmd_decoder_pos(Structure): x: c_uint32 y: c_uint32 -mtmd_decoder_pos_p = POINTER(mtmd_decoder_pos) -mtmd_decoder_pos_p_ctypes = c_void_p +mtmd_decoder_pos_p_ctypes = POINTER(mtmd_decoder_pos) # // get position for decoder attention, to be used by M-RoPE models # // i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1 @@ -955,14 +1010,171 @@ def mtmd_get_cap_from_file(mmproj_fname: c_char_p) -> mtmd_caps: ... +# // EXPERIMENTAL API for audio generation, subjected to breaking changes + +# // represent the pipeline type +# enum mtmd_gen_audio_type { +# MTMD_GEN_AUDIO_TYPE_NONE, // not supported +# MTMD_GEN_AUDIO_TYPE_QWEN3TTS, +# }; +class mtmd_gen_audio_type(enum.IntEnum): + """Generated audio pipeline type.""" + MTMD_GEN_AUDIO_TYPE_NONE = 0 + MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 + +# struct mtmd_gen_audio_info { +# enum mtmd_gen_audio_type type; +# int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +# }; +class mtmd_gen_audio_info(Structure): + """Audio generation pipeline information.""" + + _fields_ = [ + ("type", c_int), + ("sample_rate", c_int32), + ] + + if TYPE_CHECKING: + type: int + sample_rate: int + +# MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); +@ctypes_function_mtmd( + "mtmd_gen_audio_get_info", + [ + mtmd_context_p_ctypes, + ], + mtmd_gen_audio_info, +) +def mtmd_gen_audio_get_info( + ctx: mtmd_context_p, +) -> mtmd_gen_audio_info: + ... + +# enum mtmd_gen_process_type { +# MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.) +# MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert semantic to PCM audio +# // for qwen3tts, this is code2wav +# }; +class mtmd_gen_process_type(enum.IntEnum): + """Generated audio processing stage.""" + # hidden state -> semantic codes + MTMD_GEN_PROCESS_TYPE_GEN_CODE = 0 + # semantic codes -> PCM audio + MTMD_GEN_PROCESS_TYPE_GEN_WAV = 1 + +# struct mtmd_gen_inp { +# enum mtmd_gen_process_type type; + +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# int32_t code0; // the sampled codebook 0 entry from backbone +# float * embd; // the hidden state from backbone, must have n_text_embd elements +# int32_t top_k; +# float top_p; + +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# int32_t * codes; +# size_t n_codes; +# const char * state_data; +# size_t state_size; +# }; +class mtmd_gen_inp(Structure): + """Audio generation input.""" + + _fields_ = [ + ("type", c_int), + # GEN_CODE + ("code0", c_int32), + ("embd", POINTER(c_float)), + ("top_k", c_int32), + ("top_p", c_float), + # GEN_WAV + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("state_data", c_char_p), + ("state_size", c_size_t), + ] + + if TYPE_CHECKING: + type: int + code0: int + embd: POINTER[c_float] + top_k: int + top_p: float + codes: POINTER[c_int32] + n_codes: int + state_data: bytes + state_size: int + +# struct mtmd_gen_out { +# // note: output memory is allocated by the context, valid until next process() call +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# const int32_t * codes; +# size_t n_codes; +# const float * embd; // the generated hidden state, to be fed back to backbone +# // it must have n_text_embd elements +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# const float * audio; +# size_t n_samples; +# const char * state_data; +# size_t state_size; +# }; +class mtmd_gen_out(Structure): + """Audio generation output. + Memory is owned by mtmd_context and valid until + the next mtmd_gen_audio_process() call. + """ + + _fields_ = [ + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("embd", POINTER(c_float)), + ("audio", POINTER(c_float)), + ("n_samples", c_size_t), + ("state_data", c_char_p), + ("state_size", c_size_t), + ] + + if TYPE_CHECKING: + codes: POINTER[c_int32] + n_codes: int + embd: POINTER[c_float] + audio: POINTER[c_float] + n_samples: int + state_data: bytes + state_size: int + +# // note: this API is stateless, caller must handle state management and audio frame accumulation +# MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, +# const struct mtmd_gen_inp * inp, +# struct mtmd_gen_out * out); +@ctypes_function_mtmd( + "mtmd_gen_audio_process", + [ + mtmd_context_p_ctypes, + POINTER(mtmd_gen_inp), + POINTER(mtmd_gen_out), + ], + c_int32, +) +def mtmd_gen_audio_process( + ctx: mtmd_context_p, + inp: POINTER(mtmd_gen_inp), # type: ignore + out: POINTER(mtmd_gen_out), # type: ignore +) -> int: + """ + note: this API is stateless, caller must handle state management and audio frame accumulation + """ + ... + # // test function, to be used in test-mtmd-c-api.c # MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); @ctypes_function_mtmd( "mtmd_test_create_input_chunks", [], - mtmd_input_chunk_p_ctypes, + mtmd_input_chunks_p_ctypes, ) -def mtmd_test_create_input_chunks() -> mtmd_input_chunk_p: +def mtmd_test_create_input_chunks() -> mtmd_input_chunks_p: ... @@ -1111,14 +1323,14 @@ def mtmd_helper_get_n_pos(chunks: mtmd_input_chunks_p) -> c_int32: @ctypes_function_mtmd("mtmd_helper_image_get_decoder_pos", [ mtmd_image_tokens_p_ctypes, c_int32, - mtmd_decoder_pos_p_ctypes + mtmd_decoder_pos_p_ctypes, ], None) def mtmd_helper_image_get_decoder_pos( image: mtmd_image_tokens_p, pos_0: c_int32, - out_pos: mtmd_decoder_pos_p # type: ignore -) -> c_int32: + out_pos: POINTER(mtmd_decoder_pos) # type: ignore +): """ helper to get the list of relative positions corresponding to the embedding tokens, to be used by M-RoPE out_pos must have length == mtmd_helper_get_n_tokens(image) From 81190b03f6d177988112dad5fc919491a77705d1 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 8 Aug 2026 19:53:24 +0800 Subject: [PATCH 20/29] Bump version to 0.3.46 - This release mainly focuses on API synchronization and binding improvements. Some of the newly exposed MTMD interfaces are experimental API adaptations at this stage; the corresponding higher-level features have not yet been integrated. Signed-off-by: JamePeng --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 075b97884..b3d5800c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.46] Extended Model APIs, MTMD Binding Updates, and Improved Runtime Compatibility + +- feat(mtmd): update bindings for audio generation and chunk serialization + - add input chunk save/load APIs + - add experimental generated-audio types and processing APIs + - support HunyuanVL decoder positions + - sync enum values and correct ctypes signatures + +- feat(model): expose target layer ids and token embeddings + - Add LlamaModel helpers for accessing target layer metadata and extracting + the token embedding matrix from the native model. + - The new APIs provide: + - target_layer_ids() for retrieving target model layer indices + - get_tok_embd() for copying the token embedding matrix as a NumPy array + - Add validation for native return values, including null pointers, unexpected + embedding sizes, and incomplete copy operations to provide clearer runtime + errors. + - Also update sampling parameter comments to match the current llama.cpp + behavior for penalty window configuration. + +- feat(internals): expose NextN embedding APIs on LlamaContext + - Add accessors for NextN and layer input embeddings + - Support selecting the NextN layer offset + - Expose the auxiliary context handle + - Validate layer IDs, offsets, and unavailable outputs + +- fix(windows): handle conflicting OpenMP and ggml libraries + - Allow duplicate OpenMP runtimes in complex environments such as ComfyUI + - Some ComfyUI environments include complex software packages and may also contain additional OpenMP libraries (such as `libiomp5md.dll`); + - the best approach is to delete the **conflicting libraries** (i.e., OpenMP dynamic libraries that are not the VC143 version). + - Stop searching the deprecated /bin directory for ggml dynamic libraries + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/69bf6437914596fbbc4caf09a7ac16f2acdd1a94](https://github.com/ggml-org/llama.cpp/commit/69bf6437914596fbbc4caf09a7ac16f2acdd1a94) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260808 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/d9d27a7bdf1c27d50c1490ad6acd825f31804902...3397ecb3d64a4f7ba21f0877baa34f6f6a852386 + ## [0.3.45] Reactivated Built-in Embeddings, Modern Model Loading, and Stronger Cross-Platform Reliability - fix(ctypes): correct llama-ext binding signatures diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index b359355f9..d3fec3b86 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.45" +__version__ = "0.3.46" From 1b46d6f2dfd1f36236f1055aeaf8fd1ca997a048 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Mon, 10 Aug 2026 23:41:19 +0800 Subject: [PATCH 21/29] Update Submodule vendor/llama.cpp 69bf643..dd1ea52 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 69bf64379..dd1ea5243 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 69bf6437914596fbbc4caf09a7ac16f2acdd1a94 +Subproject commit dd1ea524333b1e697489067d7a4c39c60d32beee From 6da23a3ef05eb5e74a2efe655441c03eeb0cd337 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 11 Aug 2026 00:02:14 +0800 Subject: [PATCH 22/29] feat(llama): add multi-output backend sampler API support - expose per-sequence output limits in context parameters - sync sampler reset and state-copy interfaces with llama.cpp - preserve advanced context settings when reconstructing Llama instances - document ordered multi-output sampling behavior Signed-off-by: JamePeng --- docs/wiki/core/Llama.md | 5 ++- llama_cpp/llama.py | 16 +++++++- llama_cpp/llama_cpp.py | 91 ++++++++++++++++++++++++++++++++++------- 3 files changed, 94 insertions(+), 18 deletions(-) diff --git a/docs/wiki/core/Llama.md b/docs/wiki/core/Llama.md index 00add6ea4..af5a3ce51 100644 --- a/docs/wiki/core/Llama.md +++ b/docs/wiki/core/Llama.md @@ -3,7 +3,7 @@ title: Llama Class module_name: llama_cpp.llama source_file: llama_cpp/llama.py class_name: Llama -last_updated: 2026-07-29 +last_updated: 2026-08-10 version_target: "latest" --- @@ -87,7 +87,8 @@ mapping: | `n_ubatch` | `int` | `512` | Maximum number of tokens in a physical micro-batch processed by llama.cpp. | | `n_seq_max` | `int` | `1` | Maximum independent sequence states in one decode batch. Embedding calls split automatically at this limit; larger values enable more parallel sequences. | | `n_rs_seq` | `int` | `0` | Experimental recurrent-state snapshots retained per sequence for rollback. `0` disables rollback snapshots. | -| `n_outputs_max` | `int` | `0` | Maximum outputs in a physical batch. `0` is converted to the effective `n_batch`. | +| `n_outputs_max` | `int` | `0` | Maximum outputs in a physical batch. `0` lets llama.cpp use the effective `n_batch`. | +| `n_outputs_max_per_seq` | `int` | `1` | Maximum outputs per sequence. `0` lets llama.cpp use the effective `n_outputs_max`. | | `n_threads` | `int` | `None` | Number of threads for generation (defaults to CPU count // 2). | | `n_threads_batch` | `int` | `None` | Number of threads for batch processing (defaults to CPU count). | | `ctx_type` | `int` | `LLAMA_CONTEXT_TYPE_DEFAULT` | Context implementation selected by llama.cpp. Keep the default unless a model or backend requires another context type. | diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 8092f8695..2ca91888a 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -126,6 +126,7 @@ def __init__( n_seq_max: int = 1, n_rs_seq: int = 0, n_outputs_max: int = 0, + n_outputs_max_per_seq: int = 1, n_threads: Optional[int] = None, n_threads_batch: Optional[int] = None, ctx_type: Optional[ @@ -234,8 +235,12 @@ def __init__( n_batch: Prompt processing maximum batch size n_ubatch: Physical batch size n_seq_max: max number of sequences (i.e. distinct states for recurrent models) + n_rs_seq: Number of recurrent-state snapshots per sequence for rollback. 0 disables rollback snapshots. Experimental. + n_outputs_max: Maximum outputs in a physical batch. 0 lets llama.cpp use the effective n_batch. + n_outputs_max_per_seq: Maximum outputs per sequence. 0 lets llama.cpp use the effective n_outputs_max. n_threads: Number of threads to use for generation n_threads_batch: Number of threads to use for batch processing + ctx_type: Context implementation type, such as the MTP context type. rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggml-org/llama.cpp/pull/2054 pooling_type: Pooling type, from `enum llama_pooling_type`. attention_type: attention type to use for embeddings @@ -497,6 +502,7 @@ def __init__( self.n_seq_max = n_seq_max self.n_rs_seq = n_rs_seq self.n_outputs_max = n_outputs_max + self.n_outputs_max_per_seq = n_outputs_max_per_seq self.n_threads = n_threads or max(multiprocessing.cpu_count() // 2, 1) self.n_threads_batch = n_threads_batch or multiprocessing.cpu_count() @@ -514,7 +520,8 @@ def __init__( raise RuntimeError(f"n_seq_max must be <= {llama_cpp_lib.LLAMA_MAX_SEQ}") self.context_params.n_rs_seq = self.n_rs_seq - self.context_params.n_outputs_max = self.n_batch if self.n_outputs_max == 0 else self.n_outputs_max + self.context_params.n_outputs_max = max(self.n_outputs_max, 0) + self.context_params.n_outputs_max_per_seq = max(self.n_outputs_max_per_seq, 0) self.context_params.n_threads = self.n_threads self.context_params.n_threads_batch = self.n_threads_batch @@ -3470,10 +3477,15 @@ def __getstate__(self): # Context Params seed=self._seed, n_ctx=self.context_params.n_ctx, - n_batch=self.n_batch, + n_batch=self.context_params.n_batch, n_ubatch=self.context_params.n_ubatch, + n_seq_max=self.context_params.n_seq_max, + n_rs_seq=self.context_params.n_rs_seq, + n_outputs_max=self.context_params.n_outputs_max, + n_outputs_max_per_seq=self.context_params.n_outputs_max_per_seq, n_threads=self.context_params.n_threads, n_threads_batch=self.context_params.n_threads_batch, + ctx_type=self.context_params.ctx_type, rope_scaling_type=self.context_params.rope_scaling_type, pooling_type=self.context_params.pooling_type, attention_type=self.context_params.attention_type, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index ed1179df2..efd58e3ba 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -886,14 +886,15 @@ class llama_sampler_seq_config(ctypes.Structure): # // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations # // https://github.com/ggml-org/llama.cpp/pull/7544 # struct llama_context_params { -# uint32_t n_ctx; // text context, 0 = from model -# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode -# uint32_t n_ubatch; // physical maximum batch size -# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) -# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] -# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) -# int32_t n_threads; // number of threads to use for generation -# int32_t n_threads_batch; // number of threads to use for batch processing +# uint32_t n_ctx; // text context, 0 = from model +# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode +# uint32_t n_ubatch; // physical maximum batch size +# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) +# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] +# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) +# uint32_t n_outputs_max_per_seq; // max outputs per sequence (0 = n_outputs_max) +# int32_t n_threads; // number of threads to use for generation +# int32_t n_threads_batch; // number of threads to use for batch processing # enum llama_context_type ctx_type; // set the context type (e.g. MTP) # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` @@ -954,6 +955,7 @@ class llama_context_params(ctypes.Structure): n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_rs_seq (int): number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] n_outputs_max (int): max outputs in a ubatch (0 = n_batch) + n_outputs_max_per_seq (int): max outputs per sequence (0 = n_outputs_max) n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing @@ -1001,6 +1003,7 @@ class llama_context_params(ctypes.Structure): n_seq_max: int n_rs_seq: int n_outputs_max: int + n_outputs_max_per_seq: int n_threads: int n_threads_batch: int ctx_type: int @@ -1039,6 +1042,7 @@ class llama_context_params(ctypes.Structure): ("n_seq_max", ctypes.c_uint32), ("n_rs_seq", ctypes.c_uint32), ("n_outputs_max", ctypes.c_uint32), + ("n_outputs_max_per_seq", ctypes.c_uint32), ("n_threads", ctypes.c_int32), ("n_threads_batch", ctypes.c_int32), ("ctx_type", ctypes.c_int), @@ -3295,6 +3299,9 @@ def llama_get_embeddings_seq( # // # // Get the backend sampled token for the ith token. +# // With multiple outputs, sampler state advances when the token is accepted, +# // not when it is read through this function. +# // When accepting multiple outputs, accept a contiguous prefix in output order. # // Returns LLAMA_TOKEN_NULL if no token was sampled. # LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); @ctypes_function( @@ -3307,6 +3314,9 @@ def llama_get_sampled_token_ith( ) -> ctypes.c_int32: """ Get the backend sampled token for the ith token. + With multiple outputs, sampler state advances when the token is accepted, + not when it is read through this function. + When accepting multiple outputs, accept a contiguous prefix in output order. Returns LLAMA_TOKEN_NULL if no token was sampled. """ ... @@ -4164,9 +4174,12 @@ class llama_sampler_data(ctypes.Structure): # // [EXPERIMENTAL] # // backend sampling interface: -# // return true if the backend supports all ops needed by the sampler +# // return true if the backend supports all ops needed by the sampler and can handle up to n_outputs_max_per_seq outputs per sequence # // note: call once per sampler -# bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft); +# bool (*backend_init)( +# struct llama_sampler * smpl, +# ggml_backend_buffer_type_t buft, +# uint32_t n_outputs_max_per_seq); # // call after .backend_apply() # void (*backend_accept)( @@ -4184,6 +4197,13 @@ class llama_sampler_data(ctypes.Structure): # // called before graph execution to set inputs for the current ubatch # void (*backend_set_input)(struct llama_sampler * smpl); + +# // called before rebuilding a sampling graph to clear any internal sampler state +# void (*backend_reset)(struct llama_sampler * smpl); + +# // copy mutable state from src into dst while keeping dst's references to the current sampling graph +# // src and dst must have the same type and configuration +# void (*copy_state)(const struct llama_sampler * src, struct llama_sampler * dst); # }; # const char * (*name)(const struct llama_sampler * smpl); @@ -4226,13 +4246,20 @@ class llama_sampler_data(ctypes.Structure): # --- EXPERIMENTAL Backend Sampling Interface --- -# bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft); +# // return true if the backend supports all ops needed by the sampler and can handle up to n_outputs_max_per_seq outputs per sequence +# // note: call once per sampler +# bool (*backend_init)( +# struct llama_sampler * smpl, +# ggml_backend_buffer_type_t buft, +# uint32_t n_outputs_max_per_seq); llama_sampler_backend_init_fn = ctypes.CFUNCTYPE( ctypes.c_bool, # return bool ctypes.c_void_p, # smpl - ctypes.c_void_p # buft + ctypes.c_void_p, # buft + ctypes.c_uint32, # n_outputs_max_per_seq ) +# // call after .backend_apply() # void (*backend_accept)(struct llama_sampler * smpl, struct ggml_context * ctx, struct ggml_cgraph * gf, struct ggml_tensor * selected_token); llama_sampler_backend_accept_fn = ctypes.CFUNCTYPE( None, # return void @@ -4242,6 +4269,7 @@ class llama_sampler_data(ctypes.Structure): ctypes.c_void_p # selected_token ) +# // call after .backend_init() # void (*backend_apply)(struct llama_sampler * smpl, struct ggml_context * ctx, struct ggml_cgraph * gf, struct llama_sampler_data * data); llama_sampler_backend_apply_fn = ctypes.CFUNCTYPE( None, # return void @@ -4251,12 +4279,29 @@ class llama_sampler_data(ctypes.Structure): ctypes.POINTER(llama_sampler_data) # data ) +# // called before graph execution to set inputs for the current ubatch # void (*backend_set_input)(struct llama_sampler * smpl); llama_sampler_backend_set_input_fn = ctypes.CFUNCTYPE( None, # return void ctypes.c_void_p # smpl ) +# // called before rebuilding a sampling graph to clear any internal sampler state +# void (*backend_reset)(struct llama_sampler * smpl); +llama_sampler_backend_reset_fn = ctypes.CFUNCTYPE( + None, # return void + ctypes.c_void_p # smpl +) + +# // copy mutable state from src into dst while keeping dst's references to the current sampling graph +# // src and dst must have the same type and configuration +# void (*copy_state)(const struct llama_sampler * src, struct llama_sampler * dst); +llama_sampler_copy_state_fn = ctypes.CFUNCTYPE( + None, # return void + ctypes.c_void_p, # src + ctypes.c_void_p, # dst +) + class llama_sampler_i(ctypes.Structure): _fields_ = [ ("name", llama_sampler_name_fn), @@ -4271,6 +4316,8 @@ class llama_sampler_i(ctypes.Structure): ("backend_accept", llama_sampler_backend_accept_fn), ("backend_apply", llama_sampler_backend_apply_fn), ("backend_set_input", llama_sampler_backend_set_input_fn), + ("backend_reset", llama_sampler_backend_reset_fn), + ("copy_state", llama_sampler_copy_state_fn), ] @@ -4352,8 +4399,7 @@ def llama_sampler_accept(smpl: llama_sampler_p, token: Union[llama_token, int], None, ) def llama_sampler_apply( - smpl: llama_sampler_p, cur_p: CtypesPointer[llama_token_data_array], / -): + smpl: llama_sampler_p, cur_p: CtypesPointer[llama_token_data_array]): ... @@ -4377,6 +4423,22 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... +# // copy mutable sampler state without changing dst or its sampling graph bindings +# // src and dst must have the same type and configuration +# LLAMA_API void llama_sampler_copy (const struct llama_sampler * src, struct llama_sampler * dst); +@ctypes_function( + "llama_sampler_copy", + [llama_sampler_p_ctypes, llama_sampler_p_ctypes], + None, +) +def llama_sampler_copy(src: llama_sampler_p, dst: llama_sampler_p): + """ + copy mutable sampler state without changing dst or its sampling graph bindings + src and dst must have the same type and configuration + """ + ... + + # // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) # LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); @ctypes_function( @@ -4823,6 +4885,7 @@ def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: # /// @details Sample and accept a token from the idx-th output of the last evaluation +# // For multiple outputs from one sampler, call this function in output order without gaps. # // # // Shorthand for: # // const auto * logits = llama_get_logits_ith(ctx, idx); From a0c9c41033412fe46e19cdf613767f200b0ebe87 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 11 Aug 2026 02:05:33 +0800 Subject: [PATCH 23/29] fix(internals): disable new backend hooks for custom samplers - Explicitly set backend_reset and copy_state to NULL - Clarify CPU callback behavior for CustomSampler - Document inherited backend behavior in ReasoningBudgetSampler Signed-off-by: JamePeng --- llama_cpp/_internals.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index 226c8d873..64077e705 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -2489,12 +2489,18 @@ def force_reasoning_budget(self) -> bool: class CustomSampler: """ - Base class for Python-backed custom samplers in the Llama sampler chain. + CPU sampler adapter backed by Python callbacks. Responsibilities: - - Provides apply, accept, reset, free and clone callbacks for the C sampler chain. - - Keeps Python references alive to prevent GC while C sampler still holds function pointers. - - Implements safe close to clear all callback references. + - Expose Python apply, accept, reset, free, and clone functions through + llama_sampler_i callbacks. + - Keep callback references alive while llama.cpp holds their function + pointers. + - Release the native sampler and break callback reference cycles on close. + + Backend sampling is intentionally unsupported. Every backend hook in + llama_sampler_i is explicitly initialized to NULL, including backend_reset + and copy_state, so llama.cpp keeps this sampler on the CPU callback path. """ def __init__( @@ -2545,7 +2551,7 @@ def _cb_clone(_): self._cb_free_ref = llama_cpp.llama_sampler_free_fn(_cb_free) self._cb_clone_ref = llama_cpp.llama_sampler_clone_fn(_cb_clone) - # Build llama_sampler_i + # Build the CPU-facing llama_sampler_i callback table. self.llama_sampler_i = llama_cpp.llama_sampler_i() self.llama_sampler_i.name = self._cb_name_ref @@ -2555,7 +2561,9 @@ def _cb_clone(_): self.llama_sampler_i.free = self._cb_free_ref self.llama_sampler_i.clone = self._cb_clone_ref - # Disable backend hooks + # Explicitly disable every backend hook instead of relying on ctypes + # zero-initialization. Python-backed samplers operate through the CPU + # callbacks above and do not own backend sampling graph state. self.llama_sampler_i.backend_init = ctypes.cast( 0, llama_cpp.llama_sampler_backend_init_fn ) @@ -2568,6 +2576,12 @@ def _cb_clone(_): self.llama_sampler_i.backend_set_input = ctypes.cast( 0, llama_cpp.llama_sampler_backend_set_input_fn ) + self.llama_sampler_i.backend_reset = ctypes.cast( + 0, llama_cpp.llama_sampler_backend_reset_fn + ) + self.llama_sampler_i.copy_state = ctypes.cast( + 0, llama_cpp.llama_sampler_copy_state_fn + ) self.sampler_p = llama_cpp.llama_sampler_init( ctypes.pointer(self.llama_sampler_i), @@ -2625,6 +2639,11 @@ class ReasoningBudgetSampler(CustomSampler): This mirrors the core idea of llama.cpp's reasoning-budget sampler while keeping the Python API small and explicit. + + As a CustomSampler subclass, this remains CPU/Python-backed. Its backend + hooks, including backend_reset and copy_state, stay disabled; runtime state + is managed by the regular _accept(), _apply(), _reset(), and _clone() + callbacks instead. """ def __init__( From 8a3bcbfb5e9ed628327ad94b2587cd278507ffa9 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Tue, 11 Aug 2026 02:43:31 +0800 Subject: [PATCH 24/29] fix(types): use size_t for sampler count Signed-off-by: JamePeng --- llama_cpp/llama_cpp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index efd58e3ba..6ae590ea4 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1071,7 +1071,7 @@ class llama_context_params(ctypes.Structure): ("swa_full", ctypes.c_bool), ("kv_unified", ctypes.c_bool), ("samplers", llama_sampler_seq_config_p), - ("n_samplers", ctypes.c_int), + ("n_samplers", ctypes.c_size_t), ("ctx_other", ctypes.c_void_p), ] From d5a108d726aff2c0ee9b440ed7d33e55e27b689f Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 14 Aug 2026 01:59:35 +0800 Subject: [PATCH 25/29] Update Submodule vendor/llama.cpp dd1ea52..9c5531e Signed-off-by: JamePeng --- llama_cpp/llama.py | 2 +- llama_cpp/llama_cpp.py | 44 ++++++++++++++++++++++-------------- llama_cpp/server/settings.py | 2 +- vendor/llama.cpp | 2 +- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 2ca91888a..410eee3a9 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -104,7 +104,7 @@ def __init__( cpu_moe: bool = False, n_cpu_moe: int = 0, split_mode: int = llama_cpp_lib.llama_split_mode.LLAMA_SPLIT_MODE_LAYER, - load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_MMAP, + load_mode: int = llama_cpp_lib.llama_load_mode.LLAMA_LOAD_MODE_AUTO, main_gpu: int = 0, tensor_split: Optional[List[float]] = None, kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 6ae590ea4..41c6de4dd 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -511,18 +511,20 @@ class llama_split_mode(enum.IntEnum): LLAMA_SPLIT_MODE_TENSOR = 3 # enum llama_load_mode { -# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode -# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model -# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing -# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing -# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# LLAMA_LOAD_MODE_AUTO = -1, // auto-detect based on device capabilities +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available # }; class llama_load_mode(enum.IntEnum): - LLAMA_LOAD_MODE_NONE = 0 # no special loading mode - LLAMA_LOAD_MODE_MMAP = 1 # memory map the model - LLAMA_LOAD_MODE_MLOCK = 2 # force system to keep model in RAM rather than swapping or compressing - LLAMA_LOAD_MODE_MMAP_MLOCK = 3 # mmap + force system to keep model in RAM rather than swapping or compressing - LLAMA_LOAD_MODE_DIRECT_IO = 4 # use direct I/O if available + LLAMA_LOAD_MODE_AUTO = -1 # auto-detect based on device capabilities + LLAMA_LOAD_MODE_NONE = 0 # no special loading mode + LLAMA_LOAD_MODE_MMAP = 1 # memory map the model + LLAMA_LOAD_MODE_MLOCK = 2 # force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_MMAP_MLOCK = 3 # mmap + force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_DIRECT_IO = 4 # use direct I/O if available # LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); @ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) @@ -1298,6 +1300,17 @@ class llama_chat_message(ctypes.Structure): llama_adapter_cvec_p_ctypes = ctypes.POINTER(ctypes.c_void_p) +# LLAMA_API const char * llama_version(void); +@ctypes_function( + "llama_version", + [], + ctypes.c_char_p, +) +def llama_version() -> bytes: + """Get libllama version""" + ... + + # // Helpers for getting default parameters # LLAMA_API struct llama_model_params llama_model_default_params(void); @ctypes_function( @@ -2853,7 +2866,7 @@ def llama_state_seq_save_file( ) -> int: ... - +# If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded # LLAMA_API size_t llama_state_seq_load_file( # struct llama_context * ctx, # const char * filepath, @@ -2882,6 +2895,9 @@ def llama_state_seq_load_file( n_token_count_out: CtypesPointerOrRef[ctypes.c_size_t], /, ) -> int: + """ + If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded + """ ... # define LLAMA_STATE_SEQ_FLAGS_NONE 0 @@ -4423,8 +4439,6 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... -# // copy mutable sampler state without changing dst or its sampling graph bindings -# // src and dst must have the same type and configuration # LLAMA_API void llama_sampler_copy (const struct llama_sampler * src, struct llama_sampler * dst); @ctypes_function( "llama_sampler_copy", @@ -4432,10 +4446,6 @@ def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: None, ) def llama_sampler_copy(src: llama_sampler_p, dst: llama_sampler_p): - """ - copy mutable sampler state without changing dst or its sampling graph bindings - src and dst must have the same type and configuration - """ ... diff --git a/llama_cpp/server/settings.py b/llama_cpp/server/settings.py index 62ce3b504..8652d6d94 100644 --- a/llama_cpp/server/settings.py +++ b/llama_cpp/server/settings.py @@ -35,7 +35,7 @@ class ModelSettings(BaseSettings): description="how to split the model across multiple GPUs", ) load_mode: int = Field( - default=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_MMAP, + default=llama_cpp.llama_load_mode.LLAMA_LOAD_MODE_AUTO, description="how to load the model", ) main_gpu: int = Field( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index dd1ea5243..9c5531e2b 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit dd1ea524333b1e697489067d7a4c39c60d32beee +Subproject commit 9c5531e2bf2c95e86eeac807d7109de29266ca7b From 962eef15b5375a84345d8d82150595d63084b91e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 14 Aug 2026 02:21:01 +0800 Subject: [PATCH 26/29] fix(llama): fully clear model state on reset - Clear native context memory and invalidate hybrid checkpoints to keep Python state synchronized across standard, recurrent, and hybrid models. Signed-off-by: JamePeng --- llama_cpp/llama.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 410eee3a9..7da01140a 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -1089,9 +1089,20 @@ def set_seed(self, seed: int): self._seed = seed def reset(self): - """Reset the model state.""" + """Reset all Python and native model state.""" + # Use a full memory clear rather than sequence removal: recurrent state + # cannot always be partially truncated, and hybrid memory must clear + # both its attention KV cache and recurrent state. + self._ctx.memory_clear(True) + + # Keep the Python-side token cursor in sync with the empty native state. self.n_tokens = 0 + # Hybrid checkpoints contain snapshots of the state cleared above and + # must not be reused after a reset. + if self.is_hybrid and self._hybrid_cache_mgr is not None: + self._hybrid_cache_mgr.clear() + def abort(self) -> None: """ Safely aborts any ongoing text generation. @@ -1737,10 +1748,7 @@ def generate( ) if reset: # No prefix matched at all. Completely clear the KV cache to prevent context poisoning. - self.n_tokens = 0 - self._ctx.memory_clear(True) - if self.is_hybrid and self._hybrid_cache_mgr is not None: - self._hybrid_cache_mgr.clear() + self.reset() if self.verbose: print("Llama.generate: Context reset requested or no prefix match. Cleared KV cache.", file=sys.stderr) From 403771f9979505db02bdd5071dfce1505d15cf30 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Fri, 14 Aug 2026 05:39:42 +0800 Subject: [PATCH 27/29] feat(mtmd): sync Pocket TTS and audio helper API bindings - add Pocket TTS audio generation types and fields - update generated-audio structures for the latest MTMD ABI - expose default generation parameters and audio helper APIs - add multimodal chat capability detection Signed-off-by: JamePeng --- llama_cpp/mtmd_cpp.py | 296 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 293 insertions(+), 3 deletions(-) diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 7754b58b7..ec99b7ac6 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -1016,15 +1016,18 @@ def mtmd_get_cap_from_file(mmproj_fname: c_char_p) -> mtmd_caps: # enum mtmd_gen_audio_type { # MTMD_GEN_AUDIO_TYPE_NONE, // not supported # MTMD_GEN_AUDIO_TYPE_QWEN3TTS, +# MTMD_GEN_AUDIO_TYPE_POCKETTTS, # }; class mtmd_gen_audio_type(enum.IntEnum): """Generated audio pipeline type.""" - MTMD_GEN_AUDIO_TYPE_NONE = 0 - MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 + MTMD_GEN_AUDIO_TYPE_NONE = 0 + MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 + MTMD_GEN_AUDIO_TYPE_POCKETTTS = 2 # struct mtmd_gen_audio_info { # enum mtmd_gen_audio_type type; # int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +# const char * model_variant; // name of the weight variant, can be nullptr if not applicable # }; class mtmd_gen_audio_info(Structure): """Audio generation pipeline information.""" @@ -1032,11 +1035,13 @@ class mtmd_gen_audio_info(Structure): _fields_ = [ ("type", c_int), ("sample_rate", c_int32), + ("model_variant", c_char_p), ] if TYPE_CHECKING: type: int sample_rate: int + model_variant: Optional[bytes] # MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); @ctypes_function_mtmd( @@ -1055,6 +1060,7 @@ def mtmd_gen_audio_get_info( # MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.) # MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert semantic to PCM audio # // for qwen3tts, this is code2wav +# // for pocket-tts, this is mimi decoder # }; class mtmd_gen_process_type(enum.IntEnum): """Generated audio processing stage.""" @@ -1071,10 +1077,15 @@ class mtmd_gen_process_type(enum.IntEnum): # float * embd; // the hidden state from backbone, must have n_text_embd elements # int32_t top_k; # float top_p; +# uint32_t seed; // UINT32_MAX for random +# float temp; // sampling temperature, or noise scale for flow-matching decoders # // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# // pass either codes (discrete) or feats (continuous), depending on the pipeline # int32_t * codes; # size_t n_codes; +# const float * feats; +# size_t n_feats; # const char * state_data; # size_t state_size; # }; @@ -1088,21 +1099,31 @@ class mtmd_gen_inp(Structure): ("embd", POINTER(c_float)), ("top_k", c_int32), ("top_p", c_float), + ("seed", c_uint32), + ("temp", c_float), # GEN_WAV ("codes", POINTER(c_int32)), ("n_codes", c_size_t), + ("feats", POINTER(c_float)), + ("n_feats", c_size_t), ("state_data", c_char_p), ("state_size", c_size_t), ] if TYPE_CHECKING: + # GEN_CODE type: int code0: int embd: POINTER[c_float] top_k: int top_p: float + seed: int + temp: float + # GEN_WAV codes: POINTER[c_int32] n_codes: int + feats: POINTER[c_float] + n_feats: int state_data: bytes state_size: int @@ -1110,9 +1131,12 @@ class mtmd_gen_inp(Structure): # // note: output memory is allocated by the context, valid until next process() call # // for MTMD_GEN_PROCESS_TYPE_GEN_CODE # const int32_t * codes; -# size_t n_codes; +# size_t n_codes; +# const float * feats; // continuous counterpart of codes +# size_t n_feats; # const float * embd; // the generated hidden state, to be fed back to backbone # // it must have n_text_embd elements +# bool is_eos; // only set by pipelines having the EOS head inside mmproj # // for MTMD_GEN_PROCESS_TYPE_GEN_WAV # const float * audio; # size_t n_samples; @@ -1128,7 +1152,10 @@ class mtmd_gen_out(Structure): _fields_ = [ ("codes", POINTER(c_int32)), ("n_codes", c_size_t), + ("feats", POINTER(c_float)), + ("n_feats", c_size_t), ("embd", POINTER(c_float)), + ("is_eos", c_bool), ("audio", POINTER(c_float)), ("n_samples", c_size_t), ("state_data", c_char_p), @@ -1138,12 +1165,32 @@ class mtmd_gen_out(Structure): if TYPE_CHECKING: codes: POINTER[c_int32] n_codes: int + feats: POINTER[c_float] + n_feats: int embd: POINTER[c_float] + is_eos: bool audio: POINTER[c_float] n_samples: int state_data: bytes state_size: int +# // defaults tuned for the loaded pipeline, callers override only what they care about +# MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx); +@ctypes_function_mtmd( + "mtmd_gen_inp_default", + [ + mtmd_context_p_ctypes, + ], + mtmd_gen_inp, +) +def mtmd_gen_inp_default( + ctx: mtmd_context_p, +) -> mtmd_gen_inp: + """ + defaults tuned for the loaded pipeline, callers override only what they care about + """ + ... + # // note: this API is stateless, caller must handle state management and audio frame accumulation # MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, # const struct mtmd_gen_inp * inp, @@ -1653,3 +1700,246 @@ def mtmd_helper_video_read_next( -2 on error """ ... + +# // return true if model can be used for chat +# MTMD_API bool mtmd_helper_model_can_chat(struct llama_context * lctx, struct mtmd_context * mctx); +@ctypes_function_mtmd( + "mtmd_helper_model_can_chat", [ + llama_cpp_lib.llama_context_p_ctypes, + mtmd_context_p_ctypes, + ], + c_bool, +) +def mtmd_helper_model_can_chat( + lctx: llama_cpp_lib.llama_context_p, + mctx: mtmd_context_p, + /, +) -> bool: + """ + return true if model can be used for chat + """ + ... + +# // +# // Audio generation helpers +# // (early-stage experimental, subjected to breaking changes) +# // + +# // audio generation helper context +# // contains accumulator for generated audio features and PCM audio +# struct mtmd_helper_gen_audio { +# std::unique_ptr pipeline; +# }; +# typedef struct mtmd_helper_gen_audio mtmd_helper_gen_audio; +mtmd_helper_gen_audio_p = NewType("mtmd_helper_gen_audio_p", int) +mtmd_helper_gen_audio_p_ctypes = c_void_p + +# enum mtmd_helper_gen_audio_outtype { +# MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM, // raw PCM +# MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono +# }; +class mtmd_helper_gen_audio_outtype(enum.IntEnum): + MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM = 0 # raw PCM + MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV = 1 # WAV PCM 16-bit LE, mono + +# struct mtmd_helper_gen_audio_inp { +# llama_seq_id seq_id; +# const char * prompt; +# size_t prompt_len; +# mtmd_bitmap * speaker_ref; // optional, can be NULL +# const char * lang; // optional, can be NULL +# int32_t top_k; +# float top_p; +# uint32_t seed; // UINT32_MAX for random (default: random) +# enum mtmd_helper_gen_audio_outtype out_type; +# }; +class mtmd_helper_gen_audio_inp(Structure): + _fields_ = [ + ("seq_id", c_int32), + ("prompt", c_char_p), + ("prompt_len", c_size_t), + ("speaker_ref", mtmd_bitmap_p_ctypes), + ("lang", c_char_p), + ("top_k", c_int32), + ("top_p", c_float), + ("seed", c_uint32), + ("out_type", c_int), + ] + + if TYPE_CHECKING: + seq_id: int + prompt: bytes + prompt_len: int + speaker_ref: mtmd_bitmap_p + lang: Optional[bytes] + top_k: int + top_p: float + seed: int + out_type: int + +# MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( +# struct llama_context * lctx, +# struct mtmd_context * mctx); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_init", + [ + llama_cpp_lib.llama_context_p_ctypes, + mtmd_context_p_ctypes, + ], + mtmd_helper_gen_audio_p_ctypes, +) +def mtmd_helper_gen_audio_init( + lctx: llama_cpp_lib.llama_context_p, + mctx: mtmd_context_p, + /, +) -> mtmd_helper_gen_audio_p: + """ + Initialize the experimental audio generation helper context. + """ + ... + +# MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_free", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_free( + ctx: mtmd_helper_gen_audio_p, + /, +): + ... + +# MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_reset", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_reset( + ctx: mtmd_helper_gen_audio_p, + /, +): + ... + +# MTMD_API int32_t mtmd_helper_gen_audio_set_input( +# mtmd_helper_gen_audio * ctx, +# const struct mtmd_helper_gen_audio_inp * inp); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_set_input", + [ + mtmd_helper_gen_audio_p_ctypes, + POINTER(mtmd_helper_gen_audio_inp), + ], + c_int32, +) +def mtmd_helper_gen_audio_set_input( + ctx: mtmd_helper_gen_audio_p, + inp: POINTER(mtmd_helper_gen_audio_inp), # type: ignore + /, +) -> c_int32: + ... + +# // processes at most n_batch prompt tokens per call +# // returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error +# MTMD_API int32_t mtmd_helper_gen_audio_step_prompt( +# mtmd_helper_gen_audio * ctx, +# int32_t n_batch); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_step_prompt", + [ + mtmd_helper_gen_audio_p_ctypes, + c_int32, + ], + c_int32, +) +def mtmd_helper_gen_audio_step_prompt( + ctx: mtmd_helper_gen_audio_p, + n_batch: c_int32, + /, +) -> c_int32: + """ + Process at most n_batch prompt tokens per call. + Returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error + """ + ... + +# // generates one frame; must only be called after step_prompt() has returned 0 +# // sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token +# // out_stop (optional) is set on end-of-speech, the caller must then stop the loop +# // h_state_out is valid until next step_gen() or reset() call, null if no frame is generated +# MTMD_API int32_t mtmd_helper_gen_audio_step_gen( +# mtmd_helper_gen_audio * ctx, +# llama_token sampled, +# const float * h_state_in, +# const float ** h_state_out, +# bool * out_stop); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_step_gen", + [ + mtmd_helper_gen_audio_p_ctypes, + llama_cpp_lib.llama_token, + POINTER(c_float), + POINTER(POINTER(c_float)), + POINTER(c_bool), + ], + c_int32, +) +def mtmd_helper_gen_audio_step_gen( + ctx: mtmd_helper_gen_audio_p, + sampled: llama_cpp_lib.llama_token, + h_state_in: POINTER(c_float), # type: ignore + h_state_out: POINTER(POINTER(c_float)), # type: ignore + out_stop: POINTER(c_bool), # type: ignore + /, +) -> c_int32: + """ + Generate one audio frame. + + Must only be called after mtmd_helper_gen_audio_step_prompt() returns 0. + + sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token. + + out_stop (optional) is set on end-of-speech, the caller must then stop the loop. + + h_state_out is owned by the helper context and remains valid until the + next step_gen() or reset() call, null if no frame is generated. + """ + ... + +# // out_data valid until next get_output() or reset() call +# // out_n_samples (optional, can be NULL) receives the number of generated PCM samples +# MTMD_API int32_t mtmd_helper_gen_audio_get_output( +# mtmd_helper_gen_audio * ctx, +# int32_t * out_sample_rate, +# const char ** out_data, +# size_t * out_data_len, +# int64_t * out_n_samples); +@ctypes_function_mtmd( + "mtmd_helper_gen_audio_get_output", + [ + mtmd_helper_gen_audio_p_ctypes, + POINTER(c_int32), + POINTER(c_char_p), + POINTER(c_size_t), + POINTER(c_int64), + ], + c_int32, +) +def mtmd_helper_gen_audio_get_output( + ctx: mtmd_helper_gen_audio_p, + out_sample_rate: POINTER(c_int32), # type: ignore + out_data: POINTER(c_char_p), # type: ignore + out_data_len: POINTER(c_size_t), # type: ignore + out_n_samples: POINTER(c_int64), # type: ignore + /, +) -> c_int32: + """ + Get accumulated generated audio output. + + out_data is owned by the helper context and remains valid until the next + get_output() or reset() call. + + out_n_samples (optional, can be NULL) receives the number of generated PCM samples. + """ + ... From 9acda8b4b35482d9b2dac9e191bbb9880ddf094e Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sat, 15 Aug 2026 08:23:17 +0800 Subject: [PATCH 28/29] Update Submodule vendor/llama.cpp 9c5531e..ad1de39 Signed-off-by: JamePeng --- vendor/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 9c5531e2b..ad1de39e0 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 9c5531e2bf2c95e86eeac807d7109de29266ca7b +Subproject commit ad1de39e0708e3ced9c71bb3c82d93a2c046a73f From 4854c7d305650b6bc9cf2dc805931a5bf2e40dd0 Mon Sep 17 00:00:00 2001 From: JamePeng Date: Sun, 16 Aug 2026 00:10:26 +0800 Subject: [PATCH 29/29] Bump version to 0.3.47 - Version 0.3.47 is a small iterative update focused on keeping the Python bindings synchronized with recent llama.cpp changes, especially around MTMD audio generation, sampler backends, and model state management. Signed-off-by: JamePeng --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ llama_cpp/__init__.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d5800c0..fe70cdd6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.47] Multi-Output Sampling, Pocket TTS and audio helper API Bindings, and Llama State Reset Improvements + +- feat(mtmd): sync Pocket TTS and audio helper API bindings + - add Pocket TTS audio generation types and fields + - update generated-audio structures for the latest MTMD ABI + - expose default generation parameters and audio helper APIs + - add multimodal chat capability detection + +- fix(llama): fully clear model state on reset + - Clear native context memory and invalidate hybrid checkpoints to keep + Python state synchronized across standard, recurrent, and hybrid models. + +- fix(internals): disable new backend hooks for custom samplers + - Explicitly set backend_reset and copy_state to NULL + - Clarify CPU callback behavior for CustomSampler + - Document inherited backend behavior in ReasoningBudgetSampler + +- feat(llama): add multi-output backend sampler API support + - expose per-sequence output limits in context parameters + - sync sampler reset and state-copy interfaces with llama.cpp + - preserve advanced context settings when reconstructing Llama instances + - document ordered multi-output sampling behavior + - fix(types): use size_t for sampler count + +- feat: Update llama.cpp to [ggml-org/llama.cpp/commit/ad1de39e0708e3ced9c71bb3c82d93a2c046a73f](https://github.com/ggml-org/llama.cpp/commit/ad1de39e0708e3ced9c71bb3c82d93a2c046a73f) + +- feat: Sync llama.cpp llama/mtmd/ggml API Binding 20260813 + +More information see: https://github.com/JamePeng/llama-cpp-python/compare/81190b03f6d177988112dad5fc919491a77705d1...9acda8b4b35482d9b2dac9e191bbb9880ddf094e + ## [0.3.46] Extended Model APIs, MTMD Binding Updates, and Improved Runtime Compatibility - feat(mtmd): update bindings for audio generation and chunk serialization diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index d3fec3b86..89e056542 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.46" +__version__ = "0.3.47"