From 791661917a04e95b2c41c22dd2a4747ab66df874 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 5 Jul 2026 12:23:45 -0700 Subject: [PATCH 1/7] feat: update llama.cpp to 78d2f5246 (#2327) * feat: update llama.cpp to 78d2f5246 * fix: align llama.cpp binding comments * fix: align llama.cpp binding docstrings (#2328) * fix: align llama.cpp binding docstrings * fix: use None in Python binding docstrings * fix: format binding docstring --- CHANGELOG.md | 2 ++ llama_cpp/llama_cpp.py | 47 ++++++++++++++++++++++++++++++++++-------- vendor/llama.cpp | 2 +- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b5e88578..5aa71dcdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@78d2f5246 + ## [0.3.32] - feat(example): support chained NextN heads for server MTP drafting diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 176709d96..d703a2fcd 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1272,6 +1272,14 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: ... +# // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" +# LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); +@ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) +def llama_ftype_name(ftype: int, /) -> Optional[bytes]: + '''Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"''' + ... + + # // Initialize the llama + ggml backend # // If numa is true, use NUMA optimizations # // Call once at the start of the program @@ -1777,7 +1785,8 @@ def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: ... # LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); @ctypes_function("llama_model_n_cls_out", [llama_model_p_ctypes], ctypes.c_uint32) def llama_model_n_cls_out(model: llama_model_p, /) -> int: - """Returns the number of classifier outputs (only valid for classifier models)""" + """Returns the number of classifier outputs (only valid for classifier models) + Undefined behavior for non-classifier models""" ... @@ -1843,7 +1852,7 @@ def llama_model_meta_count(model: llama_model_p, /) -> int: # LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key); @ctypes_function("llama_model_meta_key_str", [ctypes.c_int], ctypes.c_char_p) def llama_model_meta_key_str(key: int, /) -> Optional[bytes]: - """Get sampling metadata key name. Returns None if the key is invalid.""" + """Get sampling metadata key name. Returns None if the key is invalid""" ... @@ -1910,6 +1919,14 @@ def llama_model_desc( ... +# // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 +# LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); +@ctypes_function("llama_model_ftype", [llama_model_p_ctypes], ctypes.c_int) +def llama_model_ftype(model: llama_model_p, /) -> int: + """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0""" + ... + + # // Returns the total size of all the tensors in the model in bytes # LLAMA_API uint64_t llama_model_size(const struct llama_model * model); @ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64) @@ -2485,7 +2502,9 @@ def llama_memory_can_shift(mem: llama_memory_t, /) -> bool: # LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_state_get_size(ctx: llama_context_p, /) -> int: - """Returns the *actual* size in bytes of the state (logits, embedding and memory)""" + """Returns the *actual* size in bytes of the state + (logits, embedding and memory) + Only use when saving the state, not when restoring it, otherwise the size may be too small.""" ... @@ -3046,9 +3065,12 @@ def llama_batch_free(batch: llama_batch, /): # struct llama_batch batch); @ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: - """Process a batch of tokens using the encoder. + """Process a batch of tokens. + In contrast to llama_decode() - this call does not use KV cache. + For encode-decoder contexts, processes the batch using the encoder. + Can store the encoder output internally for later use by the decoder's cross-attention layers. 0 - success - < 0 - error""" + < 0 - error. the memory state is restored to the state before this call""" ... @@ -3070,9 +3092,15 @@ def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: @ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int: """Process a batch of tokens. + Requires the context to have a memory. + For encode-decoder contexts, processes the batch using the decoder. + Positive return values does not mean a fatal error, but rather a warning. + Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context + To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + Upon other return values, the memory state is restored to the state before this call 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) - 2 - aborted (processed ubatches will remain in the context's memory) + 2 - aborted (processed ubatches will remain in the context's memory) -1 - invalid input batch < -1 - fatal error (processed ubatches will remain in the context's memory)""" ... @@ -3108,7 +3136,7 @@ def llama_set_n_threads( # LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); @ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads(ctx: llama_context_p, /) -> int: - """Get the number of threads used for generation of a single token""" + """Get the number of threads used for generation of a single token.""" ... @@ -3116,7 +3144,7 @@ def llama_n_threads(ctx: llama_context_p, /) -> int: # LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); @ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads_batch(ctx: llama_context_p, /) -> int: - """Get the number of threads used for prompt and batch processing (multiple token)""" + """Get the number of threads used for prompt and batch processing (multiple token).""" ... @@ -3125,7 +3153,8 @@ def llama_n_threads_batch(ctx: llama_context_p, /) -> int: # LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); @ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None) def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /): - """Set whether the context outputs embeddings or not""" + """Set whether the context outputs embeddings or not + TODO: rename to avoid confusion with llama_get_embeddings()""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b3fed31b9..78d2f5246 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b3fed31b99f9bd37725833674252bccb429bb183 +Subproject commit 78d2f524682d9fee790a6460c93d018dafeb5229 From e894f0d6010be8de14400359c10c87c16ddb3829 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 5 Jul 2026 15:43:11 -0700 Subject: [PATCH 2/7] chore: bump version to 0.3.33 (#2329) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aa71dcdb..381e953bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.33] + - feat: update llama.cpp to ggml-org/llama.cpp@78d2f5246 ## [0.3.32] diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 45f9c8f27..ff142e6bf 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.32" +__version__ = "0.3.33" From 7e3e8897e128e4b8bd9864c51902964baa62b3bc Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 11 Jul 2026 21:05:39 -0700 Subject: [PATCH 3/7] feat: update llama.cpp to e3546c794 (#2338) * feat: update llama.cpp to 13f2b28b0 * feat: update llama.cpp to e3546c794 --- CHANGELOG.md | 2 ++ llama_cpp/llama_cpp.py | 8 ++++++-- vendor/llama.cpp | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 381e953bc..4dd18d2f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 + ## [0.3.33] - feat: update llama.cpp to ggml-org/llama.cpp@78d2f5246 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d703a2fcd..64399fe31 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -89,7 +89,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # GGML_TYPE_MXFP4 = 39, # GGML_TYPE_NVFP4 = 40, # GGML_TYPE_Q1_0 = 41, -# GGML_TYPE_COUNT = 42, +# GGML_TYPE_Q2_0 = 42, +# GGML_TYPE_COUNT = 43, # }; GGML_TYPE_F32 = 0 GGML_TYPE_F16 = 1 @@ -122,7 +123,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: GGML_TYPE_MXFP4 = 39 GGML_TYPE_NVFP4 = 40 GGML_TYPE_Q1_0 = 41 -GGML_TYPE_COUNT = 42 +GGML_TYPE_Q2_0 = 42 +GGML_TYPE_COUNT = 43 # from ggml-backend.h # typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); @@ -411,6 +413,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -452,6 +455,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_MOSTLY_NVFP4 = 39 LLAMA_FTYPE_MOSTLY_Q1_0 = 40 +LLAMA_FTYPE_MOSTLY_Q2_0 = 41 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 78d2f5246..e3546c794 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 78d2f524682d9fee790a6460c93d018dafeb5229 +Subproject commit e3546c7948e3af463d0b401e6421d5a4c2faf565 From 629bd1b333f60d24a01886c5f99019f4c7c3ea6c Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 11 Jul 2026 21:27:50 -0700 Subject: [PATCH 4/7] chore: bump version to 0.3.34 (#2337) --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dd18d2f4..629b645a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.34] + - feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 ## [0.3.33] diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index ff142e6bf..5a0a40d10 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.33" +__version__ = "0.3.34" From 67014fdf1d05ec2aae5bd574bf731182b884a03a Mon Sep 17 00:00:00 2001 From: Andrei Date: Mon, 10 Aug 2026 02:40:06 -0700 Subject: [PATCH 5/7] feat: update llama.cpp to 936918514 (#2348) * feat: update llama.cpp to d2a818231 * feat: update llama.cpp to 11924d4c1 * fix: close contexts before models * feat: update llama.cpp to fc3f10b38 * feat: update llama.cpp to 42e98813e * feat: update llama.cpp to 936918514 --- CHANGELOG.md | 2 + .../low_level_api/low_level_api_chat_cpp.py | 10 +- examples/server/server.py | 22 +- llama_cpp/_internals.py | 6 +- llama_cpp/llama.py | 25 +- llama_cpp/llama_chat_format.py | 8 +- llama_cpp/llama_cpp.py | 91 +++-- llama_cpp/mtmd_cpp.py | 364 ++++++++++++++++++ tests/test_llama.py | 13 +- vendor/llama.cpp | 2 +- 10 files changed, 500 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 629b645a5..76c1ac272 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@936918514 + ## [0.3.34] - feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 diff --git a/examples/low_level_api/low_level_api_chat_cpp.py b/examples/low_level_api/low_level_api_chat_cpp.py index 20f7a158a..43b0b31bb 100644 --- a/examples/low_level_api/low_level_api_chat_cpp.py +++ b/examples/low_level_api/low_level_api_chat_cpp.py @@ -76,8 +76,14 @@ def __init__(self, params: GptParams) -> None: self.lparams.n_parts = self.params.n_parts self.lparams.seed = self.params.seed self.lparams.memory_f16 = self.params.memory_f16 - self.lparams.use_mlock = self.params.use_mlock - self.lparams.use_mmap = self.params.use_mmap + if self.params.use_mmap and self.params.use_mlock: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif self.params.use_mlock: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif self.params.use_mmap: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + self.lparams.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE self.model = llama_cpp.llama_model_load_from_file( self.params.model.encode("utf8"), self.lparams diff --git a/examples/server/server.py b/examples/server/server.py index 72adc7905..452654ed1 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -10299,6 +10299,7 @@ def __init__( llama_cpp.llama_sampler_chain_add( self._sampler, llama_cpp.llama_sampler_init_penalties( + n_vocab, 64, 1.0, frequency_penalty, @@ -11030,7 +11031,9 @@ def _build_prompt_plan_locked( "multiple videos require MTMD to report frame counts" ) input_text = mtmd_cpp.mtmd_input_text() - input_text.text = prompt.encode("utf-8") + input_text_bytes = prompt.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = False input_text.parse_special = True chunks = mtmd_cpp.mtmd_input_chunks_init() @@ -11310,6 +11313,7 @@ def __init__( vocab_only=vocab_only, use_mmap=use_mmap, use_mlock=use_mlock, + load_mtp=draft_model == "draft-mtp", kv_overrides=kv_overrides, ) ) @@ -11588,6 +11592,7 @@ def build_model_params( vocab_only: Optional[bool], use_mmap: Optional[bool], use_mlock: Optional[bool], + load_mtp: bool, kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]], ) -> Tuple[Any, Optional[Any], Optional[Any]]: model_params = llama_cpp.llama_model_default_params() @@ -11609,10 +11614,17 @@ def build_model_params( model_params.tensor_split = tensor_split_ref if vocab_only is not None: model_params.vocab_only = vocab_only - if use_mmap is not None: - model_params.use_mmap = use_mmap - if use_mlock is not None: - model_params.use_mlock = use_mlock + model_params.load_mtp = load_mtp + if use_mlock and use_mmap is not False: + model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif use_mlock: + model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif use_mmap is not None: + model_params.load_mode = ( + llama_cpp.LLAMA_LOAD_MODE_MMAP + if use_mmap + else llama_cpp.LLAMA_LOAD_MODE_NONE + ) kv_overrides_ref = None if kv_overrides is not None: diff --git a/llama_cpp/_internals.py b/llama_cpp/_internals.py index b0fe94d01..b45d34b2c 100644 --- a/llama_cpp/_internals.py +++ b/llama_cpp/_internals.py @@ -276,6 +276,8 @@ def free_ctx(): self.ctx = None self._exit_stack.callback(free_ctx) + # The native context must be freed before its model. + self.model._exit_stack.callback(self.close) def close(self): self._exit_stack.close() @@ -784,12 +786,14 @@ def add_grammar_lazy_patterns( def add_penalties( self, + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, penalty_present: float, ): sampler = llama_cpp.llama_sampler_init_penalties( + n_vocab, penalty_last_n, penalty_repeat, penalty_freq, @@ -800,7 +804,6 @@ def add_penalties( def add_dry( self, model: LlamaModel, - n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, @@ -814,7 +817,6 @@ def add_dry( sampler = llama_cpp.llama_sampler_init_dry( model.vocab, - n_ctx_train, dry_multiplier, dry_base, dry_allowed_length, diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index b5bffd46b..14e2f8500 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -244,8 +244,15 @@ def __init__( ) # keep a reference to the array so it is not gc'd self.model_params.tensor_split = self._c_tensor_split self.model_params.vocab_only = vocab_only - self.model_params.use_mmap = use_mmap if lora_path is None else False - self.model_params.use_mlock = use_mlock + use_mmap = use_mmap and lora_path is None + if use_mmap and use_mlock: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif use_mlock: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif use_mmap: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + self.model_params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE # kv_overrides is the original python dict self.kv_overrides = kv_overrides @@ -737,7 +744,7 @@ def apply_func(token_data_array: llama_cpp.llama_token_data_array_p): sampler.add_custom(apply_func) sampler.add_penalties( - # n_vocab=self._n_vocab, + n_vocab=self._n_vocab, # special_eos_id=self._token_eos, # linefeed_id=self._token_nl, penalty_last_n=self.last_n_tokens_size, @@ -2142,8 +2149,16 @@ def __getstate__(self): main_gpu=self.model_params.main_gpu, tensor_split=self.tensor_split, vocab_only=self.model_params.vocab_only, - use_mmap=self.model_params.use_mmap, - use_mlock=self.model_params.use_mlock, + use_mmap=self.model_params.load_mode + in ( + llama_cpp.LLAMA_LOAD_MODE_MMAP, + llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK, + ), + use_mlock=self.model_params.load_mode + in ( + llama_cpp.LLAMA_LOAD_MODE_MLOCK, + llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK, + ), kv_overrides=self.kv_overrides, # Context Params seed=self._seed, diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 0034bdae9..4f41c2eb7 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -2936,7 +2936,9 @@ def __call__( # Create input text structure input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode("utf-8") + input_text_bytes = text.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = True input_text.parse_special = True @@ -3485,7 +3487,9 @@ def raise_exception(message: str): bitmap_cleanup.append(bitmap) input_text = self._mtmd_cpp.mtmd_input_text() - input_text.text = text.encode("utf-8") + input_text_bytes = text.encode("utf-8") + input_text.text = input_text_bytes + input_text.text_len = len(input_text_bytes) input_text.add_special = True input_text.parse_special = True diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 64399fe31..362516b4d 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -520,6 +520,20 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_SPLIT_MODE_TENSOR = 3 +# enum llama_load_mode { +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# }; +LLAMA_LOAD_MODE_NONE = 0 +LLAMA_LOAD_MODE_MMAP = 1 +LLAMA_LOAD_MODE_MLOCK = 2 +LLAMA_LOAD_MODE_MMAP_MLOCK = 3 +LLAMA_LOAD_MODE_DIRECT_IO = 4 + + # enum llama_context_type { # LLAMA_CONTEXT_TYPE_DEFAULT = 0, # LLAMA_CONTEXT_TYPE_MTP = 1, @@ -789,8 +803,9 @@ class llama_model_imatrix_data(ctypes.Structure): # // NULL-terminated list of buffer types to use for tensors that match a pattern # const struct llama_model_tensor_buft_override * tensor_buft_overrides; -# int32_t n_gpu_layers; // number of layers to store in VRAM +# int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers # enum llama_split_mode split_mode; // how to split the model across multiple GPUs +# enum llama_load_mode load_mode; // how to load the model # // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE # int32_t main_gpu; @@ -812,13 +827,11 @@ class llama_model_imatrix_data(ctypes.Structure): # // Keep the booleans together to avoid misalignment during copy-by-value. # bool vocab_only; // only load the vocabulary, no weights -# bool use_mmap; // use mmap if possible -# bool use_direct_io; // use direct io, takes precedence over use_mmap when supported -# bool use_mlock; // force system to keep model in RAM # bool check_tensors; // validate model tensor data # bool use_extra_bufts; // use extra buffer types (used for weight repacking) # bool no_host; // bypass host buffer allowing extra buffers to be used # bool no_alloc; // only load metadata and simulate memory allocations +# bool load_mtp; // whether to load MTP layers # }; class llama_model_params(ctypes.Structure): """Parameters for llama_model @@ -826,21 +839,20 @@ class llama_model_params(ctypes.Structure): Attributes: devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern - n_gpu_layers (int): number of layers to store in VRAM + n_gpu_layers (int): number of layers to store in VRAM, a negative value means all layers split_mode (int): how to split the model across multiple GPUs + load_mode (int): how to load the model main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted. progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data vocab_only (bool): only load the vocabulary, no weights - use_mmap (bool): use mmap if possible - use_direct_io (bool): use direct io, takes precedence over use_mmap when supported - use_mlock (bool): force system to keep model in RAM check_tensors (bool): validate model tensor data use_extra_bufts (bool): use extra buffer types (used for weight repacking) no_host (bool): bypass host buffer allowing extra buffers to be used - no_alloc (bool): only load metadata and simulate memory allocations""" + no_alloc (bool): only load metadata and simulate memory allocations + load_mtp (bool): whether to load MTP layers""" if TYPE_CHECKING: devices: CtypesArray[ctypes.c_void_p] # NOTE: unused @@ -849,38 +861,36 @@ class llama_model_params(ctypes.Structure): ] # NOTE: unused n_gpu_layers: int split_mode: int + load_mode: int main_gpu: int tensor_split: CtypesArray[ctypes.c_float] progress_callback: Callable[[float, ctypes.c_void_p], bool] progress_callback_user_data: ctypes.c_void_p kv_overrides: CtypesArray[llama_model_kv_override] vocab_only: bool - use_mmap: bool - use_direct_io: bool - use_mlock: bool check_tensors: bool use_extra_bufts: bool no_host: bool no_alloc: bool + load_mtp: bool _fields_ = [ ("devices", ctypes.c_void_p), # NOTE: unnused ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused ("n_gpu_layers", ctypes.c_int32), ("split_mode", ctypes.c_int), + ("load_mode", ctypes.c_int), ("main_gpu", ctypes.c_int32), ("tensor_split", ctypes.POINTER(ctypes.c_float)), ("progress_callback", llama_progress_callback), ("progress_callback_user_data", ctypes.c_void_p), ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), ("vocab_only", ctypes.c_bool), - ("use_mmap", ctypes.c_bool), - ("use_direct_io", ctypes.c_bool), - ("use_mlock", ctypes.c_bool), ("check_tensors", ctypes.c_bool), ("use_extra_bufts", ctypes.c_bool), ("no_host", ctypes.c_bool), ("no_alloc", ctypes.c_bool), + ("load_mtp", ctypes.c_bool), ] @@ -1276,6 +1286,20 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: ... +# LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); +@ctypes_function("llama_load_mode_name", [ctypes.c_int], ctypes.c_char_p) +def llama_load_mode_name(load_mode: int, /) -> Optional[bytes]: + """Get the model load mode name.""" + ... + + +# LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str); +@ctypes_function("llama_load_mode_from_str", [ctypes.c_char_p], ctypes.c_int) +def llama_load_mode_from_str(value: bytes, /) -> int: + """Get the model load mode from a string.""" + ... + + # // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" # LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); @ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) @@ -3531,6 +3555,20 @@ def llama_vocab_get_add_eos(vocab: llama_vocab_p, /) -> bool: ... def llama_vocab_get_add_sep(vocab: llama_vocab_p, /) -> bool: ... +# // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) +# LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens); +@ctypes_function( + "llama_vocab_get_suppress_tokens", + [llama_vocab_p_ctypes, ctypes.POINTER(ctypes.c_int32)], + ctypes.POINTER(llama_token), +) +def llama_vocab_get_suppress_tokens( + vocab: llama_vocab_p, + n_suppress_tokens: CtypesPointer[ctypes.c_int32], + /, +) -> Optional[CtypesPointer[llama_token]]: ... + + # LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); @ctypes_function( "llama_vocab_fim_pre", @@ -4686,16 +4724,24 @@ def llama_sampler_init_grammar_lazy_patterns( # /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. # LLAMA_API struct llama_sampler * llama_sampler_init_penalties( -# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) -# float penalty_repeat, // 1.0 = disabled -# float penalty_freq, // 0.0 = disabled -# float penalty_present); // 0.0 = disabled +# int32_t n_vocab, +# int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty) +# float penalty_repeat, // must be > 0.0, 1.0 = disabled +# float penalty_freq, // must be finite, 0.0 = disabled +# float penalty_present); // must be finite, 0.0 = disabled @ctypes_function( "llama_sampler_init_penalties", - [ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_float], + [ + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_float, + ctypes.c_float, + ctypes.c_float, + ], llama_sampler_p_ctypes, ) def llama_sampler_init_penalties( + n_vocab: int, penalty_last_n: int, penalty_repeat: float, penalty_freq: float, @@ -4707,18 +4753,16 @@ def llama_sampler_init_penalties( # /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 # LLAMA_API struct llama_sampler * llama_sampler_init_dry( # const struct llama_vocab * vocab, -# int32_t n_ctx_train, # float dry_multiplier, # float dry_base, # int32_t dry_allowed_length, -# int32_t dry_penalty_last_n, +# int32_t dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty) # const char ** seq_breakers, # size_t num_breakers); @ctypes_function( "llama_sampler_init_dry", [ llama_vocab_p_ctypes, - ctypes.c_int32, ctypes.c_float, ctypes.c_float, ctypes.c_int32, @@ -4730,7 +4774,6 @@ def llama_sampler_init_penalties( ) def llama_sampler_init_dry( vocab: llama_vocab_p, - n_ctx_train: int, dry_multiplier: float, dry_base: float, dry_allowed_length: int, diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 35357a327..101a7db2b 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -5,8 +5,10 @@ from ctypes import ( CFUNCTYPE, c_bool, + c_char, c_char_p, c_int, + c_int32, c_int64, c_uint8, c_uint32, @@ -68,6 +70,9 @@ mtmd_helper_video_p = NewType("mtmd_helper_video_p", int) mtmd_helper_video_p_ctypes = c_void_p +mtmd_helper_gen_audio_p = NewType("mtmd_helper_gen_audio_p", int) +mtmd_helper_gen_audio_p_ctypes = c_void_p + mtmd_image_tokens_p = NewType("mtmd_image_tokens_p", int) mtmd_image_tokens_p_ctypes = c_void_p @@ -84,6 +89,16 @@ MTMD_INPUT_CHUNK_TYPE_TEXT = 0 MTMD_INPUT_CHUNK_TYPE_IMAGE = 1 MTMD_INPUT_CHUNK_TYPE_AUDIO = 2 +MTMD_INPUT_CHUNK_TYPE_COUNT = 3 + +MTMD_GEN_AUDIO_TYPE_NONE = 0 +MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 + +MTMD_GEN_PROCESS_TYPE_GEN_CODE = 0 +MTMD_GEN_PROCESS_TYPE_GEN_WAV = 1 + +MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM = 0 +MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV = 1 mtmd_progress_callback = CFUNCTYPE(c_bool, c_float, c_void_p) @@ -133,8 +148,15 @@ class mtmd_context_params(Structure): class mtmd_input_text(Structure): """Text input passed to `mtmd_tokenize`.""" + if TYPE_CHECKING: + text: Optional[bytes] + text_len: int + add_special: bool + parse_special: bool + _fields_ = [ ("text", c_char_p), + ("text_len", c_size_t), ("add_special", c_bool), ("parse_special", c_bool), ] @@ -168,6 +190,97 @@ class mtmd_caps(Structure): ] +# struct mtmd_gen_audio_info { +# enum mtmd_gen_audio_type type; +# int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +# }; +class mtmd_gen_audio_info(Structure): + if TYPE_CHECKING: + type: int + sample_rate: int + + _fields_ = [ + ("type", c_int), + ("sample_rate", c_int32), + ] + + +# struct mtmd_gen_inp { +# enum mtmd_gen_process_type type; +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# int32_t code0; // the sampled codebook 0 entry from backbone +# float * embd; // the hidden state from backbone, must have n_text_embd elements +# int32_t top_k; +# float top_p; +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# int32_t * codes; +# size_t n_codes; +# const char * state_data; +# size_t state_size; +# }; +class mtmd_gen_inp(Structure): + if TYPE_CHECKING: + type: int + code0: int + embd: Optional["_Pointer[c_float]"] + top_k: int + top_p: float + codes: Optional["_Pointer[c_int32]"] + n_codes: int + state_data: Optional["_Pointer[c_char]"] + state_size: int + + _fields_ = [ + ("type", c_int), + ("code0", c_int32), + ("embd", POINTER(c_float)), + ("top_k", c_int32), + ("top_p", c_float), + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("state_data", POINTER(c_char)), + ("state_size", c_size_t), + ] + + +# struct mtmd_gen_out { +# // note: output memory is allocated by the context, valid until next process() call +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_CODE +# const int32_t * codes; +# size_t n_codes; +# const float * embd; // the generated hidden state, to be fed back to backbone +# // it must have n_text_embd elements +# +# // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# const float * audio; +# size_t n_samples; +# const char * state_data; +# size_t state_size; +# }; +class mtmd_gen_out(Structure): + if TYPE_CHECKING: + codes: Optional["_Pointer[c_int32]"] + n_codes: int + embd: Optional["_Pointer[c_float]"] + audio: Optional["_Pointer[c_float]"] + n_samples: int + state_data: Optional["_Pointer[c_char]"] + state_size: int + + _fields_ = [ + ("codes", POINTER(c_int32)), + ("n_codes", c_size_t), + ("embd", POINTER(c_float)), + ("audio", POINTER(c_float)), + ("n_samples", c_size_t), + ("state_data", POINTER(c_char)), + ("state_size", c_size_t), + ] + + mtmd_bitmap_lazy_callback = CFUNCTYPE( c_int, c_size_t, @@ -228,6 +341,43 @@ class mtmd_helper_video_init_params(Structure): ] +# struct mtmd_helper_gen_audio_inp { +# llama_seq_id seq_id; +# +# const char * prompt; +# size_t prompt_len; +# +# mtmd_bitmap * speaker_ref; // optional, can be NULL +# const char * lang; // optional, can be NULL +# +# int32_t top_k; +# float top_p; +# +# enum mtmd_helper_gen_audio_outtype out_type; +# }; +class mtmd_helper_gen_audio_inp(Structure): + if TYPE_CHECKING: + seq_id: int + prompt: Optional[bytes] + prompt_len: int + speaker_ref: Optional[mtmd_bitmap_p] + lang: Optional[bytes] + top_k: int + top_p: float + out_type: int + + _fields_ = [ + ("seq_id", llama_cpp.llama_seq_id), + ("prompt", c_char_p), + ("prompt_len", c_size_t), + ("speaker_ref", mtmd_bitmap_p_ctypes), + ("lang", c_char_p), + ("top_k", c_int32), + ("top_p", c_float), + ("out_type", c_int), + ] + + ################################################ # mtmd.h functions ################################################ @@ -547,6 +697,44 @@ def mtmd_input_chunk_free(chunk: mtmd_input_chunk_p, /): ... +# // save/load an input chunk to/from a buffer (useful for KV save/load) +# // important: only chunk's metadata will be saved, the actual image/audio data will not be saved +# // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() +# // out_buf can be nullptr (to query expected_out_len) +# // returns 0 on success, non-zero on failure +# MTMD_API int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len); +@ctypes_function( + "mtmd_input_chunk_save", + [mtmd_input_chunk_p_ctypes, POINTER(c_char), c_size_t, POINTER(c_size_t)], + c_int32, +) +def mtmd_input_chunk_save( + chunk: mtmd_input_chunk_p, + out_buf: Optional[CtypesArray[c_char]], + out_len: Union[c_size_t, int], + expected_out_len: "_Pointer[c_size_t]", + /, +) -> int: + """Save an input chunk's metadata to a buffer.""" + ... + + +# // returns nullptr on failure +# MTMD_API mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len); +@ctypes_function( + "mtmd_input_chunk_load", + [c_char_p, c_size_t], + mtmd_input_chunk_p_ctypes, +) +def mtmd_input_chunk_load( + buf: bytes, + length: Union[c_size_t, int], + /, +) -> Optional[mtmd_input_chunk_p]: + """Load an input chunk placeholder from saved metadata.""" + ... + + # MTMD_API size_t mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); @ctypes_function( "mtmd_image_tokens_get_n_tokens", [mtmd_image_tokens_p_ctypes], c_size_t @@ -692,6 +880,36 @@ def mtmd_get_cap_from_file(mmproj_fname: bytes, /) -> mtmd_caps: ... +# MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); +@ctypes_function( + "mtmd_gen_audio_get_info", + [mtmd_context_p_ctypes], + mtmd_gen_audio_info, +) +def mtmd_gen_audio_get_info(ctx: mtmd_context_p, /) -> mtmd_gen_audio_info: + """Get audio generation information for an MTMD context.""" + ... + + +# // note: this API is stateless, caller must handle state management and audio frame accumulation +# MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, +# const struct mtmd_gen_inp * inp, +# struct mtmd_gen_out * out); +@ctypes_function( + "mtmd_gen_audio_process", + [mtmd_context_p_ctypes, POINTER(mtmd_gen_inp), POINTER(mtmd_gen_out)], + c_int32, +) +def mtmd_gen_audio_process( + ctx: mtmd_context_p, + inp: "_Pointer[mtmd_gen_inp]", + out: "_Pointer[mtmd_gen_out]", + /, +) -> int: + """Process one audio generation step.""" + ... + + # MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void); @ctypes_function("mtmd_test_create_input_chunks", [], mtmd_input_chunks_p_ctypes) def mtmd_test_create_input_chunks() -> Optional[mtmd_input_chunks_p]: @@ -997,6 +1215,152 @@ def mtmd_helper_video_read_next( ... +# // return true if model can be used for chat +# MTMD_API bool mtmd_helper_model_can_chat(struct llama_context * lctx, struct mtmd_context * mctx); +@ctypes_function( + "mtmd_helper_model_can_chat", + [llama_cpp.llama_context_p_ctypes, mtmd_context_p_ctypes], + c_bool, +) +def mtmd_helper_model_can_chat( + lctx: llama_cpp.llama_context_p, + mctx: mtmd_context_p, + /, +) -> bool: + """Return whether the model can be used for chat.""" + ... + + +# MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( +# struct llama_context * lctx, +# struct mtmd_context * mctx); +@ctypes_function( + "mtmd_helper_gen_audio_init", + [llama_cpp.llama_context_p_ctypes, mtmd_context_p_ctypes], + mtmd_helper_gen_audio_p_ctypes, +) +def mtmd_helper_gen_audio_init( + lctx: llama_cpp.llama_context_p, + mctx: mtmd_context_p, + /, +) -> Optional[mtmd_helper_gen_audio_p]: + """Initialize an audio generation helper context.""" + ... + + +# MTMD_API void mtmd_helper_gen_audio_free(mtmd_helper_gen_audio * ctx); +@ctypes_function( + "mtmd_helper_gen_audio_free", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_free(ctx: mtmd_helper_gen_audio_p, /): ... + + +# MTMD_API void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx); +@ctypes_function( + "mtmd_helper_gen_audio_reset", + [mtmd_helper_gen_audio_p_ctypes], + None, +) +def mtmd_helper_gen_audio_reset(ctx: mtmd_helper_gen_audio_p, /): ... + + +# MTMD_API int32_t mtmd_helper_gen_audio_set_input( +# mtmd_helper_gen_audio * ctx, +# const struct mtmd_helper_gen_audio_inp * inp); +@ctypes_function( + "mtmd_helper_gen_audio_set_input", + [mtmd_helper_gen_audio_p_ctypes, POINTER(mtmd_helper_gen_audio_inp)], + c_int32, +) +def mtmd_helper_gen_audio_set_input( + ctx: mtmd_helper_gen_audio_p, + inp: "_Pointer[mtmd_helper_gen_audio_inp]", + /, +) -> int: + """Set the audio generation helper input.""" + ... + + +# // processes at most n_batch prompt tokens per call +# // returns: >0 = number of prompt tokens remaining, 0 = done, <0 = error +# MTMD_API int32_t mtmd_helper_gen_audio_step_prompt( +# mtmd_helper_gen_audio * ctx, +# int32_t n_batch); +@ctypes_function( + "mtmd_helper_gen_audio_step_prompt", + [mtmd_helper_gen_audio_p_ctypes, c_int32], + c_int32, +) +def mtmd_helper_gen_audio_step_prompt( + ctx: mtmd_helper_gen_audio_p, + n_batch: int, + /, +) -> int: + """Process up to n_batch prompt tokens.""" + ... + + +# // generates one frame; must only be called after step_prompt() has returned 0 +# // h_state_out is valid until next step_gen() or reset() call +# MTMD_API int32_t mtmd_helper_gen_audio_step_gen( +# mtmd_helper_gen_audio * ctx, +# llama_token sampled, +# const float * h_state_in, +# const float ** h_state_out); +@ctypes_function( + "mtmd_helper_gen_audio_step_gen", + [ + mtmd_helper_gen_audio_p_ctypes, + llama_cpp.llama_token, + POINTER(c_float), + POINTER(POINTER(c_float)), + ], + c_int32, +) +def mtmd_helper_gen_audio_step_gen( + ctx: mtmd_helper_gen_audio_p, + sampled: llama_cpp.llama_token, + h_state_in: Optional["_Pointer[c_float]"], + h_state_out: "_Pointer[_Pointer[c_float]]", + /, +) -> int: + """Generate one audio frame.""" + ... + + +# // out_data valid until next get_output() or reset() call +# // out_n_samples (optional, can be NULL) receives the number of generated PCM samples +# MTMD_API int32_t mtmd_helper_gen_audio_get_output( +# mtmd_helper_gen_audio * ctx, +# int32_t * out_sample_rate, +# const char ** out_data, +# size_t * out_data_len, +# int64_t * out_n_samples); +@ctypes_function( + "mtmd_helper_gen_audio_get_output", + [ + mtmd_helper_gen_audio_p_ctypes, + POINTER(c_int32), + POINTER(POINTER(c_char)), + POINTER(c_size_t), + POINTER(c_int64), + ], + c_int32, +) +def mtmd_helper_gen_audio_get_output( + ctx: mtmd_helper_gen_audio_p, + out_sample_rate: "_Pointer[c_int32]", + out_data: "_Pointer[_Pointer[c_char]]", + out_data_len: "_Pointer[c_size_t]", + out_n_samples: Optional["_Pointer[c_int64]"], + /, +) -> int: + """Get accumulated PCM or WAV audio output.""" + ... + + # MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data); @ctypes_function( "mtmd_log_set", diff --git a/tests/test_llama.py b/tests/test_llama.py index 70fce12d8..c1c16e30c 100644 --- a/tests/test_llama.py +++ b/tests/test_llama.py @@ -103,8 +103,14 @@ def test_real_model(llama_cpp_model_path): assert os.path.exists(llama_cpp_model_path) params = llama_cpp.llama_model_default_params() - params.use_mmap = llama_cpp.llama_supports_mmap() - params.use_mlock = llama_cpp.llama_supports_mlock() + if llama_cpp.llama_supports_mmap() and llama_cpp.llama_supports_mlock(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP_MLOCK + elif llama_cpp.llama_supports_mlock(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MLOCK + elif llama_cpp.llama_supports_mmap(): + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_MMAP + else: + params.load_mode = llama_cpp.LLAMA_LOAD_MODE_NONE params.check_tensors = False model = internals.LlamaModel(path_model=llama_cpp_model_path, params=params) @@ -155,6 +161,9 @@ def test_real_model(llama_cpp_model_path): assert len(output) == 4 assert output_text + model.close() + assert context.ctx is None + def test_real_llama(llama_cpp_model_path): model = llama_cpp.Llama( diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e3546c794..936918514 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e3546c7948e3af463d0b401e6421d5a4c2faf565 +Subproject commit 936918514ce522b553c0fd80b169a6440e6096c6 From 8a0810fa04bde084bd3446d126831ae5bd65e373 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 15 Aug 2026 06:39:21 -0700 Subject: [PATCH 6/7] feat: update llama.cpp to adb55e514 (#2357) --- CHANGELOG.md | 2 +- llama_cpp/llama_cpp.py | 61 ++++++++++++++++++++++++++++++++---------- llama_cpp/mtmd_cpp.py | 56 ++++++++++++++++++++++++++++++++++---- vendor/llama.cpp | 2 +- 4 files changed, 100 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76c1ac272..1e4adffb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- feat: update llama.cpp to ggml-org/llama.cpp@936918514 +- feat: update llama.cpp to ggml-org/llama.cpp@adb55e514 ## [0.3.34] diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 362516b4d..c5387a3df 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -521,12 +521,14 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # enum llama_load_mode { -# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode -# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model -# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing -# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing -# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available +# LLAMA_LOAD_MODE_AUTO = -1, // auto-detect based on device capabilities +# LLAMA_LOAD_MODE_NONE = 0, // no special loading mode +# LLAMA_LOAD_MODE_MMAP = 1, // memory map the model +# LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing +# LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available # }; +LLAMA_LOAD_MODE_AUTO = -1 LLAMA_LOAD_MODE_NONE = 0 LLAMA_LOAD_MODE_MMAP = 1 LLAMA_LOAD_MODE_MLOCK = 2 @@ -912,14 +914,15 @@ class llama_sampler_seq_config(ctypes.Structure): # // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations # // https://github.com/ggml-org/llama.cpp/pull/7544 # struct llama_context_params { -# uint32_t n_ctx; // text context, 0 = from model -# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode -# uint32_t n_ubatch; // physical maximum batch size -# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) -# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] -# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) -# int32_t n_threads; // number of threads to use for generation -# int32_t n_threads_batch; // number of threads to use for batch processing +# uint32_t n_ctx; // text context, 0 = from model +# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode +# uint32_t n_ubatch; // physical maximum batch size +# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) +# uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL] +# uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch) +# uint32_t n_outputs_max_per_seq; // max outputs per sequence (0 = n_outputs_max) +# int32_t n_threads; // number of threads to use for generation +# int32_t n_threads_batch; // number of threads to use for batch processing # enum llama_context_type ctx_type; // set the context type (e.g. MTP) # enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` @@ -978,6 +981,7 @@ class llama_context_params(ctypes.Structure): n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models) n_rs_seq (int): number of recurrent-state snapshots per sequence for rollback n_outputs_max (int): max outputs in a ubatch, 0 = n_batch + n_outputs_max_per_seq (int): max outputs per sequence, 0 = n_outputs_max n_threads (int): number of threads to use for generation n_threads_batch (int): number of threads to use for batch processing ctx_type (int): context type, from `enum llama_context_type` @@ -1017,6 +1021,7 @@ class llama_context_params(ctypes.Structure): n_seq_max: int n_rs_seq: int n_outputs_max: int + n_outputs_max_per_seq: int n_threads: int n_threads_batch: int ctx_type: int @@ -1055,6 +1060,7 @@ class llama_context_params(ctypes.Structure): ("n_seq_max", ctypes.c_uint32), ("n_rs_seq", ctypes.c_uint32), ("n_outputs_max", ctypes.c_uint32), + ("n_outputs_max_per_seq", ctypes.c_uint32), ("n_threads", ctypes.c_int32), ("n_threads_batch", ctypes.c_int32), ("ctx_type", ctypes.c_int), @@ -1234,6 +1240,13 @@ class llama_chat_message(ctypes.Structure): llama_adapter_lora_p_ctypes = ctypes.POINTER(ctypes.c_void_p) +# LLAMA_API const char * llama_version(void); +@ctypes_function("llama_version", [], ctypes.c_char_p) +def llama_version() -> bytes: + """Get the llama.cpp version.""" + ... + + # // Helpers for getting default parameters # LLAMA_API struct llama_model_params llama_model_default_params(void); @ctypes_function( @@ -2891,6 +2904,7 @@ def llama_state_seq_save_file( ) -> int: ... +# // If tokens_out is None, only the token count is reported through n_token_count_out and no state is loaded # LLAMA_API size_t llama_state_seq_load_file( # struct llama_context * ctx, # const char * filepath, @@ -3331,6 +3345,9 @@ def llama_get_embeddings_seq( # // Get the backend sampled token for the ith token. +# // With multiple outputs, sampler state advances when the token is accepted, +# // not when it is read through this function. +# // When accepting multiple outputs, accept a contiguous prefix in output order. # // Returns LLAMA_TOKEN_NULL if no token was sampled. # LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); @ctypes_function( @@ -4317,7 +4334,7 @@ class llama_sampler(ctypes.Structure): llama_sampler_i_clone = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_sampler_p_ctypes) llama_sampler_i_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) llama_sampler_i_backend_init = ctypes.CFUNCTYPE( - ctypes.c_bool, llama_sampler_p_ctypes, ctypes.c_void_p + ctypes.c_bool, llama_sampler_p_ctypes, ctypes.c_void_p, ctypes.c_uint32 ) llama_sampler_i_backend_accept = ctypes.CFUNCTYPE( None, @@ -4334,6 +4351,10 @@ class llama_sampler(ctypes.Structure): ctypes.POINTER(llama_sampler_data), ) llama_sampler_i_backend_set_input = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i_backend_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i_copy_state = ctypes.CFUNCTYPE( + None, llama_sampler_p_ctypes, llama_sampler_p_ctypes +) llama_sampler_i._fields_ = [ ("name", llama_sampler_i_name), @@ -4346,6 +4367,8 @@ class llama_sampler(ctypes.Structure): ("backend_accept", llama_sampler_i_backend_accept), ("backend_apply", llama_sampler_i_backend_apply), ("backend_set_input", llama_sampler_i_backend_set_input), + ("backend_reset", llama_sampler_i_backend_reset), + ("copy_state", llama_sampler_i_copy_state), ] @@ -4422,6 +4445,15 @@ def llama_sampler_reset(smpl: llama_sampler_p, /): ... def llama_sampler_clone(smpl: llama_sampler_p, /) -> llama_sampler_p: ... +# LLAMA_API void llama_sampler_copy (const struct llama_sampler * src, struct llama_sampler * dst); +@ctypes_function( + "llama_sampler_copy", + [llama_sampler_p_ctypes, llama_sampler_p_ctypes], + None, +) +def llama_sampler_copy(src: llama_sampler_p, dst: llama_sampler_p, /): ... + + # // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) # LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); @ctypes_function( @@ -4835,6 +4867,7 @@ def llama_sampler_get_seed(smpl: llama_sampler_p, /) -> int: ... # /// @details Sample and accept a token from the idx-th output of the last evaluation +# // For multiple outputs from one sampler, call this function in output order without gaps. # LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); @ctypes_function( "llama_sampler_sample", diff --git a/llama_cpp/mtmd_cpp.py b/llama_cpp/mtmd_cpp.py index 101a7db2b..79f417cba 100644 --- a/llama_cpp/mtmd_cpp.py +++ b/llama_cpp/mtmd_cpp.py @@ -93,6 +93,7 @@ MTMD_GEN_AUDIO_TYPE_NONE = 0 MTMD_GEN_AUDIO_TYPE_QWEN3TTS = 1 +MTMD_GEN_AUDIO_TYPE_POCKETTTS = 2 MTMD_GEN_PROCESS_TYPE_GEN_CODE = 0 MTMD_GEN_PROCESS_TYPE_GEN_WAV = 1 @@ -193,15 +194,18 @@ class mtmd_caps(Structure): # struct mtmd_gen_audio_info { # enum mtmd_gen_audio_type type; # int32_t sample_rate; // in Hz, for example 24000 for qwen3tts +# const char * model_variant; // name of the weight variant, can be None if not applicable # }; class mtmd_gen_audio_info(Structure): if TYPE_CHECKING: type: int sample_rate: int + model_variant: Optional[bytes] _fields_ = [ ("type", c_int), ("sample_rate", c_int32), + ("model_variant", c_char_p), ] @@ -213,10 +217,15 @@ class mtmd_gen_audio_info(Structure): # float * embd; // the hidden state from backbone, must have n_text_embd elements # int32_t top_k; # float top_p; +# uint32_t seed; // UINT32_MAX for random +# float temp; // sampling temperature, or noise scale for flow-matching decoders # # // for MTMD_GEN_PROCESS_TYPE_GEN_WAV +# // pass either codes (discrete) or feats (continuous), depending on the pipeline # int32_t * codes; # size_t n_codes; +# const float * feats; +# size_t n_feats; # const char * state_data; # size_t state_size; # }; @@ -227,8 +236,12 @@ class mtmd_gen_inp(Structure): embd: Optional["_Pointer[c_float]"] top_k: int top_p: float + seed: int + temp: float codes: Optional["_Pointer[c_int32]"] n_codes: int + feats: Optional["_Pointer[c_float]"] + n_feats: int state_data: Optional["_Pointer[c_char]"] state_size: int @@ -238,8 +251,12 @@ class mtmd_gen_inp(Structure): ("embd", POINTER(c_float)), ("top_k", c_int32), ("top_p", c_float), + ("seed", c_uint32), + ("temp", c_float), ("codes", POINTER(c_int32)), ("n_codes", c_size_t), + ("feats", POINTER(c_float)), + ("n_feats", c_size_t), ("state_data", POINTER(c_char)), ("state_size", c_size_t), ] @@ -250,9 +267,12 @@ class mtmd_gen_inp(Structure): # # // for MTMD_GEN_PROCESS_TYPE_GEN_CODE # const int32_t * codes; -# size_t n_codes; +# size_t n_codes; +# const float * feats; // continuous counterpart of codes +# size_t n_feats; # const float * embd; // the generated hidden state, to be fed back to backbone # // it must have n_text_embd elements +# bool is_eos; // only set by pipelines having the EOS head inside mmproj # # // for MTMD_GEN_PROCESS_TYPE_GEN_WAV # const float * audio; @@ -264,7 +284,10 @@ class mtmd_gen_out(Structure): if TYPE_CHECKING: codes: Optional["_Pointer[c_int32]"] n_codes: int + feats: Optional["_Pointer[c_float]"] + n_feats: int embd: Optional["_Pointer[c_float]"] + is_eos: bool audio: Optional["_Pointer[c_float]"] n_samples: int state_data: Optional["_Pointer[c_char]"] @@ -273,7 +296,10 @@ class mtmd_gen_out(Structure): _fields_ = [ ("codes", POINTER(c_int32)), ("n_codes", c_size_t), + ("feats", POINTER(c_float)), + ("n_feats", c_size_t), ("embd", POINTER(c_float)), + ("is_eos", c_bool), ("audio", POINTER(c_float)), ("n_samples", c_size_t), ("state_data", POINTER(c_char)), @@ -350,8 +376,9 @@ class mtmd_helper_video_init_params(Structure): # mtmd_bitmap * speaker_ref; // optional, can be NULL # const char * lang; // optional, can be NULL # -# int32_t top_k; -# float top_p; +# int32_t top_k; +# float top_p; +# uint32_t seed; // UINT32_MAX for random (default: random) # # enum mtmd_helper_gen_audio_outtype out_type; # }; @@ -364,6 +391,7 @@ class mtmd_helper_gen_audio_inp(Structure): lang: Optional[bytes] top_k: int top_p: float + seed: int out_type: int _fields_ = [ @@ -374,6 +402,7 @@ class mtmd_helper_gen_audio_inp(Structure): ("lang", c_char_p), ("top_k", c_int32), ("top_p", c_float), + ("seed", c_uint32), ("out_type", c_int), ] @@ -891,6 +920,18 @@ def mtmd_gen_audio_get_info(ctx: mtmd_context_p, /) -> mtmd_gen_audio_info: ... +# // defaults tuned for the loaded pipeline, callers override only what they care about +# MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx); +@ctypes_function( + "mtmd_gen_inp_default", + [mtmd_context_p_ctypes], + mtmd_gen_inp, +) +def mtmd_gen_inp_default(ctx: mtmd_context_p, /) -> mtmd_gen_inp: + """Get default audio generation input parameters for an MTMD context.""" + ... + + # // note: this API is stateless, caller must handle state management and audio frame accumulation # MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, # const struct mtmd_gen_inp * inp, @@ -1303,12 +1344,15 @@ def mtmd_helper_gen_audio_step_prompt( # // generates one frame; must only be called after step_prompt() has returned 0 -# // h_state_out is valid until next step_gen() or reset() call +# // sampled can be LLAMA_TOKEN_NULL for pipelines with no discrete backbone token +# // out_stop (optional) is set on end-of-speech, the caller must then stop the loop +# // h_state_out is valid until next step_gen() or reset() call, None if no frame is generated # MTMD_API int32_t mtmd_helper_gen_audio_step_gen( # mtmd_helper_gen_audio * ctx, # llama_token sampled, # const float * h_state_in, -# const float ** h_state_out); +# const float ** h_state_out, +# bool * out_stop); @ctypes_function( "mtmd_helper_gen_audio_step_gen", [ @@ -1316,6 +1360,7 @@ def mtmd_helper_gen_audio_step_prompt( llama_cpp.llama_token, POINTER(c_float), POINTER(POINTER(c_float)), + POINTER(c_bool), ], c_int32, ) @@ -1324,6 +1369,7 @@ def mtmd_helper_gen_audio_step_gen( sampled: llama_cpp.llama_token, h_state_in: Optional["_Pointer[c_float]"], h_state_out: "_Pointer[_Pointer[c_float]]", + out_stop: Optional["_Pointer[c_bool]"], /, ) -> int: """Generate one audio frame.""" diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 936918514..adb55e514 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 936918514ce522b553c0fd80b169a6440e6096c6 +Subproject commit adb55e5148dc93bcdca7212a2d1df3ccc422959a From 987e3df20a8f0f62b8b595191065fd60c3f484d1 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 15 Aug 2026 23:07:23 -0700 Subject: [PATCH 7/7] fix(example): retain recurrent state for MTP rollback (#2358) --- CHANGELOG.md | 1 + examples/server/server.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e4adffb3..6f7dcb668 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix(example): retain recurrent state for server MTP rollback - feat: update llama.cpp to ggml-org/llama.cpp@adb55e514 ## [0.3.34] diff --git a/examples/server/server.py b/examples/server/server.py index 452654ed1..2eb31aed3 100644 --- a/examples/server/server.py +++ b/examples/server/server.py @@ -11362,6 +11362,11 @@ def __init__( "speculative decoding is only supported for attention models" ) n_ctx_train = int(llama_cpp.llama_model_n_ctx_train(llama_model)) + target_n_rs_seq = ( + max(1, draft_model_num_pred_tokens) + if normalized_draft_model == "draft-mtp" + else None + ) context_params = self.build_context_params( n_ctx=n_ctx if n_ctx is not None else n_ctx_train, @@ -11389,7 +11394,7 @@ def __init__( type_k=type_k, type_v=type_v, kv_unified=kv_unified, - n_rs_seq=None, + n_rs_seq=target_n_rs_seq, ctx_type=None, ) ctx = llama_cpp.llama_init_from_model(llama_model, context_params) @@ -11419,6 +11424,15 @@ def __init__( "MTP requires runtime n_batch to fit the pending token plus draft tokens " f"(required {required_mtp_batch}, got {self.n_batch})" ) + if ( + target_n_rs_seq is not None + and self.exact_checkpoints_only + and self.n_rs_seq < target_n_rs_seq + ): + raise RuntimeError( + "MTP requires retained recurrent-state slots for rollback " + f"(required {target_n_rs_seq}, got {self.n_rs_seq})" + ) self.n_ctx_train = n_ctx_train self.n_vocab = int(llama_cpp.llama_vocab_n_tokens(self.vocab)) self.n_embd = int(llama_cpp.llama_model_n_embd(self.llama_model))