From 5acaa80e66d5083a6114d4d72551c1c4b153789a Mon Sep 17 00:00:00 2001 From: czxdev <36617515+czxdev@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:36:00 +0800 Subject: [PATCH 01/10] Refine CUDA versions and CMake arguments in workflow Updated the CUDA versions and modified CMake arguments for building wheels. Disable AVX512 --- .github/workflows/build-wheels-cuda.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 59dc3558b..88eb0b148 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -26,11 +26,11 @@ jobs: id: set-matrix run: | $matrix = @{ - 'os' = @('ubuntu-22.04', 'windows-2022') + 'os' = @('ubuntu-22.04') # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic, # so one builder per toolkit version is sufficient. 'pyver' = @("3.9") - 'cuda' = @("11.8.0", "12.1.1", "12.2.2", "12.3.2", "12.4.1", "12.5.1", "13.0.2", "13.2.1") + 'cuda' = @("12.4.1") 'releasetag' = @("basic") 'exclude' = @( @{ 'os' = 'windows-2022'; 'cuda' = '12.1.1' }, @@ -231,7 +231,9 @@ jobs: # one forward-compatible PTX target instead of embedding PTX for every # SM. This keeps the wheel under GitHub's 2 GiB release-asset limit. $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON -DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=$cudaArchs -DCMAKE_CUDA_FLAGS=-allow-unsupported-compiler -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler $env:CMAKE_ARGS" - $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off' + $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_NATIVE=OFF -DGGML_AVX512=OFF' + $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=ON -DGGML_FMA=ON -DGGML_F16C=ON' + if ($IsLinux) { $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_OPENMP=OFF' } From 70b2189c8aae30adb78134dce3a55b3cc572379d Mon Sep 17 00:00:00 2001 From: czxdev <36617515+czxdev@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:09:00 +0800 Subject: [PATCH 02/10] Update build-wheels-cuda.yaml --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 88eb0b148..08feffe62 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -30,7 +30,7 @@ jobs: # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic, # so one builder per toolkit version is sufficient. 'pyver' = @("3.9") - 'cuda' = @("12.4.1") + 'cuda' = @("13.2.1") 'releasetag' = @("basic") 'exclude' = @( @{ 'os' = 'windows-2022'; 'cuda' = '12.1.1' }, From 90b1cd9578b768e9447ac220cbb41a738e0828f2 Mon Sep 17 00:00:00 2001 From: czxdev <36617515+czxdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 13:46:35 +0800 Subject: [PATCH 03/10] Update CUDA versions and OS in workflow configuration --- .github/workflows/build-wheels-cuda.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 08feffe62..11bc83df0 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -26,11 +26,11 @@ jobs: id: set-matrix run: | $matrix = @{ - 'os' = @('ubuntu-22.04') + 'os' = @('windows-2022') # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic, # so one builder per toolkit version is sufficient. 'pyver' = @("3.9") - 'cuda' = @("13.2.1") + 'cuda' = @("13.2.1", "12.4.1") 'releasetag' = @("basic") 'exclude' = @( @{ 'os' = 'windows-2022'; 'cuda' = '12.1.1' }, From ea7d2ba326d8bec5a8c896f63cbc283ea9973977 Mon Sep 17 00:00:00 2001 From: czxdev <36617515+czxdev@users.noreply.github.com> Date: Wed, 8 Jul 2026 19:35:11 +0800 Subject: [PATCH 04/10] Update workflow to remove release triggers Removed workflow_run trigger for new releases. --- .github/workflows/generate-index-from-release.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml index edf292387..4a4ef566c 100644 --- a/.github/workflows/generate-index-from-release.yaml +++ b/.github/workflows/generate-index-from-release.yaml @@ -1,12 +1,6 @@ name: Wheels Index on: - # Trigger on new release - workflow_run: - workflows: ["Release", "Build Wheels (CUDA)", "Build Wheels (Metal)", "Build Wheels (Vulkan)", "Build Wheels (ROCm)"] - types: - - completed - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From ab765986657409eb6afc19eac2c26fb144d04165 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 5 Jul 2026 12:23:45 -0700 Subject: [PATCH 05/10] feat: update llama.cpp to 78d2f5246 (#2327) * feat: update llama.cpp to 78d2f5246 * fix: align llama.cpp binding comments * fix: align llama.cpp binding docstrings (#2328) * fix: align llama.cpp binding docstrings * fix: use None in Python binding docstrings * fix: format binding docstring Co-authored-by: czxdev <36617515+czxdev@users.noreply.github.com> --- CHANGELOG.md | 2 ++ llama_cpp/llama_cpp.py | 47 ++++++++++++++++++++++++++++++++++-------- vendor/llama.cpp | 2 +- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b5e88578..5aa71dcdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@78d2f5246 + ## [0.3.32] - feat(example): support chained NextN heads for server MTP drafting diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index 176709d96..d703a2fcd 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -1272,6 +1272,14 @@ def llama_flash_attn_type_name(flash_attn_type: int, /) -> Optional[bytes]: ... +# // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium" +# LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype); +@ctypes_function("llama_ftype_name", [ctypes.c_int], ctypes.c_char_p) +def llama_ftype_name(ftype: int, /) -> Optional[bytes]: + '''Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"''' + ... + + # // Initialize the llama + ggml backend # // If numa is true, use NUMA optimizations # // Call once at the start of the program @@ -1777,7 +1785,8 @@ def llama_model_rope_freq_scale_train(model: llama_model_p, /) -> float: ... # LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); @ctypes_function("llama_model_n_cls_out", [llama_model_p_ctypes], ctypes.c_uint32) def llama_model_n_cls_out(model: llama_model_p, /) -> int: - """Returns the number of classifier outputs (only valid for classifier models)""" + """Returns the number of classifier outputs (only valid for classifier models) + Undefined behavior for non-classifier models""" ... @@ -1843,7 +1852,7 @@ def llama_model_meta_count(model: llama_model_p, /) -> int: # LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key); @ctypes_function("llama_model_meta_key_str", [ctypes.c_int], ctypes.c_char_p) def llama_model_meta_key_str(key: int, /) -> Optional[bytes]: - """Get sampling metadata key name. Returns None if the key is invalid.""" + """Get sampling metadata key name. Returns None if the key is invalid""" ... @@ -1910,6 +1919,14 @@ def llama_model_desc( ... +# // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0 +# LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model); +@ctypes_function("llama_model_ftype", [llama_model_p_ctypes], ctypes.c_int) +def llama_model_ftype(model: llama_model_p, /) -> int: + """Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0""" + ... + + # // Returns the total size of all the tensors in the model in bytes # LLAMA_API uint64_t llama_model_size(const struct llama_model * model); @ctypes_function("llama_model_size", [llama_model_p_ctypes], ctypes.c_uint64) @@ -2485,7 +2502,9 @@ def llama_memory_can_shift(mem: llama_memory_t, /) -> bool: # LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); @ctypes_function("llama_state_get_size", [llama_context_p_ctypes], ctypes.c_size_t) def llama_state_get_size(ctx: llama_context_p, /) -> int: - """Returns the *actual* size in bytes of the state (logits, embedding and memory)""" + """Returns the *actual* size in bytes of the state + (logits, embedding and memory) + Only use when saving the state, not when restoring it, otherwise the size may be too small.""" ... @@ -3046,9 +3065,12 @@ def llama_batch_free(batch: llama_batch, /): # struct llama_batch batch); @ctypes_function("llama_encode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: - """Process a batch of tokens using the encoder. + """Process a batch of tokens. + In contrast to llama_decode() - this call does not use KV cache. + For encode-decoder contexts, processes the batch using the encoder. + Can store the encoder output internally for later use by the decoder's cross-attention layers. 0 - success - < 0 - error""" + < 0 - error. the memory state is restored to the state before this call""" ... @@ -3070,9 +3092,15 @@ def llama_encode(ctx: llama_context_p, batch: llama_batch, /) -> int: @ctypes_function("llama_decode", [llama_context_p_ctypes, llama_batch], ctypes.c_int32) def llama_decode(ctx: llama_context_p, batch: llama_batch, /) -> int: """Process a batch of tokens. + Requires the context to have a memory. + For encode-decoder contexts, processes the batch using the decoder. + Positive return values does not mean a fatal error, but rather a warning. + Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context + To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + Upon other return values, the memory state is restored to the state before this call 0 - success 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) - 2 - aborted (processed ubatches will remain in the context's memory) + 2 - aborted (processed ubatches will remain in the context's memory) -1 - invalid input batch < -1 - fatal error (processed ubatches will remain in the context's memory)""" ... @@ -3108,7 +3136,7 @@ def llama_set_n_threads( # LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); @ctypes_function("llama_n_threads", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads(ctx: llama_context_p, /) -> int: - """Get the number of threads used for generation of a single token""" + """Get the number of threads used for generation of a single token.""" ... @@ -3116,7 +3144,7 @@ def llama_n_threads(ctx: llama_context_p, /) -> int: # LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); @ctypes_function("llama_n_threads_batch", [llama_context_p_ctypes], ctypes.c_int32) def llama_n_threads_batch(ctx: llama_context_p, /) -> int: - """Get the number of threads used for prompt and batch processing (multiple token)""" + """Get the number of threads used for prompt and batch processing (multiple token).""" ... @@ -3125,7 +3153,8 @@ def llama_n_threads_batch(ctx: llama_context_p, /) -> int: # LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); @ctypes_function("llama_set_embeddings", [llama_context_p_ctypes, ctypes.c_bool], None) def llama_set_embeddings(ctx: llama_context_p, embeddings: bool, /): - """Set whether the context outputs embeddings or not""" + """Set whether the context outputs embeddings or not + TODO: rename to avoid confusion with llama_get_embeddings()""" ... diff --git a/vendor/llama.cpp b/vendor/llama.cpp index b3fed31b9..78d2f5246 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit b3fed31b99f9bd37725833674252bccb429bb183 +Subproject commit 78d2f524682d9fee790a6460c93d018dafeb5229 From 1a57b73495c0023a2fe9d90112b3abcb31e6a0af Mon Sep 17 00:00:00 2001 From: Andrei Date: Sun, 5 Jul 2026 15:43:11 -0700 Subject: [PATCH 06/10] chore: bump version to 0.3.33 (#2329) Co-authored-by: czxdev <36617515+czxdev@users.noreply.github.com> --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aa71dcdb..381e953bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.33] + - feat: update llama.cpp to ggml-org/llama.cpp@78d2f5246 ## [0.3.32] diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index 45f9c8f27..ff142e6bf 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.32" +__version__ = "0.3.33" From 79a106348358a5cd7a1d5ea0ffe9aa30efc55be8 Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 11 Jul 2026 21:05:39 -0700 Subject: [PATCH 07/10] feat: update llama.cpp to e3546c794 (#2338) * feat: update llama.cpp to 13f2b28b0 * feat: update llama.cpp to e3546c794 Co-authored-by: czxdev <36617515+czxdev@users.noreply.github.com> --- CHANGELOG.md | 2 ++ llama_cpp/llama_cpp.py | 8 ++++++-- vendor/llama.cpp | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 381e953bc..4dd18d2f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 + ## [0.3.33] - feat: update llama.cpp to ggml-org/llama.cpp@78d2f5246 diff --git a/llama_cpp/llama_cpp.py b/llama_cpp/llama_cpp.py index d703a2fcd..64399fe31 100644 --- a/llama_cpp/llama_cpp.py +++ b/llama_cpp/llama_cpp.py @@ -89,7 +89,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # GGML_TYPE_MXFP4 = 39, # GGML_TYPE_NVFP4 = 40, # GGML_TYPE_Q1_0 = 41, -# GGML_TYPE_COUNT = 42, +# GGML_TYPE_Q2_0 = 42, +# GGML_TYPE_COUNT = 43, # }; GGML_TYPE_F32 = 0 GGML_TYPE_F16 = 1 @@ -122,7 +123,8 @@ def _warn_deprecated(symbol: str, hint: str) -> None: GGML_TYPE_MXFP4 = 39 GGML_TYPE_NVFP4 = 40 GGML_TYPE_Q1_0 = 41 -GGML_TYPE_COUNT = 42 +GGML_TYPE_Q2_0 = 42 +GGML_TYPE_COUNT = 43 # from ggml-backend.h # typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); @@ -411,6 +413,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: # LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors # LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors # LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors +# LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors # # LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file # }; @@ -452,6 +455,7 @@ def _warn_deprecated(symbol: str, hint: str) -> None: LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38 LLAMA_FTYPE_MOSTLY_NVFP4 = 39 LLAMA_FTYPE_MOSTLY_Q1_0 = 40 +LLAMA_FTYPE_MOSTLY_Q2_0 = 41 LLAMA_FTYPE_GUESSED = 1024 # enum llama_rope_scaling_type { diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 78d2f5246..e3546c794 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 78d2f524682d9fee790a6460c93d018dafeb5229 +Subproject commit e3546c7948e3af463d0b401e6421d5a4c2faf565 From aad9763fcc27fdf0f5e20bc593dc58a2b61ed03c Mon Sep 17 00:00:00 2001 From: Andrei Date: Sat, 11 Jul 2026 21:27:50 -0700 Subject: [PATCH 08/10] chore: bump version to 0.3.34 (#2337) Co-authored-by: czxdev <36617515+czxdev@users.noreply.github.com> --- CHANGELOG.md | 2 ++ llama_cpp/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dd18d2f4..629b645a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.34] + - feat: update llama.cpp to ggml-org/llama.cpp@e3546c794 ## [0.3.33] diff --git a/llama_cpp/__init__.py b/llama_cpp/__init__.py index ff142e6bf..5a0a40d10 100644 --- a/llama_cpp/__init__.py +++ b/llama_cpp/__init__.py @@ -1,4 +1,4 @@ from .llama_cpp import * from .llama import * -__version__ = "0.3.33" +__version__ = "0.3.34" From 498a208a1db44e48b1841d37fa7ee10a4f7ba43b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 9 Aug 2026 04:09:17 +0000 Subject: [PATCH 09/10] Include Ubuntu and Windows in CUDA build matrix Co-authored-by: czxdev <36617515+czxdev@users.noreply.github.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 11bc83df0..583407882 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -26,7 +26,7 @@ jobs: id: set-matrix run: | $matrix = @{ - 'os' = @('windows-2022') + 'os' = @('ubuntu-22.04', 'windows-2022') # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic, # so one builder per toolkit version is sufficient. 'pyver' = @("3.9") From 39c9bcfe9ea396d9a44219ae042606c6a3b217ff Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:26:44 +0000 Subject: [PATCH 10/10] Build CUDA wheels for 13.2 only Co-authored-by: czxdev <36617515+czxdev@users.noreply.github.com> --- .github/workflows/build-wheels-cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml index 583407882..02670b475 100644 --- a/.github/workflows/build-wheels-cuda.yaml +++ b/.github/workflows/build-wheels-cuda.yaml @@ -30,7 +30,7 @@ jobs: # wheel.py-api = "py3" makes the CUDA wheel interpreter-agnostic, # so one builder per toolkit version is sufficient. 'pyver' = @("3.9") - 'cuda' = @("13.2.1", "12.4.1") + 'cuda' = @("13.2.1") 'releasetag' = @("basic") 'exclude' = @( @{ 'os' = 'windows-2022'; 'cuda' = '12.1.1' },