From 782b760e9153fac556bdb58800e69f1393b7832e Mon Sep 17 00:00:00 2001 From: Nouamane Tazi Date: Sat, 20 May 2023 18:05:52 +0200 Subject: [PATCH 1/4] try fix eval ctx --- convert-hf-to-ggml.py | 2 +- main.cpp | 27 ++++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/convert-hf-to-ggml.py b/convert-hf-to-ggml.py index c3ddfe7..20b9b4c 100644 --- a/convert-hf-to-ggml.py +++ b/convert-hf-to-ggml.py @@ -75,7 +75,7 @@ def bytes_to_unicode(): vocab_size = hparams["vocab_size"] fout.write(struct.pack("i", vocab_size)) # fout.write(struct.pack("i", len(encoder))) -fout.write(struct.pack("i", hparams["n_positions"])) +fout.write(struct.pack("i", hparams["n_positions"])) # n_ctx fout.write(struct.pack("i", hparams["n_embd"])) fout.write(struct.pack("i", hparams["n_head"])) fout.write(struct.pack("i", hparams["n_layer"])) diff --git a/main.cpp b/main.cpp index 5cdfa74..4427903 100644 --- a/main.cpp +++ b/main.cpp @@ -126,12 +126,15 @@ bool starcoder_model_load(const std::string & fname, starcoder_model & model, gp } std::string word; + std::vector buf(128); + for (int i = 0; i < n_vocab; i++) { uint32_t len; fin.read((char *) &len, sizeof(len)); - word.resize(len); - fin.read((char *) word.data(), len); + buf.resize(len); + fin.read((char *) buf.data(), len); + word.assign(buf.data(), len); vocab.token_to_id[word] = i; vocab.id_to_token[i] = word; @@ -413,6 +416,14 @@ bool starcoder_eval( static size_t buf_size = 256u*1024*1024; static void * buf = malloc(buf_size); + // use 2 scratch buffers + // TODO: very hacky solution - reimplement in a more elegant way + static size_t scr0_size = 128u*1024*1024; + static void * scr0 = malloc(scr0_size); + + static size_t scr1_size = 128u*1024*1024; + static void * scr1 = malloc(scr1_size); + if (mem_per_token > 0 && mem_per_token*N > buf_size) { const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead //printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new); @@ -453,6 +464,8 @@ bool starcoder_eval( for (int il = 0; il < n_layer; ++il) { struct ggml_tensor * cur; + ggml_set_scratch(ctx0, { 0, scr0_size, scr0, }); + // norm { // [ 768, N] @@ -599,6 +612,8 @@ bool starcoder_eval( struct ggml_tensor * inpFF = cur; + ggml_set_scratch(ctx0, { 0, scr1_size, scr1, }); + // feed-forward network { // norm @@ -655,6 +670,8 @@ bool starcoder_eval( inpL = ggml_add(ctx0, cur, inpFF); } + ggml_set_scratch(ctx0, { 0, scr0_size, scr0, }); + // norm { // [ 768, N] @@ -669,6 +686,8 @@ bool starcoder_eval( ggml_repeat(ctx0, model.ln_f_b, inpL)); } + ggml_set_scratch(ctx0, { 0, 0, nullptr, }); + // inpL = WTE * inpL // [ 768, 50257] - model.lm_head // [ 768, N] - inpL @@ -696,7 +715,7 @@ bool starcoder_eval( if (mem_per_token == 0) { mem_per_token = ggml_used_mem(ctx0)/N; } - //printf("used_mem = %zu\n", ggml_used_mem(ctx0)); + //printf("used_mem = %zu MB\n", ggml_used_mem(ctx0)/(1024*1024)); ggml_free(ctx0); @@ -704,6 +723,8 @@ bool starcoder_eval( } int main(int argc, char ** argv) { + ggml_time_init(); + const int64_t t_main_start_us = ggml_time_us(); gpt_params params; From 152b6943ebf1c077ff017601ccf18adb7277a979 Mon Sep 17 00:00:00 2001 From: Nouamane Tazi Date: Sat, 20 May 2023 22:21:35 +0200 Subject: [PATCH 2/4] bump scratch buffers to 256 MB --- ggml.c | 3 ++- main.cpp | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ggml.c b/ggml.c index da3d914..782b9cf 100644 --- a/ggml.c +++ b/ggml.c @@ -3983,7 +3983,8 @@ struct ggml_tensor * ggml_new_tensor_impl( }; } else { if (ctx->scratch.offs + size_needed > ctx->scratch.size) { - GGML_PRINT("%s: not enough space in the scratch memory\n", __func__); + GGML_PRINT("%s: not enough space in the scratch memory pool (needed %zu, available %zu)\n", + __func__, ctx->scratch.offs + size_needed, ctx->scratch.size); assert(false); return NULL; } diff --git a/main.cpp b/main.cpp index 4427903..388bd29 100644 --- a/main.cpp +++ b/main.cpp @@ -418,10 +418,10 @@ bool starcoder_eval( // use 2 scratch buffers // TODO: very hacky solution - reimplement in a more elegant way - static size_t scr0_size = 128u*1024*1024; + static size_t scr0_size = 256u*1024*1024; static void * scr0 = malloc(scr0_size); - static size_t scr1_size = 128u*1024*1024; + static size_t scr1_size = 256u*1024*1024; static void * scr1 = malloc(scr1_size); if (mem_per_token > 0 && mem_per_token*N > buf_size) { From 1deec561d1f6458d7dd9d31cc2b6004fc389a1ea Mon Sep 17 00:00:00 2001 From: Noob ML Dude Date: Fri, 23 Jun 2023 08:27:25 +0200 Subject: [PATCH 3/4] Mention starchat-beta support on README Verified starchat-beta support using the same scripts for quantization and generation --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2bab489..0c38a22 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ The example supports the following 💫 StarCoder models: - `bigcode/starcoder` - `bigcode/gpt_bigcode-santacoder` aka the smol StarCoder +- `HuggingFaceH4/starchat-beta` - the coding assistants based on StarCoderPlus Sample performance on MacBook M1 Pro: From 39204517b087dc53821bf9bb1384edcdd0306f98 Mon Sep 17 00:00:00 2001 From: Nouamane Tazi Date: Wed, 26 Jul 2023 14:19:21 +0200 Subject: [PATCH 4/4] add reqs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8651995 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +transformers[torch]==4.31.0 \ No newline at end of file