From 652da2fa9ae74b9dd9da1f458d2bbee021be9e21 Mon Sep 17 00:00:00 2001 From: Loubna Ben Allal <44069155+loubnabnl@users.noreply.github.com> Date: Thu, 25 May 2023 18:50:20 +0200 Subject: [PATCH 1/7] Add hardware requirements section --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf1e22a..2deaf80 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ huggingface-cli login - [Datasets](#datasets) - [Stack Exchange](#stack-exchange-se) - [Merging PEFT adapter layers](#merging-peft-adapter-layers) +3. [Evaluation](#evaluation) +4. [Inference hardware requirements](#inference-hardware-requirements) # Quickstart StarCoder was trained on GitHub code, thus it can be used to perform code generation. More precisely, the model can complete the implementation of a function or infer the following characters in a line of code. This can be done with the help of the 🤗's [transformers](https://github.com/huggingface/transformers) library. @@ -63,6 +65,7 @@ tokenizer = AutoTokenizer.from_pretrained(checkpoint) pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, device=0) print( pipe("def hello():") ) ``` +For hardware requirements, check the secyoon [Inference hardware requirements](#inference-hardware-requirements). ## Text-generation-inference @@ -189,5 +192,21 @@ For example python finetune/merge_peft_adapters.py --model_name_or_path bigcode/starcoder --peft_model_path checkpoints/checkpoint-1000 --push_to_hub ``` -## Evaluation +# Evaluation To evaluate StarCoder and its derivatives, you can use the [BigCode-Evaluation-Harness](https://github.com/bigcode-project/bigcode-evaluation-harness) for evaluating Code LLMs. + +# Inference hardware requirements +In FP32 the model requires more than 60GB of RAM, you can load it in FP16 or BF16 in ~30GB, or in 8bit under 20GB of RAM with +```python +# make sure you have accelerate and bitsandbytes installed +from transformers import AutoModelForCausalLM, AutoTokenizer + +tokenizer = AutoTokenizer.from_pretrained("bigcode/starcoder") +# for fp16 replace with `load_in_8bit=True` with `torch_dtype=torch.float16` +model = AutoModelForCausalLM.from_pretrained("bigcode/starcoder", device_map="auto", load_in_8bit=True) +print(f"Memory footprint: {model.get_memory_footprint() / 1e6:.2f} MB") +```` +``` +Memory footprint: 15939.61 MB +``` +You can also try [starcoder.cpp](https://github.com/bigcode-project/starcoder.cpp), a C++ implementation with [ggml](https://github.com/ggerganov/ggml) library. From 576502e8f8fb53207cea1cbc3f4fd157c81e04a7 Mon Sep 17 00:00:00 2001 From: Loubna Ben Allal <44069155+loubnabnl@users.noreply.github.com> Date: Thu, 25 May 2023 20:50:14 +0200 Subject: [PATCH 2/7] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2deaf80..47355f8 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ tokenizer = AutoTokenizer.from_pretrained(checkpoint) pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, device=0) print( pipe("def hello():") ) ``` -For hardware requirements, check the secyoon [Inference hardware requirements](#inference-hardware-requirements). +For hardware requirements, check the section [Inference hardware requirements](#inference-hardware-requirements). ## Text-generation-inference From e25ab3ac73cef7c63c7fce6d7e54d4a705adda34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eryk=20Mazu=C5=9B?= Date: Wed, 31 May 2023 12:58:41 +0200 Subject: [PATCH 3/7] Update README.md (#48) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 47355f8..5e53236 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ checkpoint = "bigcode/starcoder" device = "cuda" # for GPU usage or "cpu" for CPU usage tokenizer = AutoTokenizer.from_pretrained(checkpoint) -# to save memory consider using fp16 or bf16 by specifying torch.dtype=torch.float16 for example +# to save memory consider using fp16 or bf16 by specifying torch_dtype=torch.float16 for example model = AutoModelForCausalLM.from_pretrained(checkpoint).to(device) inputs = tokenizer.encode("def print_hello_world():", return_tensors="pt").to(device) From 80a39c2e0a0c064636df2d1e387b38086cd9c425 Mon Sep 17 00:00:00 2001 From: Arjun Guha Date: Wed, 31 May 2023 13:20:13 -0400 Subject: [PATCH 4/7] Removed unused line --- finetune/merge_peft_adapters.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/finetune/merge_peft_adapters.py b/finetune/merge_peft_adapters.py index 790b6fd..218728f 100644 --- a/finetune/merge_peft_adapters.py +++ b/finetune/merge_peft_adapters.py @@ -25,7 +25,6 @@ def main(): model = PeftModel.from_pretrained(base_model, args.peft_model_path) model = model.merge_and_unload() - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") tokenizer = AutoTokenizer.from_pretrained(args.base_model_name_or_path) if args.push_to_hub: @@ -38,4 +37,4 @@ def main(): print(f"Model saved to {args.base_model_name_or_path}-merged") if __name__ == "__main__" : - main() \ No newline at end of file + main() From 1f8b704981f046e240709db5256d5e5a268f3610 Mon Sep 17 00:00:00 2001 From: Daniel Fried Date: Wed, 7 Jun 2023 12:20:04 -0400 Subject: [PATCH 5/7] Update README.md Add `clean_up_tokenization_spaces=False` argument for `tokenizer.decode` to README. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e53236..b800f42 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ model = AutoModelForCausalLM.from_pretrained(checkpoint).to(device) inputs = tokenizer.encode("def print_hello_world():", return_tensors="pt").to(device) outputs = model.generate(inputs) -print(tokenizer.decode(outputs[0])) +# clean_up_tokenization_spaces=False prevents a tokenizer edge case which can result in spaces being removed around punctuation +print(tokenizer.decode(outputs[0], clean_up_tokenization_spaces=False)) ``` or ```python From 0d275cddf7a63fc7c32851af5e1ebdb918bb61c0 Mon Sep 17 00:00:00 2001 From: ArmelRandy <76953833+ArmelRandy@users.noreply.github.com> Date: Wed, 28 Jun 2023 16:55:59 +0200 Subject: [PATCH 6/7] Removing argument prompt_type, which is not supported by the parser. --- chat/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chat/generate.py b/chat/generate.py index 64a3905..c67d7d8 100644 --- a/chat/generate.py +++ b/chat/generate.py @@ -127,7 +127,7 @@ def main(): print() raw_model_name = args.model_id.split("/")[-1] - model_name = f"{raw_model_name}-{args.prompt_type}" + model_name = f"{raw_model_name}" if args.revision is not None: model_name += f"-{args.revision}" From d72c7fe3dda81d47ad9b851f9567393fb6b551b9 Mon Sep 17 00:00:00 2001 From: ArmelRandy <76953833+ArmelRandy@users.noreply.github.com> Date: Thu, 29 Jun 2023 10:07:07 +0200 Subject: [PATCH 7/7] Add load_best_model_at_end=True --- finetune/finetune.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/finetune/finetune.py b/finetune/finetune.py index 96ab961..525b37f 100644 --- a/finetune/finetune.py +++ b/finetune/finetune.py @@ -267,6 +267,8 @@ def run_training(args, train_data, val_data): output_dir=args.output_dir, dataloader_drop_last=True, evaluation_strategy="steps", + save_strategy="steps", + load_best_model_at_end=True, max_steps=args.max_steps, eval_steps=args.eval_freq, save_steps=args.save_freq, @@ -309,4 +311,4 @@ def main(args): logging.set_verbosity_error() - main(args) \ No newline at end of file + main(args)