From 458efdc1b58fda3c37479c51a3e5908ab34877eb Mon Sep 17 00:00:00 2001 From: Mk Bashar Date: Thu, 14 Mar 2024 22:54:19 -0400 Subject: [PATCH] finetune updated to load l2s data --- .gitignore | 5 +- finetune/finetune.py | 42 ++++++---- playground.ipynb | 182 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 15 deletions(-) create mode 100644 playground.ipynb diff --git a/.gitignore b/.gitignore index 497d87e..62c01f1 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,7 @@ cython_debug/ #.idea/ data/ -wandb/ \ No newline at end of file +wandb/ +dataset/ +checkpoints/ +checkpoints2/ \ No newline at end of file diff --git a/finetune/finetune.py b/finetune/finetune.py index 525b37f..95179a6 100644 --- a/finetune/finetune.py +++ b/finetune/finetune.py @@ -1,11 +1,13 @@ import argparse import os - +import sys +import glob +from datasets import IterableDataset import torch from accelerate import Accelerator from datasets import load_dataset from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training, set_peft_model_state_dict -from torch.utils.data import IterableDataset +# from torch.utils.data import IterableDataset from tqdm import tqdm from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments, logging, set_seed from transformers import TrainerCallback, TrainingArguments, TrainerState, TrainerControl @@ -76,16 +78,15 @@ def get_args(): parser.add_argument("--num_warmup_steps", type=int, default=100) parser.add_argument("--weight_decay", type=float, default=0.05) - parser.add_argument("--local_rank", type=int, default=0) parser.add_argument("--no_fp16", action="store_false") parser.add_argument("--bf16", action="store_true", default=True) parser.add_argument("--no_gradient_checkpointing", action="store_false", default=False) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--num_workers", type=int, default=None) parser.add_argument("--output_dir", type=str, default="./checkpoints") - parser.add_argument("--log_freq", default=100, type=int) - parser.add_argument("--eval_freq", default=100, type=int) - parser.add_argument("--save_freq", default=1000, type=int) + parser.add_argument("--log_freq", default=1, type=int) + parser.add_argument("--eval_freq", default=10, type=int) + parser.add_argument("--save_freq", default=10, type=int) return parser.parse_args() @@ -190,16 +191,29 @@ def __iter__(self): "labels": torch.LongTensor(input_ids), } +def gen(): + prompts_list = glob.glob('dataset/train/prompts/*.txt') + prompts_list.sort() + ref_list = glob.glob('dataset/train/ref_scenarios/*.xosc') + ref_list.sort() + target_list = glob.glob('dataset/train/target_scenarios/*.xosc') + target_list.sort() + + for i in range(len(ref_list)): + with open(prompts_list[i], 'r') as prompt: + with open(ref_list[i], 'r') as ref: + with open(target_list[i], 'r') as target: + prompt = prompt.read() + ref = ref.read() + add = "\n" + ref = add + ref + target = target.read() + yield {'prompt': ref, 'target': target} def create_datasets(tokenizer, args): - dataset = load_dataset( - args.dataset_name, - data_dir=args.subset, - split=args.split, - use_auth_token=True, - num_proc=args.num_workers if not args.streaming else None, - streaming=args.streaming, - ) + + dataset = IterableDataset.from_generator(gen) + if args.streaming: print("Loading the dataset in streaming mode") valid_data = dataset.take(args.size_valid_set) diff --git a/playground.ipynb b/playground.ipynb new file mode 100644 index 0000000..98430b6 --- /dev/null +++ b/playground.ipynb @@ -0,0 +1,182 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-03-14 00:15:51.417834: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-03-14 00:15:51.417871: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-03-14 00:15:51.418843: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2024-03-14 00:15:51.423695: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-03-14 00:15:51.969823: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "Overriding torch_dtype=None with `torch_dtype=torch.float16` due to requirements of `bitsandbytes` to enable model loading in mixed int8. Either pass torch_dtype=torch.float16 or don't pass this argument at all to remove this warning.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5c134c3dcf014aba814aa57b8ce8a921", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/7 [00:00\n", + " main(args)\n", + " File \"/home/mkbashar/Downloads/nlp/project/starcoder/finetune/finetune.py\", line 325, in main\n", + " train_dataset, eval_dataset = create_datasets(tokenizer, args)\n", + " File \"/home/mkbashar/Downloads/nlp/project/starcoder/finetune/finetune.py\", line 223, in create_datasets\n", + " dataset = IterableDataset.from_generator(gen)\n", + "AttributeError: type object 'IterableDataset' has no attribute 'from_generator'\n" + ] + } + ], + "source": [ + "!python -m torch.distributed.run --nproc_per_node 2 finetune/finetune.py --model_path=\"bigcode/starcoderbase-1b\" --dataset_name=\"ArmelR/stack-exchange-instruction\" --subset=\"data/finetune\" --split=\"train\" --size_valid_set 2000 --streaming --seq_length 4096 --max_steps 100 --batch_size 1 --input_column_name=\"prompt\" --output_column_name=\"target\" --gradient_accumulation_steps 16 --learning_rate 1e-4 --lr_scheduler_type=\"cosine\" --num_warmup_steps 10 --weight_decay 0.05 --output_dir=\"./checkpoints\" --bf16" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'prompt': '\\n\\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n', 'target': '\\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n
\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n'}\n" + ] + } + ], + "source": [ + "import glob\n", + "from datasets import IterableDataset\n", + "\n", + "def gen():\n", + " prompts_list = glob.glob('dataset/train/prompts/*.txt')\n", + " prompts_list.sort()\n", + " ref_list = glob.glob('dataset/train/ref_scenarios/*.xosc')\n", + " ref_list.sort()\n", + " target_list = glob.glob('dataset/train/target_scenarios/*.xosc')\n", + " target_list.sort()\n", + "\n", + " for i in range(len(ref_list)):\n", + " with open(prompts_list[i], 'r') as prompt:\n", + " with open(ref_list[i], 'r') as ref:\n", + " with open(target_list[i], 'r') as target:\n", + " prompt = prompt.read()\n", + " ref = ref.read()\n", + " add = \"\\n\"\n", + " ref = add + ref\n", + " target = target.read()\n", + " yield {'prompt': ref, 'target': target}\n", + "\n", + "ds = IterableDataset.from_generator(gen)\n", + "\n", + "for example in ds:\n", + " print(example)\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "orbbec", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}