diff --git a/.gitignore b/.gitignore index 497d87e..2152ca5 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,5 @@ cython_debug/ #.idea/ data/ -wandb/ \ No newline at end of file +wandb/ +checkpoints/ \ No newline at end of file diff --git a/finetune/finetune.py b/finetune/finetune.py index 525b37f..806bd62 100644 --- a/finetune/finetune.py +++ b/finetune/finetune.py @@ -2,6 +2,7 @@ import os import torch +import safetensors.torch as sttorch from accelerate import Accelerator from datasets import load_dataset from peft import LoraConfig, get_peft_model, prepare_model_for_int8_training, set_peft_model_state_dict @@ -41,8 +42,8 @@ def on_train_end( **kwargs, ): print(f"Loading best peft model from {state.best_model_checkpoint} (score: {state.best_metric}).") - best_model_path = os.path.join(state.best_model_checkpoint, "adapter_model.bin") - adapters_weights = torch.load(best_model_path) + best_model_path = os.path.join(state.best_model_checkpoint, "adapter_model.safetensors") + adapters_weights = sttorch.load_file(best_model_path) model = kwargs["model"] set_peft_model_state_dict(model, adapters_weights) return control @@ -247,9 +248,9 @@ def run_training(args, train_data, val_data): model = prepare_model_for_int8_training(model) lora_config = LoraConfig( - r=args.lora_r, - lora_alpha=args.lora_alpha, - lora_dropout=args.lora_dropout, + r=args.lora_r, # !!!!!!!!! + lora_alpha=args.lora_alpha, # !!!!!!!!! + lora_dropout=args.lora_dropout, # !!!!!!!!! bias="none", task_type="CAUSAL_LM", target_modules = ["c_proj", "c_attn", "q_attn"] @@ -269,26 +270,33 @@ def run_training(args, train_data, val_data): evaluation_strategy="steps", save_strategy="steps", load_best_model_at_end=True, - max_steps=args.max_steps, + max_steps=args.max_steps, # !!!!!!!!! eval_steps=args.eval_freq, save_steps=args.save_freq, logging_steps=args.log_freq, per_device_train_batch_size=args.batch_size, per_device_eval_batch_size=args.batch_size, - learning_rate=args.learning_rate, - lr_scheduler_type=args.lr_scheduler_type, - warmup_steps=args.num_warmup_steps, - gradient_accumulation_steps=args.gradient_accumulation_steps, + learning_rate=args.learning_rate, # !!!!!!!!! + lr_scheduler_type=args.lr_scheduler_type, # !!!!!!!!! + warmup_steps=args.num_warmup_steps, # !!!!!!!!! + gradient_accumulation_steps=args.gradient_accumulation_steps, # !!!!!!!!! gradient_checkpointing=not args.no_gradient_checkpointing, fp16=not args.no_fp16, bf16=args.bf16, - weight_decay=args.weight_decay, + weight_decay=args.weight_decay, # !!!!!!!!! run_name="StarCoder-finetuned", report_to="wandb", ddp_find_unused_parameters=False, + disable_tqdm=False, ) - trainer = Trainer(model=model, args=training_args, train_dataset=train_data, eval_dataset=val_data, callbacks=[SavePeftModelCallback, LoadBestPeftModelCallback]) + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_data, + eval_dataset=val_data, + callbacks=[SavePeftModelCallback, LoadBestPeftModelCallback], + ) print("Training...") trainer.train() diff --git a/finetune/merge_peft_adapters.py b/finetune/merge_peft_adapters.py index 218728f..31b8c29 100644 --- a/finetune/merge_peft_adapters.py +++ b/finetune/merge_peft_adapters.py @@ -1,4 +1,4 @@ -from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers import AutoModelForCausalLM, AutoTokenizer, logging from peft import PeftModel import torch @@ -9,11 +9,13 @@ def get_args(): parser = argparse.ArgumentParser() parser.add_argument("--base_model_name_or_path", type=str, default="bigcode/large-model") parser.add_argument("--peft_model_path", type=str, default="/") - parser.add_argument("--push_to_hub", action="store_true", default=True) + parser.add_argument("--push_to_hub", action="store_true", default=False) return parser.parse_args() def main(): + logging.set_verbosity_info() + logging.enable_progress_bar() args = get_args() base_model = AutoModelForCausalLM.from_pretrained( diff --git a/requirements.txt b/requirements.txt index cce4192..edc3826 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,159 @@ +accelerate==0.25.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +anyio @ file:///home/conda/feedstock_root/build_artifacts/anyio_1702909220329/work +appdirs==1.4.4 +argon2-cffi @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi_1692818318753/work +argon2-cffi-bindings @ file:///home/conda/feedstock_root/build_artifacts/argon2-cffi-bindings_1695386546427/work +arrow @ file:///home/conda/feedstock_root/build_artifacts/arrow_1696128962909/work +asttokens @ file:///home/conda/feedstock_root/build_artifacts/asttokens_1698341106958/work +async-lru @ file:///home/conda/feedstock_root/build_artifacts/async-lru_1690563019058/work +async-timeout==4.0.3 +attrs @ file:///home/conda/feedstock_root/build_artifacts/attrs_1704011227531/work +Babel @ file:///home/conda/feedstock_root/build_artifacts/babel_1702422572539/work +beautifulsoup4 @ file:///home/conda/feedstock_root/build_artifacts/beautifulsoup4_1680888073205/work +bitsandbytes==0.42.0 +bleach @ file:///home/conda/feedstock_root/build_artifacts/bleach_1696630167146/work +Brotli @ file:///home/conda/feedstock_root/build_artifacts/brotli-split_1695989787169/work +cached-property @ file:///home/conda/feedstock_root/build_artifacts/cached_property_1615209429212/work +certifi @ file:///home/conda/feedstock_root/build_artifacts/certifi_1700303426725/work/certifi +cffi @ file:///home/conda/feedstock_root/build_artifacts/cffi_1696001684923/work +charset-normalizer @ file:///home/conda/feedstock_root/build_artifacts/charset-normalizer_1698833585322/work +click==8.1.7 +comm @ file:///home/conda/feedstock_root/build_artifacts/comm_1704278392174/work +datasets==2.16.1 +debugpy @ file:///home/conda/feedstock_root/build_artifacts/debugpy_1695534290310/work +decorator @ file:///home/conda/feedstock_root/build_artifacts/decorator_1641555617451/work +defusedxml @ file:///home/conda/feedstock_root/build_artifacts/defusedxml_1615232257335/work +dill==0.3.6 +docker-pycreds==0.4.0 +entrypoints @ file:///home/conda/feedstock_root/build_artifacts/entrypoints_1643888246732/work +exceptiongroup @ file:///home/conda/feedstock_root/build_artifacts/exceptiongroup_1700579780973/work +executing @ file:///home/conda/feedstock_root/build_artifacts/executing_1698579936712/work +fastjsonschema @ file:///home/conda/feedstock_root/build_artifacts/python-fastjsonschema_1703780968325/work/dist +filelock==3.13.1 +fqdn @ file:///home/conda/feedstock_root/build_artifacts/fqdn_1638810296540/work/dist +frozenlist==1.4.1 +fsspec==2023.10.0 +gitdb==4.0.11 +GitPython==3.1.40 +huggingface-hub==0.20.2 +idna @ file:///home/conda/feedstock_root/build_artifacts/idna_1701026962277/work +importlib-metadata @ file:///home/conda/feedstock_root/build_artifacts/importlib-metadata_1703269254275/work +importlib-resources @ file:///home/conda/feedstock_root/build_artifacts/importlib_resources_1699364556997/work +inquirerpy==0.3.4 +ipykernel @ file:///home/conda/feedstock_root/build_artifacts/ipykernel_1703631723894/work +ipython @ file:///home/conda/feedstock_root/build_artifacts/ipython_1704718870316/work +ipywidgets==8.1.1 +isoduration @ file:///home/conda/feedstock_root/build_artifacts/isoduration_1638811571363/work/dist +jedi @ file:///home/conda/feedstock_root/build_artifacts/jedi_1696326070614/work +Jinja2 @ file:///home/conda/feedstock_root/build_artifacts/jinja2_1654302431367/work +json5 @ file:///home/conda/feedstock_root/build_artifacts/json5_1688248289187/work +jsonpointer @ file:///home/conda/feedstock_root/build_artifacts/jsonpointer_1695397238043/work +jsonschema @ file:///home/conda/feedstock_root/build_artifacts/jsonschema-meta_1700159890288/work +jsonschema-specifications @ file:///tmp/tmpkv1z7p57/src +jupyter-events @ file:///home/conda/feedstock_root/build_artifacts/jupyter_events_1699285872613/work +jupyter-lsp @ file:///home/conda/feedstock_root/build_artifacts/jupyter-lsp-meta_1701091994466/work/jupyter-lsp +jupyter_client @ file:///home/conda/feedstock_root/build_artifacts/jupyter_client_1699283905679/work +jupyter_core @ file:///home/conda/feedstock_root/build_artifacts/jupyter_core_1704727030956/work +jupyter_server @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_1704338572969/work +jupyter_server_terminals @ file:///home/conda/feedstock_root/build_artifacts/jupyter_server_terminals_1703611053195/work +jupyterlab @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_1703881392925/work +jupyterlab-widgets==3.0.9 +jupyterlab_pygments @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_pygments_1700744013163/work +jupyterlab_server @ file:///home/conda/feedstock_root/build_artifacts/jupyterlab_server-split_1700310846957/work +MarkupSafe @ file:///home/conda/feedstock_root/build_artifacts/markupsafe_1695367434228/work +matplotlib-inline @ file:///home/conda/feedstock_root/build_artifacts/matplotlib-inline_1660814786464/work +mistune @ file:///home/conda/feedstock_root/build_artifacts/mistune_1698947099619/work +mpmath==1.3.0 +multidict==6.0.4 +multiprocess==0.70.14 +nbclient @ file:///home/conda/feedstock_root/build_artifacts/nbclient_1684790896106/work +nbconvert @ file:///home/conda/feedstock_root/build_artifacts/nbconvert-meta_1704158779767/work +nbformat @ file:///home/conda/feedstock_root/build_artifacts/nbformat_1690814868471/work +nest-asyncio @ file:///home/conda/feedstock_root/build_artifacts/nest-asyncio_1697083700168/work +networkx==3.2.1 +notebook_shim @ file:///home/conda/feedstock_root/build_artifacts/notebook-shim_1682360583588/work +numpy==1.26.3 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.18.1 +nvidia-nvjitlink-cu12==12.3.101 +nvidia-nvtx-cu12==12.1.105 +overrides @ file:///home/conda/feedstock_root/build_artifacts/overrides_1691338815398/work +packaging @ file:///home/conda/feedstock_root/build_artifacts/packaging_1696202382185/work +pandas==2.1.4 +pandocfilters @ file:///home/conda/feedstock_root/build_artifacts/pandocfilters_1631603243851/work +parso @ file:///home/conda/feedstock_root/build_artifacts/parso_1638334955874/work +peft==0.7.1 +pexpect @ file:///home/conda/feedstock_root/build_artifacts/pexpect_1667297516076/work +pfzy==0.3.4 +pickleshare @ file:///home/conda/feedstock_root/build_artifacts/pickleshare_1602536217715/work +pkgutil_resolve_name @ file:///home/conda/feedstock_root/build_artifacts/pkgutil-resolve-name_1694617248815/work +platformdirs @ file:///home/conda/feedstock_root/build_artifacts/platformdirs_1701708255999/work +prometheus-client @ file:///home/conda/feedstock_root/build_artifacts/prometheus_client_1700579315247/work +prompt-toolkit @ file:///home/conda/feedstock_root/build_artifacts/prompt-toolkit_1702399386289/work +protobuf==4.25.1 +psutil @ file:///home/conda/feedstock_root/build_artifacts/psutil_1702833087123/work +ptyprocess @ file:///home/conda/feedstock_root/build_artifacts/ptyprocess_1609419310487/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl +pure-eval @ file:///home/conda/feedstock_root/build_artifacts/pure_eval_1642875951954/work +pyarrow==14.0.2 +pyarrow-hotfix==0.6 +pycparser @ file:///home/conda/feedstock_root/build_artifacts/pycparser_1636257122734/work +Pygments @ file:///home/conda/feedstock_root/build_artifacts/pygments_1700607939962/work +PySocks @ file:///home/conda/feedstock_root/build_artifacts/pysocks_1661604839144/work +python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/python-dateutil_1626286286081/work +python-json-logger @ file:///home/conda/feedstock_root/build_artifacts/python-json-logger_1677079630776/work +pytz @ file:///home/conda/feedstock_root/build_artifacts/pytz_1693930252784/work +PyYAML @ file:///home/conda/feedstock_root/build_artifacts/pyyaml_1695373428874/work +pyzmq @ file:///home/conda/feedstock_root/build_artifacts/pyzmq_1701783169502/work +referencing @ file:///home/conda/feedstock_root/build_artifacts/referencing_1704489226496/work +regex==2023.12.25 +requests @ file:///home/conda/feedstock_root/build_artifacts/requests_1684774241324/work +responses==0.18.0 +rfc3339-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3339-validator_1638811747357/work +rfc3986-validator @ file:///home/conda/feedstock_root/build_artifacts/rfc3986-validator_1598024191506/work +rpds-py @ file:///home/conda/feedstock_root/build_artifacts/rpds-py_1703822618592/work +safetensors==0.4.1 +scipy==1.11.4 +Send2Trash @ file:///home/conda/feedstock_root/build_artifacts/send2trash_1682601222253/work +sentry-sdk==1.39.1 +setproctitle==1.3.3 +six @ file:///home/conda/feedstock_root/build_artifacts/six_1620240208055/work +smmap==5.0.1 +sniffio @ file:///home/conda/feedstock_root/build_artifacts/sniffio_1662051266223/work +soupsieve @ file:///home/conda/feedstock_root/build_artifacts/soupsieve_1693929250441/work +stack-data @ file:///home/conda/feedstock_root/build_artifacts/stack_data_1669632077133/work +sympy==1.12 +terminado @ file:///home/conda/feedstock_root/build_artifacts/terminado_1699810101464/work +tinycss2 @ file:///home/conda/feedstock_root/build_artifacts/tinycss2_1666100256010/work +tokenizers==0.13.3 +tomli @ file:///home/conda/feedstock_root/build_artifacts/tomli_1644342247877/work +torch==2.1.2 +tornado @ file:///home/conda/feedstock_root/build_artifacts/tornado_1695373560918/work tqdm==4.65.0 +traitlets @ file:///home/conda/feedstock_root/build_artifacts/traitlets_1704212992681/work transformers==4.28.1 -datasets==2.11.0 -huggingface-hub==0.13.4 -accelerate==0.18.0 +triton==2.1.0 +types-python-dateutil @ file:///home/conda/feedstock_root/build_artifacts/types-python-dateutil_1704512562698/work +typing-utils @ file:///home/conda/feedstock_root/build_artifacts/typing_utils_1622899189314/work +typing_extensions @ file:///home/conda/feedstock_root/build_artifacts/typing_extensions_1702176139754/work +tzdata==2023.4 +uri-template @ file:///home/conda/feedstock_root/build_artifacts/uri-template_1688655812972/work/dist +urllib3 @ file:///home/conda/feedstock_root/build_artifacts/urllib3_1699933488691/work +wandb==0.16.1 +wcwidth @ file:///home/conda/feedstock_root/build_artifacts/wcwidth_1704731205417/work +webcolors @ file:///home/conda/feedstock_root/build_artifacts/webcolors_1679900785843/work +webencodings @ file:///home/conda/feedstock_root/build_artifacts/webencodings_1694681268211/work +websocket-client @ file:///home/conda/feedstock_root/build_artifacts/websocket-client_1701630677416/work +widgetsnbextension==4.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zipp @ file:///home/conda/feedstock_root/build_artifacts/zipp_1695255097490/work