From b2b4de36e9dd1d7d05ced90bf6614a19f9039a08 Mon Sep 17 00:00:00 2001 From: Ted Chen Date: Mon, 29 Apr 2024 10:11:19 +0800 Subject: [PATCH 1/4] test: add log to debug prompt --- llama_cpp/llama_chat_format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 71aac80614..6519bbc8fd 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -923,6 +923,7 @@ def format_llama2( if system_message: system_message = _system_template.format(system_message=system_message) _prompt = _format_llama2(system_message, _messages, " ", "") + "[/INST]" + print("llama-2 prompt:", _prompt) return ChatFormatterResponse(prompt=_prompt) @@ -1244,6 +1245,7 @@ def format_mistral_instruct( ): prompt += " [/INST]" + message["content"] + eos prompt += " [/INST]" + print("mistral-instruct prompt:", prompt) return ChatFormatterResponse(prompt=prompt, stop=stop) From 0b2e2c2a8ce2b8d0b2361bc436d065a115ddebaa Mon Sep 17 00:00:00 2001 From: Ted Chen Date: Mon, 29 Apr 2024 10:25:12 +0800 Subject: [PATCH 2/4] test: add log to debug prompt --- llama_cpp/llama_chat_format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 6519bbc8fd..37792f6864 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -924,6 +924,7 @@ def format_llama2( system_message = _system_template.format(system_message=system_message) _prompt = _format_llama2(system_message, _messages, " ", "") + "[/INST]" print("llama-2 prompt:", _prompt) + logger.info("llama-2 prompt: %s", _prompt) return ChatFormatterResponse(prompt=_prompt) @@ -1246,6 +1247,7 @@ def format_mistral_instruct( prompt += " [/INST]" + message["content"] + eos prompt += " [/INST]" print("mistral-instruct prompt:", prompt) + logger.info("mistral-instruct prompt: %s", prompt) return ChatFormatterResponse(prompt=prompt, stop=stop) From b4413247794c7408c7915462f734570234297790 Mon Sep 17 00:00:00 2001 From: Ted Chen Date: Mon, 29 Apr 2024 10:39:29 +0800 Subject: [PATCH 3/4] test: print to stderr --- llama_cpp/llama_chat_format.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 37792f6864..190475608a 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys import os import json import ctypes @@ -923,7 +924,7 @@ def format_llama2( if system_message: system_message = _system_template.format(system_message=system_message) _prompt = _format_llama2(system_message, _messages, " ", "") + "[/INST]" - print("llama-2 prompt:", _prompt) + print("llama-2 prompt:", _prompt, file=sys.stderr) logger.info("llama-2 prompt: %s", _prompt) return ChatFormatterResponse(prompt=_prompt) @@ -1246,7 +1247,7 @@ def format_mistral_instruct( ): prompt += " [/INST]" + message["content"] + eos prompt += " [/INST]" - print("mistral-instruct prompt:", prompt) + print("mistral-instruct prompt:", prompt, file=sys.stderr) logger.info("mistral-instruct prompt: %s", prompt) return ChatFormatterResponse(prompt=prompt, stop=stop) From 7113747f9a06fd729034474f3d8b7c26b3106a69 Mon Sep 17 00:00:00 2001 From: Ted Chen Date: Mon, 29 Apr 2024 11:09:30 +0800 Subject: [PATCH 4/4] test: log llama 3 prompt --- llama_cpp/llama_chat_format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 190475608a..717d6f4197 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -946,6 +946,8 @@ def format_llama3( _messages = _map_roles(messages, _roles) _messages.append((_roles["assistant"], None)) _prompt = _format_no_colon_single(_begin_token, _messages, _sep) + print("llama-3 prompt:", _prompt, file=sys.stderr) + logger.info("llama-3 prompt: %s", _prompt) return ChatFormatterResponse(prompt=_prompt, stop=_sep)