From e8a893156e097d8c964d0a8ff195b00fdb536fad Mon Sep 17 00:00:00 2001 From: Christopher Carroll Smith <75859865+chriscarrollsmith@users.noreply.github.com> Date: Wed, 12 Apr 2023 13:45:54 -0400 Subject: [PATCH 01/15] Implemented .env file API key storage --- .env.sample | 1 + .gitignore | 2 ++ README.md | 5 ++++- wolverine.py | 8 ++++---- 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 .env.sample diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..3ad03db --- /dev/null +++ b/.env.sample @@ -0,0 +1 @@ +OPENAI_API_KEY=your_api_key \ No newline at end of file diff --git a/.gitignore b/.gitignore index 673fe09..32fabad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ venv openai_key.txt +.venv +.env \ No newline at end of file diff --git a/README.md b/README.md index 55813f7..c4f589f 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,11 @@ For a quick demonstration see my [demo video on twitter](https://twitter.com/bio python3 -m venv venv source venv/bin/activate pip install -r requirements.txt + cp .env.sample .env -Add your openAI api key to `openai_key.txt` - _warning!_ by default this uses GPT-4 and may make many repeated calls to the api. +Add your openAI api key to `.env` + +_warning!_ By default wolverine uses GPT-4 and may make many repeated calls to the api. ## Example Usage diff --git a/wolverine.py b/wolverine.py index 42f7ff8..f1d910d 100644 --- a/wolverine.py +++ b/wolverine.py @@ -5,13 +5,14 @@ import os import shutil import subprocess import sys - import openai from termcolor import cprint +from dotenv import load_dotenv + # Set up the OpenAI API -with open("openai_key.txt") as f: - openai.api_key = f.read().strip() +load_dotenv() +openai.api_key = os.getenv("OPENAI_API_KEY") def run_script(script_name, script_args): @@ -51,7 +52,6 @@ def send_error_to_gpt(file_path, args, error_message, model): ) # print(prompt) - response = openai.ChatCompletion.create( model=model, messages=[ From 4863df689877d0628520a10346c6ad7cbb7cd9cd Mon Sep 17 00:00:00 2001 From: Hemang Joshi Date: Fri, 14 Apr 2023 10:27:32 +0530 Subject: [PATCH 02/15] added `star-history` added `star-history` --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 55813f7..7df05b7 100644 --- a/README.md +++ b/README.md @@ -36,3 +36,7 @@ This is just a quick prototype I threw together in a few hours. There are many p - multiple files / codebases: send GPT everything that appears in the stacktrace - graceful handling of large files - should we just send GPT relevant classes / functions? - extension to languages other than python + +## Star History + +[![Star History Chart](https://api.star-history.com/svg?repos=biobootloader/wolverine&type=Date)](https://star-history.com/#biobootloader/wolverine) From 6afb4db2ffc7878e2a125cd53917a1abfacb8790 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Fri, 14 Apr 2023 16:37:05 +0900 Subject: [PATCH 03/15] fix typo in README.md reliablity -> reliability --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 55813f7..0a8cae3 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ This is just a quick prototype I threw together in a few hours. There are many p - add flags to customize usage, such as asking for user confirmation before running changed code - further iterations on the edit format that GPT responds in. Currently it struggles a bit with indentation, but I'm sure that can be improved -- a suite of example buggy files that we can test prompts on to ensure reliablity and measure improvement +- a suite of example buggy files that we can test prompts on to ensure reliability and measure improvement - multiple files / codebases: send GPT everything that appears in the stacktrace - graceful handling of large files - should we just send GPT relevant classes / functions? - extension to languages other than python From 9af5480b89ec58edc11bf8b721eefc137e197d59 Mon Sep 17 00:00:00 2001 From: Christopher Carroll Smith <75859865+chriscarrollsmith@users.noreply.github.com> Date: Fri, 14 Apr 2023 16:53:02 -0400 Subject: [PATCH 04/15] Added python-dotenv to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 9d61d58..3923778 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ multidict==6.0.4 openai==0.27.2 pycodestyle==2.10.0 pyflakes==3.0.1 +python-dotenv==1.0.0 requests==2.28.2 six==1.16.0 termcolor==2.2.0 From 6343f6f50be274a49f850426c382cede5652ad62 Mon Sep 17 00:00:00 2001 From: biobootloader <128252497+biobootloader@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:59:31 -0700 Subject: [PATCH 05/15] Apply suggestions from code review --- .env.sample | 2 +- .gitignore | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.env.sample b/.env.sample index 3ad03db..9b73729 100644 --- a/.env.sample +++ b/.env.sample @@ -1 +1 @@ -OPENAI_API_KEY=your_api_key \ No newline at end of file +OPENAI_API_KEY=your_api_key diff --git a/.gitignore b/.gitignore index 32fabad..9c3fa66 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ venv -openai_key.txt .venv -.env \ No newline at end of file +.env From c62f91eaeed80501ca3bcc6d15d243bb8aa65a7a Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Thu, 13 Apr 2023 11:23:44 -0400 Subject: [PATCH 06/15] Update .gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9c3fa66..f617754 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ venv -.venv +openai_key.txt .env +env/ +.vscode/ From 7c072fba2ab3bea728ca5dc3c7b16dc05e3dc54a Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Thu, 13 Apr 2023 11:24:41 -0400 Subject: [PATCH 07/15] update prompt to make it pay attention to indentation --- prompt.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/prompt.txt b/prompt.txt index 4376ab2..ec0582a 100644 --- a/prompt.txt +++ b/prompt.txt @@ -4,10 +4,13 @@ Because you are part of an automated system, the format you respond in is very s In addition to the changes, please also provide short explanations of the what went wrong. A single explanation is required, but if you think it's helpful, feel free to provide more explanations for groups of more complicated changes. Be careful to use proper indentation and spacing in your changes. An example response could be: +Be ABSOLUTELY SURE to include the CORRECT INDENTATION when making replacements. + +example response: [ {"explanation": "this is just an example, this would usually be a brief explanation of what went wrong"}, {"operation": "InsertAfter", "line": 10, "content": "x = 1\ny = 2\nz = x * y"}, {"operation": "Delete", "line": 15, "content": ""}, - {"operation": "Replace", "line": 18, "content": "x += 1"}, + {"operation": "Replace", "line": 18, "content": " x += 1"}, {"operation": "Delete", "line": 20, "content": ""} ] From 0656a83da73dd9446406edabdcd607fc5bccc263 Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Thu, 13 Apr 2023 11:29:06 -0400 Subject: [PATCH 08/15] recursive calls if not json parsable - makes recursive calls to API (with a comment about it not being parsable) if response was not parsable - pass prompt.txt as system prompt - use env var for `DEFAULT_MODEL` - use env var for OPENAI_API_KEY --- wolverine.py | 103 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 29 deletions(-) diff --git a/wolverine.py b/wolverine.py index f1d910d..d9eff4a 100644 --- a/wolverine.py +++ b/wolverine.py @@ -1,18 +1,27 @@ import difflib -import fire import json import os import shutil import subprocess import sys -import openai + +import fire +from dotenv import load_dotenv from termcolor import cprint from dotenv import load_dotenv -# Set up the OpenAI API load_dotenv() -openai.api_key = os.getenv("OPENAI_API_KEY") + +import openai + +openai.api_key = os.environ.get("OPENAI_API_KEY") + +DEFAULT_MODEL = os.environ.get("DEFAULT_MODEL", "gpt-4") + + +with open("prompt.txt") as f: + SYSTEM_PROMPT = f.read() def run_script(script_name, script_args): @@ -26,7 +35,47 @@ def run_script(script_name, script_args): return result.decode("utf-8"), 0 -def send_error_to_gpt(file_path, args, error_message, model): +def send_error_to_gpt(file_path, args, error_message, model=DEFAULT_MODEL): + def json_validated_response(model, messages): + """ + This function is needed because the API can return a non-json response. + This will run recursively until a valid json response is returned. + """ + response = openai.ChatCompletion.create( + model=model, + messages=messages, + temperature=0.5, + ) + messages.append(response.choices[0].message) + content = response.choices[0].message.content + # see if json can be parsed + try: + json_start_index = content.index( + "[" + ) # find the starting position of the JSON data + json_data = content[ + json_start_index: + ] # extract the JSON data from the response string + json_response = json.loads(json_data) + except (json.decoder.JSONDecodeError, ValueError) as e: + cprint(f"{e}. Re-running the query.", "red") + # debug + cprint(f"\n\GPT RESPONSE:\n\n{content}\n\n", "yellow") + # append a user message that says the json is invalid + messages.append( + { + "role": "user", + "content": "Your response could not be parsed by json.loads. Please restate your last message as pure JSON.", + } + ) + # rerun the api call + return json_validated_response(model, messages) + except Exception as e: + cprint(f"Unknown error: {e}", "red") + cprint(f"\n\GPT RESPONSE:\n\n{content}\n\n", "yellow") + raise e + return json_response + with open(file_path, "r") as f: file_lines = f.readlines() @@ -35,12 +84,7 @@ def send_error_to_gpt(file_path, args, error_message, model): file_with_lines.append(str(i + 1) + ": " + line) file_with_lines = "".join(file_with_lines) - with open("prompt.txt") as f: - initial_prompt_text = f.read() - prompt = ( - initial_prompt_text + - "\n\n" "Here is the script that needs fixing:\n\n" f"{file_with_lines}\n\n" "Here are the arguments it was provided:\n\n" @@ -52,26 +96,27 @@ def send_error_to_gpt(file_path, args, error_message, model): ) # print(prompt) - response = openai.ChatCompletion.create( - model=model, - messages=[ - { - "role": "user", - "content": prompt, - } - ], - temperature=1.0, - ) + messages = [ + { + "role": "system", + "content": SYSTEM_PROMPT, + }, + { + "role": "user", + "content": prompt, + }, + ] - return response.choices[0].message.content.strip() + return json_validated_response(model, messages) -def apply_changes(file_path, changes_json): +def apply_changes(file_path, changes: list): + """ + Pass changes as loaded json (list of dicts) + """ with open(file_path, "r") as f: original_file_lines = f.readlines() - changes = json.loads(changes_json) - # Filter out explanation elements operation_changes = [change for change in changes if "operation" in change] explanations = [ @@ -114,7 +159,7 @@ def apply_changes(file_path, changes_json): print(line, end="") -def main(script_name, *script_args, revert=False, model="gpt-4"): +def main(script_name, *script_args, revert=False, model=DEFAULT_MODEL): if revert: backup_file = script_name + ".bak" if os.path.exists(backup_file): @@ -140,11 +185,11 @@ def main(script_name, *script_args, revert=False, model="gpt-4"): print("Output:", output) json_response = send_error_to_gpt( - file_path=script_name, - args=script_args, - error_message=output, - model=model, + file_path=script_name, + args=script_args, + error_message=output, ) + apply_changes(script_name, json_response) cprint("Changes applied. Rerunning...", "blue") From 923f7057e36016208f4dbfdf227a0953eba59c47 Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Thu, 13 Apr 2023 11:35:24 -0400 Subject: [PATCH 09/15] update readme - updated readme to mention .env - added model arg back --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e5e62e..401b812 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ For a quick demonstration see my [demo video on twitter](https://twitter.com/bio pip install -r requirements.txt cp .env.sample .env -Add your openAI api key to `.env` +Add your openAI api key to `.env` - _warning!_ by default this uses GPT-4 and may make many repeated calls to the api. -_warning!_ By default wolverine uses GPT-4 and may make many repeated calls to the api. + OPENAI_API_KEY=sk-abcdefghijklmopqrstuvwxyz ## Example Usage @@ -29,6 +29,10 @@ You can also run with other models, but be warned they may not adhere to the edi python wolverine.py --model=gpt-3.5-turbo buggy_script.py "subtract" 20 3 +If you want to use GPT-3.5 by default instead of GPT-4 add this to `.env`: + + DEFAULT_MODEL=gpt-3.5-turbo + ## Future Plans This is just a quick prototype I threw together in a few hours. There are many possible extensions and contributions are welcome: From 2497fb816b862ac1b5d27751c5742fb8171d4207 Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Fri, 14 Apr 2023 16:29:45 -0400 Subject: [PATCH 10/15] move json_validated_response to standalone function --- wolverine.py | 91 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/wolverine.py b/wolverine.py index d9eff4a..903bf0c 100644 --- a/wolverine.py +++ b/wolverine.py @@ -35,47 +35,49 @@ def run_script(script_name, script_args): return result.decode("utf-8"), 0 -def send_error_to_gpt(file_path, args, error_message, model=DEFAULT_MODEL): - def json_validated_response(model, messages): - """ - This function is needed because the API can return a non-json response. - This will run recursively until a valid json response is returned. - """ - response = openai.ChatCompletion.create( - model=model, - messages=messages, - temperature=0.5, +def json_validated_response(model, messages): + """ + This function is needed because the API can return a non-json response. + This will run recursively until a valid json response is returned. + todo: might want to stop after a certain number of retries + """ + response = openai.ChatCompletion.create( + model=model, + messages=messages, + temperature=0.5, + ) + messages.append(response.choices[0].message) + content = response.choices[0].message.content + # see if json can be parsed + try: + json_start_index = content.index( + "[" + ) # find the starting position of the JSON data + json_data = content[ + json_start_index: + ] # extract the JSON data from the response string + json_response = json.loads(json_data) + except (json.decoder.JSONDecodeError, ValueError) as e: + cprint(f"{e}. Re-running the query.", "red") + # debug + cprint(f"\n\GPT RESPONSE:\n\n{content}\n\n", "yellow") + # append a user message that says the json is invalid + messages.append( + { + "role": "user", + "content": "Your response could not be parsed by json.loads. Please restate your last message as pure JSON.", + } ) - messages.append(response.choices[0].message) - content = response.choices[0].message.content - # see if json can be parsed - try: - json_start_index = content.index( - "[" - ) # find the starting position of the JSON data - json_data = content[ - json_start_index: - ] # extract the JSON data from the response string - json_response = json.loads(json_data) - except (json.decoder.JSONDecodeError, ValueError) as e: - cprint(f"{e}. Re-running the query.", "red") - # debug - cprint(f"\n\GPT RESPONSE:\n\n{content}\n\n", "yellow") - # append a user message that says the json is invalid - messages.append( - { - "role": "user", - "content": "Your response could not be parsed by json.loads. Please restate your last message as pure JSON.", - } - ) - # rerun the api call - return json_validated_response(model, messages) - except Exception as e: - cprint(f"Unknown error: {e}", "red") - cprint(f"\n\GPT RESPONSE:\n\n{content}\n\n", "yellow") - raise e - return json_response + # rerun the api call + return json_validated_response(model, messages) + except Exception as e: + cprint(f"Unknown error: {e}", "red") + cprint(f"\nGPT RESPONSE:\n\n{content}\n\n", "yellow") + raise e + return json_response + +def send_error_to_gpt(file_path, args, error_message, model=DEFAULT_MODEL): with open(file_path, "r") as f: file_lines = f.readlines() @@ -106,6 +108,16 @@ def send_error_to_gpt(file_path, args, error_message, model=DEFAULT_MODEL): "content": prompt, }, ] + messages = [ + { + "role": "system", + "content": SYSTEM_PROMPT, + }, + { + "role": "user", + "content": prompt, + }, + ] return json_validated_response(model, messages) @@ -188,6 +200,7 @@ def main(script_name, *script_args, revert=False, model=DEFAULT_MODEL): file_path=script_name, args=script_args, error_message=output, + model=model, ) apply_changes(script_name, json_response) From dd174cf30eafca66a06e08654854c93ec5297fe0 Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Fri, 14 Apr 2023 17:15:07 -0400 Subject: [PATCH 11/15] add DEFAULT_MODEL to .env.sample + fix typo --- .env.sample | 1 + wolverine.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index 9b73729..f2177b3 100644 --- a/.env.sample +++ b/.env.sample @@ -1 +1,2 @@ OPENAI_API_KEY=your_api_key +DEFAULT_MODEL=gpt-3.5-turbo diff --git a/wolverine.py b/wolverine.py index 903bf0c..3b6fb4e 100644 --- a/wolverine.py +++ b/wolverine.py @@ -60,7 +60,7 @@ def json_validated_response(model, messages): except (json.decoder.JSONDecodeError, ValueError) as e: cprint(f"{e}. Re-running the query.", "red") # debug - cprint(f"\n\GPT RESPONSE:\n\n{content}\n\n", "yellow") + cprint(f"\nGPT RESPONSE:\n\n{content}\n\n", "yellow") # append a user message that says the json is invalid messages.append( { From b044882dc391d878c570d56ae5e64bb1045d0ec0 Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Fri, 14 Apr 2023 17:37:27 -0400 Subject: [PATCH 12/15] remove duplicate code from rebase --- wolverine.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/wolverine.py b/wolverine.py index 3b6fb4e..0501f1b 100644 --- a/wolverine.py +++ b/wolverine.py @@ -108,16 +108,6 @@ def send_error_to_gpt(file_path, args, error_message, model=DEFAULT_MODEL): "content": prompt, }, ] - messages = [ - { - "role": "system", - "content": SYSTEM_PROMPT, - }, - { - "role": "user", - "content": prompt, - }, - ] return json_validated_response(model, messages) From e1d0a790f8941a74c4857ac8404c4c1c9e4fb6ed Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Fri, 14 Apr 2023 17:46:18 -0400 Subject: [PATCH 13/15] cleanup --- .env.sample | 2 +- README.md | 6 +++--- wolverine.py | 9 +++------ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.env.sample b/.env.sample index f2177b3..2184c45 100644 --- a/.env.sample +++ b/.env.sample @@ -1,2 +1,2 @@ OPENAI_API_KEY=your_api_key -DEFAULT_MODEL=gpt-3.5-turbo +#DEFAULT_MODEL=gpt-3.5-turbo diff --git a/README.md b/README.md index 401b812..f671b4e 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ For a quick demonstration see my [demo video on twitter](https://twitter.com/bio pip install -r requirements.txt cp .env.sample .env -Add your openAI api key to `.env` - _warning!_ by default this uses GPT-4 and may make many repeated calls to the api. +Add your openAI api key to `.env` - OPENAI_API_KEY=sk-abcdefghijklmopqrstuvwxyz +_warning!_ by default this uses GPT-4 and may make many repeated calls to the api. ## Example Usage @@ -29,7 +29,7 @@ You can also run with other models, but be warned they may not adhere to the edi python wolverine.py --model=gpt-3.5-turbo buggy_script.py "subtract" 20 3 -If you want to use GPT-3.5 by default instead of GPT-4 add this to `.env`: +If you want to use GPT-3.5 by default instead of GPT-4 uncomment the default model line in `.env`: DEFAULT_MODEL=gpt-3.5-turbo diff --git a/wolverine.py b/wolverine.py index 0501f1b..535db10 100644 --- a/wolverine.py +++ b/wolverine.py @@ -1,21 +1,18 @@ import difflib +import fire import json import os import shutil import subprocess import sys +import openai -import fire -from dotenv import load_dotenv from termcolor import cprint from dotenv import load_dotenv load_dotenv() - -import openai - -openai.api_key = os.environ.get("OPENAI_API_KEY") +openai.api_key = os.getenv("OPENAI_API_KEY") DEFAULT_MODEL = os.environ.get("DEFAULT_MODEL", "gpt-4") From 4db9d1bf43438a7809e47bd1bd140cad8b3b12e1 Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Fri, 14 Apr 2023 17:49:09 -0400 Subject: [PATCH 14/15] more cleanup --- .gitignore | 2 +- wolverine.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f617754..da34ea7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ venv -openai_key.txt +.venv .env env/ .vscode/ diff --git a/wolverine.py b/wolverine.py index 535db10..194a0fc 100644 --- a/wolverine.py +++ b/wolverine.py @@ -6,11 +6,11 @@ import shutil import subprocess import sys import openai - from termcolor import cprint from dotenv import load_dotenv +# Set up the OpenAI API load_dotenv() openai.api_key = os.getenv("OPENAI_API_KEY") From fe87faa2fb709b782217093ecf88379e153a0f58 Mon Sep 17 00:00:00 2001 From: Felix Boehme Date: Fri, 14 Apr 2023 17:49:48 -0400 Subject: [PATCH 15/15] cleanup --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f671b4e..e3608e6 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ For a quick demonstration see my [demo video on twitter](https://twitter.com/bio Add your openAI api key to `.env` -_warning!_ by default this uses GPT-4 and may make many repeated calls to the api. +_warning!_ By default wolverine uses GPT-4 and may make many repeated calls to the api. ## Example Usage