From 708729a67d5521cd598e35a97722ff480ea2f31a Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Tue, 17 Sep 2024 21:14:40 +0200 Subject: [PATCH 01/13] create files in preparation of running the chat from the command line --- chatgptscriptrun/README.md | 6 + ...rengine-chatgptscript-toolsdefinition.json | 203 ++++++++++++++++++ ...chatgptscript-toolsdefinition.json.version | 1 + ...rate_chatgpt_script_toolsdefinition.prompt | 13 ++ .../generate_openai_toolsdefinition.prompt | 2 + .../generate_openai_toolsdefinition.sh | 9 + .../codeveloperengine-toolsdefinition.json | 174 +++++++++++++++ ...veloperengine-toolsdefinition.json.version | 1 + src/main/resources/static/index.html | 9 +- 9 files changed, 416 insertions(+), 2 deletions(-) create mode 100644 chatgptscriptrun/README.md create mode 100644 chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json create mode 100644 chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version create mode 100644 project-bin/generate_chatgpt_script_toolsdefinition.prompt create mode 100644 project-bin/generate_openai_toolsdefinition.prompt create mode 100755 project-bin/generate_openai_toolsdefinition.sh create mode 100644 src/main/resources/static/codeveloperengine-toolsdefinition.json create mode 100644 src/main/resources/static/codeveloperengine-toolsdefinition.json.version diff --git a/chatgptscriptrun/README.md b/chatgptscriptrun/README.md new file mode 100644 index 00000000..664cf279 --- /dev/null +++ b/chatgptscriptrun/README.md @@ -0,0 +1,6 @@ +# Running the codeveloper engine with the chatgpt command line tool + +As an alternative to running the CoDeveloper GPT Engine from ChatGPT you can run it with the chatgpt +script from my [ChatGPT Toolsuite](https://github.com/stoerr/chatGPTtools). +This directory contains a tools definition file for `chatgpt` - +[codeveloperengine-chatgptscript-toolsdefinition.json](codeveloperengine-chatgptscript-toolsdefinition.json) . diff --git a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json new file mode 100644 index 00000000..033eb99f --- /dev/null +++ b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json @@ -0,0 +1,203 @@ +[ + { + "function": { + "name": "executeExternalAction", + "description": "Runs a specified external action with optional arguments and input.", + "parameters": { + "type": "object", + "properties": { + "actionName": { + "type": "string", + "description": "The name of the action to execute." + }, + "arguments": { + "type": "string", + "description": "Optional additional arguments for the action." + }, + "actionInput": { + "type": "string", + "description": "Input for the action." + } + }, + "required": ["actionName", "actionInput"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + ] + }, + { + "function": { + "name": "fetchUrlTextContent", + "description": "Fetch text content from a given URL.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to fetch content from." + }, + "raw": { + "type": "boolean", + "description": "Return raw HTML or PDF content without converting to markdown." + } + }, + "required": ["url"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + ] + }, + { + "function": { + "name": "grepFiles", + "description": "Search for lines in text files matching the given regex.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the directory or file to search." + }, + "fileRegex": { + "type": "string", + "description": "Optional regex to filter file names." + }, + "grepRegex": { + "type": "string", + "description": "Regex to filter lines in the files." + }, + "contextLines": { + "type": "integer", + "description": "Number of context lines to include with each match." + } + }, + "required": ["grepRegex"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + ] + }, + { + "function": { + "name": "listFiles", + "description": "Recursively lists files in a directory with optional filters.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the directory to list." + }, + "recursive": { + "type": "boolean", + "description": "If true, lists files recursively." + }, + "filePathRegex": { + "type": "string", + "description": "Regex to filter file paths." + }, + "grepRegex": { + "type": "string", + "description": "Regex to filter files that contain a line matching this pattern." + }, + "listDirectories": { + "type": "boolean", + "description": "If true, lists directories instead of files." + } + } + } + }, + "commandline": [ + "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + ] + }, + { + "function": { + "name": "readFile", + "description": "Read a file's content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the file." + }, + "maxLines": { + "type": "integer", + "description": "Maximum number of lines to read from the file." + }, + "startLine": { + "type": "integer", + "description": "Line number to start reading from." + } + }, + "required": ["path"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + ] + }, + { + "function": { + "name": "replaceInFile", + "description": "Replaces occurrences of strings in a file.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the file." + }, + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "The literal string to be replaced." + }, + "replace": { + "type": "string", + "description": "Literal replacement." + } + }, + "required": ["search", "replace"] + } + } + }, + "required": ["path", "replacements"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + ] + }, + { + "function": { + "name": "writeFile", + "description": "Overwrite a file with new content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the file." + }, + "content": { + "type": "string", + "description": "Content to write to the file." + } + }, + "required": ["path", "content"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + ] + } +] \ No newline at end of file diff --git a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version new file mode 100644 index 00000000..0104381d --- /dev/null +++ b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version @@ -0,0 +1 @@ +AIGenVersion(52b2a757, generate_chatgpt_script_toolsdefinition.prompt-17ac1108, codeveloperengine-toolsdefinition.json-131770f5) \ No newline at end of file diff --git a/project-bin/generate_chatgpt_script_toolsdefinition.prompt b/project-bin/generate_chatgpt_script_toolsdefinition.prompt new file mode 100644 index 00000000..324dd9cb --- /dev/null +++ b/project-bin/generate_chatgpt_script_toolsdefinition.prompt @@ -0,0 +1,13 @@ +Transform the retrieved OpenAI tools definition file into the following format: + +[ + { + "function": { + // here comes the function definition of the tool + }, + "commandline: [ + "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + ] + }, + ... // more tools +] diff --git a/project-bin/generate_openai_toolsdefinition.prompt b/project-bin/generate_openai_toolsdefinition.prompt new file mode 100644 index 00000000..dd35b7ea --- /dev/null +++ b/project-bin/generate_openai_toolsdefinition.prompt @@ -0,0 +1,2 @@ +Print an OpenAI tools definition file that describes the requests specified in the retrieved OpenAPI YAML. +Set "strict" to false. diff --git a/project-bin/generate_openai_toolsdefinition.sh b/project-bin/generate_openai_toolsdefinition.sh new file mode 100755 index 00000000..51e242f3 --- /dev/null +++ b/project-bin/generate_openai_toolsdefinition.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# generates a json schema for using the engine within other tools +scriptdir=$(dirname $(realpath $0)) +cd $(dirname $0)/../ +aigenpipeline -wvf -o src/main/resources/static/codeveloperengine-toolsdefinition.json \ + -p $scriptdir/generate_openai_toolsdefinition.prompt src/test/resources/test-expected/codeveloperengine.yaml + +aigenpipeline -wvf -o chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json \ + -p $scriptdir/generate_chatgpt_script_toolsdefinition.prompt src/main/resources/static/codeveloperengine-toolsdefinition.json diff --git a/src/main/resources/static/codeveloperengine-toolsdefinition.json b/src/main/resources/static/codeveloperengine-toolsdefinition.json new file mode 100644 index 00000000..1156e48d --- /dev/null +++ b/src/main/resources/static/codeveloperengine-toolsdefinition.json @@ -0,0 +1,174 @@ +{ + "name": "Co-Developer GPT Engine", + "description": "API for executing external actions, fetching URL content, searching files, and managing file content.", + "version": "THEVERSION", + "strict": false, + "tools": [ + { + "name": "executeExternalAction", + "description": "Runs a specified external action with optional arguments and input.", + "parameters": { + "type": "object", + "properties": { + "actionName": { + "type": "string", + "description": "The name of the action to execute." + }, + "arguments": { + "type": "string", + "description": "Optional additional arguments for the action." + }, + "actionInput": { + "type": "string", + "description": "Input for the action." + } + }, + "required": ["actionName", "actionInput"] + } + }, + { + "name": "fetchUrlTextContent", + "description": "Fetch text content from a given URL.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to fetch content from." + }, + "raw": { + "type": "boolean", + "description": "Return raw HTML or PDF content without converting to markdown." + } + }, + "required": ["url"] + } + }, + { + "name": "grepFiles", + "description": "Search for lines in text files matching the given regex.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the directory or file to search." + }, + "fileRegex": { + "type": "string", + "description": "Optional regex to filter file names." + }, + "grepRegex": { + "type": "string", + "description": "Regex to filter lines in the files." + }, + "contextLines": { + "type": "integer", + "description": "Number of context lines to include with each match." + } + }, + "required": ["grepRegex"] + } + }, + { + "name": "listFiles", + "description": "Recursively lists files in a directory with optional filters.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the directory to list." + }, + "recursive": { + "type": "boolean", + "description": "If true, lists files recursively." + }, + "filePathRegex": { + "type": "string", + "description": "Regex to filter file paths." + }, + "grepRegex": { + "type": "string", + "description": "Regex to filter files that contain a line matching this pattern." + }, + "listDirectories": { + "type": "boolean", + "description": "If true, lists directories instead of files." + } + } + } + }, + { + "name": "readFile", + "description": "Read a file's content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the file." + }, + "maxLines": { + "type": "integer", + "description": "Maximum number of lines to read from the file." + }, + "startLine": { + "type": "integer", + "description": "Line number to start reading from." + } + }, + "required": ["path"] + } + }, + { + "name": "replaceInFile", + "description": "Replaces occurrences of strings in a file.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the file." + }, + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "The literal string to be replaced." + }, + "replace": { + "type": "string", + "description": "Literal replacement." + } + }, + "required": ["search", "replace"] + } + } + }, + "required": ["path", "replacements"] + } + }, + { + "name": "writeFile", + "description": "Overwrite a file with new content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Relative path to the file." + }, + "content": { + "type": "string", + "description": "Content to write to the file." + } + }, + "required": ["path", "content"] + } + } + ] +} \ No newline at end of file diff --git a/src/main/resources/static/codeveloperengine-toolsdefinition.json.version b/src/main/resources/static/codeveloperengine-toolsdefinition.json.version new file mode 100644 index 00000000..442a2456 --- /dev/null +++ b/src/main/resources/static/codeveloperengine-toolsdefinition.json.version @@ -0,0 +1 @@ +AIGenVersion(131770f5, generate_openai_toolsdefinition.prompt-2f0558bc, codeveloperengine.yaml-69a9e54b) \ No newline at end of file diff --git a/src/main/resources/static/index.html b/src/main/resources/static/index.html index bc927ea0..99a5d597 100644 --- a/src/main/resources/static/index.html +++ b/src/main/resources/static/index.html @@ -4,6 +4,11 @@

Co-Developer GPT Engine

-

See /.well-known/ai-plugin.json for the plugin description and - codeveloperengine.yaml for the available functions.

+

+ See /.well-known/ai-plugin.json for the plugin description and + codeveloperengine.yaml for the available functions. + codeveloperengine-toolsdefinition.json contains + the list of tools as appropriate for the OpenAI chat completion API. +

+ From 60f1b3db207c35ce5f2a8f5507368620654f9bce Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Tue, 17 Sep 2024 22:48:03 +0200 Subject: [PATCH 02/13] start implementing ExecuteOpenAIToolCallAction --- ...rengine-chatgptscript-toolsdefinition.json | 137 +++++++++++------- ...chatgptscript-toolsdefinition.json.version | 2 +- ...rate_chatgpt_script_toolsdefinition.prompt | 3 +- .../generate_openai_toolsdefinition.prompt | 4 +- .../codevengine/AbstractPluginAction.java | 2 +- .../codevengine/CoDeveloperEngine.java | 1 + .../ExecuteOpenAIToolCallAction.java | 46 ++++++ .../codeveloperengine-toolsdefinition.json | 123 +++++++++------- ...veloperengine-toolsdefinition.json.version | 2 +- 9 files changed, 211 insertions(+), 109 deletions(-) create mode 100644 src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java diff --git a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json index 033eb99f..0d3e6350 100644 --- a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json +++ b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json @@ -2,7 +2,7 @@ { "function": { "name": "executeExternalAction", - "description": "Runs a specified external action with optional arguments and input.", + "description": "Runs a specified external action (given as parameter actionName), optionally with additional arguments and input. Run \"listActions\" to get a list of all available actions. Only on explicit user request.", "parameters": { "type": "object", "properties": { @@ -14,17 +14,26 @@ "type": "string", "description": "Optional additional arguments for the action." }, - "actionInput": { - "type": "string", - "description": "Input for the action." + "requestBody": { + "type": "object", + "properties": { + "actionInput": { + "type": "string", + "description": "Input for the action." + } + } } }, - "required": ["actionName", "actionInput"] + "required": [ + "actionName", + "requestBody" + ] } }, "commandline": [ "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ] + ], + "stdin": "$toolcall" }, { "function": { @@ -39,7 +48,7 @@ }, "raw": { "type": "boolean", - "description": "Return raw HTML or PDF content without converting to markdown." + "description": "return raw html or pdf content without converting to markdown" } }, "required": ["url"] @@ -47,30 +56,31 @@ }, "commandline": [ "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ] + ], + "stdin": "$toolcall" }, { "function": { - "name": "grepFiles", + "name": "grepAction", "description": "Search for lines in text files matching the given regex.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the directory or file to search." + "description": "relative path to the directory to search in or the file to search. default is the root directory = '.'." }, "fileRegex": { "type": "string", - "description": "Optional regex to filter file names." + "description": "optional regex to filter file names" }, "grepRegex": { "type": "string", - "description": "Regex to filter lines in the files." + "description": "regex to filter lines in the files" }, "contextLines": { "type": "integer", - "description": "Number of context lines to include with each match." + "description": "number of context lines to include with each match (not yet used)" } }, "required": ["grepRegex"] @@ -78,60 +88,63 @@ }, "commandline": [ "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ] + ], + "stdin": "$toolcall" }, { "function": { "name": "listFiles", - "description": "Recursively lists files in a directory with optional filters.", + "description": "Recursively lists files in a directory. Optionally filters by filename and content.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the directory to list." + "description": "relative path to directory to list. default is the root directory = '.'." }, "recursive": { "type": "boolean", - "description": "If true, lists files recursively." + "description": "if true (default) lists files recursively, else only in that directory. In that case we will also list directories." }, "filePathRegex": { "type": "string", - "description": "Regex to filter file paths." + "description": "regex to filter file paths - use for search by file name" }, "grepRegex": { "type": "string", - "description": "Regex to filter files that contain a line matching this pattern." + "description": "an optional regex that lists only files that contain a line matching this pattern" }, "listDirectories": { "type": "boolean", - "description": "If true, lists directories instead of files." + "description": "if true, lists directories instead of files" } - } + }, + "required": [] } }, "commandline": [ "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ] + ], + "stdin": "$toolcall" }, { "function": { "name": "readFile", - "description": "Read a file's content.", + "description": "Read a files content.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the file." + "description": "relative path to file" }, "maxLines": { "type": "integer", - "description": "Maximum number of lines to read from the file." + "description": "maximum number of lines to read from the file" }, "startLine": { "type": "integer", - "description": "Line number to start reading from." + "description": "line number to start reading from; 1 is the first line" } }, "required": ["path"] @@ -139,65 +152,83 @@ }, "commandline": [ "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ] + ], + "stdin": "$toolcall" }, { "function": { "name": "replaceInFile", - "description": "Replaces occurrences of strings in a file.", + "description": "Replaces the single occurrence of one or more literal strings in a file. The whole file content is matched, not line by line.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the file." + "description": "relative path to file" }, - "replacements": { - "type": "array", - "items": { - "type": "object", - "properties": { - "search": { - "type": "string", - "description": "The literal string to be replaced." - }, - "replace": { - "type": "string", - "description": "Literal replacement." + "requestBody": { + "type": "object", + "properties": { + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "The literal string to be replaced - can contain many lines, but please take care to find a small number of lines to replace. Everything that is replaced must be here. Prefer to match the whole line / several whole lines." + }, + "replace": { + "type": "string", + "description": "Literal replacement, can contain several lines. Please observe the correct indentation." + } + } } - }, - "required": ["search", "replace"] + } } } }, - "required": ["path", "replacements"] + "required": [ + "path", + "requestBody" + ] } }, "commandline": [ "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ] + ], + "stdin": "$toolcall" }, { "function": { "name": "writeFile", - "description": "Overwrite a file with new content.", + "description": "Overwrite a small file with the complete content given in one step. You cannot append to a file or write parts or write parts - use replaceInFile for inserting parts.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the file." + "description": "relative path to file" }, - "content": { - "type": "string", - "description": "Content to write to the file." + "requestBody": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Content to write to the file." + } + } } }, - "required": ["path", "content"] + "required": [ + "path", + "requestBody" + ] } }, "commandline": [ "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ] + ], + "stdin": "$toolcall" } -] \ No newline at end of file +] diff --git a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version index 0104381d..437aa162 100644 --- a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version +++ b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version @@ -1 +1 @@ -AIGenVersion(52b2a757, generate_chatgpt_script_toolsdefinition.prompt-17ac1108, codeveloperengine-toolsdefinition.json-131770f5) \ No newline at end of file +AIGenVersion(7ec718e6, generate_chatgpt_script_toolsdefinition.prompt-7e1bc539, codeveloperengine-toolsdefinition.json-2762cfb1) \ No newline at end of file diff --git a/project-bin/generate_chatgpt_script_toolsdefinition.prompt b/project-bin/generate_chatgpt_script_toolsdefinition.prompt index 324dd9cb..176533eb 100644 --- a/project-bin/generate_chatgpt_script_toolsdefinition.prompt +++ b/project-bin/generate_chatgpt_script_toolsdefinition.prompt @@ -7,7 +7,8 @@ Transform the retrieved OpenAI tools definition file into the following format: }, "commandline: [ "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ] + ], + "stdin": "$toolcall" }, ... // more tools ] diff --git a/project-bin/generate_openai_toolsdefinition.prompt b/project-bin/generate_openai_toolsdefinition.prompt index dd35b7ea..cd970e51 100644 --- a/project-bin/generate_openai_toolsdefinition.prompt +++ b/project-bin/generate_openai_toolsdefinition.prompt @@ -1,2 +1,4 @@ Print an OpenAI tools definition file that describes the requests specified in the retrieved OpenAPI YAML. -Set "strict" to false. +Set "strict" to false. Include the descriptions unmodified. +Take care that the "required" attribute contains exactly the required attributes. +If the request is a POST with requestBody, then the body content should be contained in a parameter "requestBody". diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java index d75be939..5986a795 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java @@ -40,7 +40,7 @@ public abstract class AbstractPluginAction extends HttpServlet { */ public static final Pattern BINARY_FILES_PATTERN = Pattern.compile("(?i).*\\.(gif|png|mov|jpg|jpeg|mp4|mp3|pdf|zip|gz|tgz|tar|jar|class|war|ear|exe|dll|so|o|a|lib|bin|dat|dmg|iso)"); - private final transient Gson gson = new Gson(); + protected final transient Gson gson = new Gson(); /** * Logs an error and sends it to ChatGPT, always throws {@link ExecutionAbortedException}. diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java index b271f259..a050eb9e 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java @@ -148,6 +148,7 @@ protected static void initServlets() { addHandler(new ListFilesAction()); addHandler(new ReadFileAction()); addHandler(new GrepAction()); + addHandler(new ExecuteOpenAIToolCallAction()); if (writingEnabled) { addHandler(new WriteFileAction()); ExecuteExternalAction executeExternalAction = new ExecuteExternalAction(); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java new file mode 100644 index 00000000..a0eee2a7 --- /dev/null +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java @@ -0,0 +1,46 @@ +package net.stoerr.chatgpt.codevengine; + +import static net.stoerr.chatgpt.codevengine.TbUtils.logInfo; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.Map; +import java.util.stream.Collectors; + +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +/** + * Executes an OpenAI tool call coming in as JSON - for usage outside ChatGPT. + */ +public class ExecuteOpenAIToolCallAction extends AbstractPluginAction { + + @Override + public String getUrl() { + return "/executetool"; + } + + /** + * This is not registered in the yaml description since it's not a normal action, but rather + * distributes to actions. + */ + @Override + public String openApiDescription() { + return ""; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + BufferedReader reader = req.getReader(); + String json = reader.lines().collect(Collectors.joining(System.lineSeparator())); + logInfo("Received tool call:\n" + json); + String name = getBodyParameter(resp, json, "name", true); + String arguments = getBodyParameter(resp, json, "arguments", true); + Map parsedArguments = gson.fromJson(arguments, Map.class); + logInfo("Executing tool call: " + name + " " + parsedArguments); + String body = gson.toJson(parsedArguments.get("requestBody")); + logInfo("Body: " + body); + + } + +} diff --git a/src/main/resources/static/codeveloperengine-toolsdefinition.json b/src/main/resources/static/codeveloperengine-toolsdefinition.json index 1156e48d..f6c62396 100644 --- a/src/main/resources/static/codeveloperengine-toolsdefinition.json +++ b/src/main/resources/static/codeveloperengine-toolsdefinition.json @@ -1,12 +1,8 @@ { - "name": "Co-Developer GPT Engine", - "description": "API for executing external actions, fetching URL content, searching files, and managing file content.", - "version": "THEVERSION", - "strict": false, "tools": [ { "name": "executeExternalAction", - "description": "Runs a specified external action with optional arguments and input.", + "description": "Runs a specified external action (given as parameter actionName), optionally with additional arguments and input. Run \"listActions\" to get a list of all available actions. Only on explicit user request.", "parameters": { "type": "object", "properties": { @@ -18,12 +14,20 @@ "type": "string", "description": "Optional additional arguments for the action." }, - "actionInput": { - "type": "string", - "description": "Input for the action." + "requestBody": { + "type": "object", + "properties": { + "actionInput": { + "type": "string", + "description": "Input for the action." + } + } } }, - "required": ["actionName", "actionInput"] + "required": [ + "actionName", + "requestBody" + ] } }, { @@ -38,33 +42,33 @@ }, "raw": { "type": "boolean", - "description": "Return raw HTML or PDF content without converting to markdown." + "description": "return raw html or pdf content without converting to markdown" } }, "required": ["url"] } }, { - "name": "grepFiles", + "name": "grepAction", "description": "Search for lines in text files matching the given regex.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the directory or file to search." + "description": "relative path to the directory to search in or the file to search. default is the root directory = '.'." }, "fileRegex": { "type": "string", - "description": "Optional regex to filter file names." + "description": "optional regex to filter file names" }, "grepRegex": { "type": "string", - "description": "Regex to filter lines in the files." + "description": "regex to filter lines in the files" }, "contextLines": { "type": "integer", - "description": "Number of context lines to include with each match." + "description": "number of context lines to include with each match (not yet used)" } }, "required": ["grepRegex"] @@ -72,50 +76,51 @@ }, { "name": "listFiles", - "description": "Recursively lists files in a directory with optional filters.", + "description": "Recursively lists files in a directory. Optionally filters by filename and content.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the directory to list." + "description": "relative path to directory to list. default is the root directory = '.'." }, "recursive": { "type": "boolean", - "description": "If true, lists files recursively." + "description": "if true (default) lists files recursively, else only in that directory. In that case we will also list directories." }, "filePathRegex": { "type": "string", - "description": "Regex to filter file paths." + "description": "regex to filter file paths - use for search by file name" }, "grepRegex": { "type": "string", - "description": "Regex to filter files that contain a line matching this pattern." + "description": "an optional regex that lists only files that contain a line matching this pattern" }, "listDirectories": { "type": "boolean", - "description": "If true, lists directories instead of files." + "description": "if true, lists directories instead of files" } - } + }, + "required": [] } }, { "name": "readFile", - "description": "Read a file's content.", + "description": "Read a files content.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the file." + "description": "relative path to file" }, "maxLines": { "type": "integer", - "description": "Maximum number of lines to read from the file." + "description": "maximum number of lines to read from the file" }, "startLine": { "type": "integer", - "description": "Line number to start reading from." + "description": "line number to start reading from; 1 is the first line" } }, "required": ["path"] @@ -123,52 +128,68 @@ }, { "name": "replaceInFile", - "description": "Replaces occurrences of strings in a file.", + "description": "Replaces the single occurrence of one or more literal strings in a file. The whole file content is matched, not line by line.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the file." + "description": "relative path to file" }, - "replacements": { - "type": "array", - "items": { - "type": "object", - "properties": { - "search": { - "type": "string", - "description": "The literal string to be replaced." - }, - "replace": { - "type": "string", - "description": "Literal replacement." + "requestBody": { + "type": "object", + "properties": { + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "The literal string to be replaced - can contain many lines, but please take care to find a small number of lines to replace. Everything that is replaced must be here. Prefer to match the whole line / several whole lines." + }, + "replace": { + "type": "string", + "description": "Literal replacement, can contain several lines. Please observe the correct indentation." + } + } } - }, - "required": ["search", "replace"] + } } } }, - "required": ["path", "replacements"] + "required": [ + "path", + "requestBody" + ] } }, { "name": "writeFile", - "description": "Overwrite a file with new content.", + "description": "Overwrite a small file with the complete content given in one step. You cannot append to a file or write parts or write parts - use replaceInFile for inserting parts.", "parameters": { "type": "object", "properties": { "path": { "type": "string", - "description": "Relative path to the file." + "description": "relative path to file" }, - "content": { - "type": "string", - "description": "Content to write to the file." + "requestBody": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Content to write to the file." + } + } } }, - "required": ["path", "content"] + "required": [ + "path", + "requestBody" + ] } } - ] -} \ No newline at end of file + ], + "strict": false +} diff --git a/src/main/resources/static/codeveloperengine-toolsdefinition.json.version b/src/main/resources/static/codeveloperengine-toolsdefinition.json.version index 442a2456..b052c46c 100644 --- a/src/main/resources/static/codeveloperengine-toolsdefinition.json.version +++ b/src/main/resources/static/codeveloperengine-toolsdefinition.json.version @@ -1 +1 @@ -AIGenVersion(131770f5, generate_openai_toolsdefinition.prompt-2f0558bc, codeveloperengine.yaml-69a9e54b) \ No newline at end of file +AIGenVersion(2762cfb1, generate_openai_toolsdefinition.prompt-54a86373, codeveloperengine.yaml-69a9e54b) \ No newline at end of file From fa40ce4f0116d3cc3e686fd86eb98a20a5a30512 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Tue, 17 Sep 2024 23:08:01 +0200 Subject: [PATCH 03/13] continue ExecuteOpenAIToolCallAction --- chatgptscriptrun/README.md | 6 ++++ .../codevengine/CoDeveloperEngine.java | 2 +- .../ExecuteOpenAIToolCallAction.java | 35 +++++++++++++++++-- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/chatgptscriptrun/README.md b/chatgptscriptrun/README.md index 664cf279..12c5980a 100644 --- a/chatgptscriptrun/README.md +++ b/chatgptscriptrun/README.md @@ -4,3 +4,9 @@ As an alternative to running the CoDeveloper GPT Engine from ChatGPT you can run script from my [ChatGPT Toolsuite](https://github.com/stoerr/chatGPTtools). This directory contains a tools definition file for `chatgpt` - [codeveloperengine-chatgptscript-toolsdefinition.json](codeveloperengine-chatgptscript-toolsdefinition.json) . + +## Examples + +chatgpt -tf codeveloperengine-chatgptscript-toolsdefinition.json "run the tool list files for the directory '.'" + +chatgpt -tf codeveloperengine-chatgptscript-toolsdefinition.json "write 'hallo' to file foo.bar" diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java index a050eb9e..5f035d13 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java @@ -148,7 +148,7 @@ protected static void initServlets() { addHandler(new ListFilesAction()); addHandler(new ReadFileAction()); addHandler(new GrepAction()); - addHandler(new ExecuteOpenAIToolCallAction()); + addHandler(new ExecuteOpenAIToolCallAction(HANDLERS)); if (writingEnabled) { addHandler(new WriteFileAction()); ExecuteExternalAction executeExternalAction = new ExecuteExternalAction(); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java index a0eee2a7..8252e26e 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java @@ -4,10 +4,13 @@ import java.io.BufferedReader; import java.io.IOException; +import java.io.StringReader; import java.util.Map; import java.util.stream.Collectors; +import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; import jakarta.servlet.http.HttpServletResponse; /** @@ -15,6 +18,12 @@ */ public class ExecuteOpenAIToolCallAction extends AbstractPluginAction { + private final Map handlers; + + public ExecuteOpenAIToolCallAction(Map handlers) { + this.handlers = handlers; + } + @Override public String getUrl() { return "/executetool"; @@ -30,7 +39,7 @@ public String openApiDescription() { } @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { BufferedReader reader = req.getReader(); String json = reader.lines().collect(Collectors.joining(System.lineSeparator())); logInfo("Received tool call:\n" + json); @@ -38,9 +47,31 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I String arguments = getBodyParameter(resp, json, "arguments", true); Map parsedArguments = gson.fromJson(arguments, Map.class); logInfo("Executing tool call: " + name + " " + parsedArguments); - String body = gson.toJson(parsedArguments.get("requestBody")); + Object requestBody = parsedArguments.get("requestBody"); + String body = requestBody != null ? gson.toJson(requestBody) : null; logInfo("Body: " + body); + AbstractPluginAction handler = handlers.get("/" + name); + if (null == handler) { + sendError(resp, HttpServletResponse.SC_BAD_REQUEST, "No handler for tool call: " + name); + } + // call handler with a request that has parsedArguments as parameters and body as request body (JSON request) + HttpServletRequest requestWrapper = new HttpServletRequestWrapper(req) { + @Override + public String getParameter(String name) { + return String.valueOf(parsedArguments.get(name)); + } + + @Override + public BufferedReader getReader() throws IOException { + return body != null ? new BufferedReader(new StringReader(body)) : null; + } + @Override + public String getMethod() { + return requestBody != null ? "POST" : "GET"; + } + }; + handler.service(requestWrapper, resp); } } From 6f9fb9db8f2065443a9487d53e8cf6e9a8222aaf Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Wed, 18 Sep 2024 07:42:21 +0200 Subject: [PATCH 04/13] add quiet switch to suppress info level logging --- .../codevengine/CoDeveloperEngine.java | 38 +++++++++---------- .../stoerr/chatgpt/codevengine/TbUtils.java | 7 +++- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java index 5f035d13..7e23d362 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java @@ -23,9 +23,7 @@ import org.apache.commons.cli.ParseException; import org.apache.commons.lang3.StringUtils; import org.apache.pdfbox.io.IOUtils; -import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.RequestLog; -import org.eclipse.jetty.server.Response; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ResourceHandler; import org.eclipse.jetty.servlet.FilterHolder; @@ -113,24 +111,21 @@ public class CoDeveloperEngine { chain.doFilter(rawRequest, rawResponse); }; - private static final RequestLog requestlog = new RequestLog() { - @Override - public void log(Request request, Response response) { - TbUtils.logInfo("Remote address: " + request.getRemoteAddr()); - TbUtils.logInfo("Remote host: " + request.getRemoteHost()); - TbUtils.logInfo("Remote port: " + request.getRemotePort()); - TbUtils.logInfo("Requestlog: " + request.getMethod() + " " + request.getRequestURL() + (request.getQueryString() != null && !request.getQueryString().isEmpty() ? "?" + request.getQueryString() : "") + " " + response.getStatus()); - // list all request headers - for (Enumeration e = request.getHeaderNames(); e.hasMoreElements(); ) { - String header = e.nextElement(); - TbUtils.logInfo("Request header: " + header + ": " + request.getHeader(header)); - } - // list all response headers - for (String header : response.getHeaderNames()) { - TbUtils.logInfo("Response header: " + header + ": " + response.getHeader(header)); - } - TbUtils.logInfo(""); + private static final RequestLog requestlog = (request, response) -> { + TbUtils.logInfo("Remote address: " + request.getRemoteAddr()); + TbUtils.logInfo("Remote host: " + request.getRemoteHost()); + TbUtils.logInfo("Remote port: " + request.getRemotePort()); + TbUtils.logInfo("Requestlog: " + request.getMethod() + " " + request.getRequestURL() + (request.getQueryString() != null && !request.getQueryString().isEmpty() ? "?" + request.getQueryString() : "") + " " + response.getStatus()); + // list all request headers + for (Enumeration e = request.getHeaderNames(); e.hasMoreElements(); ) { + String header = e.nextElement(); + TbUtils.logInfo("Request header: " + header + ": " + request.getHeader(header)); + } + // list all response headers + for (String header : response.getHeaderNames()) { + TbUtils.logInfo("Response header: " + header + ": " + response.getHeader(header)); } + TbUtils.logInfo(""); }; private static void addHandler(AbstractPluginAction handler) { @@ -263,6 +258,7 @@ private static void parseOptions(String[] args) { options.addOption("h", "help", false, "Display this help message"); options.addOption("g", "globalconfigdir", true, "Directory for global configuration (default: ~/.cgptcodeveloperglobal/"); options.addOption("l", "local", false, "Only use local configuration via options - ignore any global configuration"); + options.addOption("q", "quiet", false, "Suppress info level output"); CommandLineParser parser = new DefaultParser(); @@ -294,6 +290,10 @@ private static void parseOptions(String[] args) { userGlobalConfigDir = null; ignoreGlobalConfig = true; } + + if (cmd.hasOption("q")) { + TbUtils.setQuiet(true); + } } catch (ParseException e) { TbUtils.logError("Error parsing command line options: " + e); System.exit(1); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java b/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java index 758c3873..d7195fe4 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java @@ -28,6 +28,7 @@ public class TbUtils { public static final PrintStream LOG = System.out; static boolean isLoggingEnabled = true; + static boolean quiet; /** * If there is a file named .cgptcodeveloper/.requestlog.txt, we append the request data to it. @@ -72,7 +73,7 @@ static void logError(String msg) { } static void logInfo(String msg) { - if (isLoggingEnabled) { + if (isLoggingEnabled && !quiet) { LOG.println(msg); } } @@ -184,4 +185,8 @@ private static String rangeDescription(Range lastRange) { } return rangeDescr; } + + public static void setQuiet(boolean quiet) { + TbUtils.quiet = quiet; + } } From a66466b137cd95b5b8bd58a7a4f69353faa86217 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Thu, 19 Sep 2024 22:42:15 +0200 Subject: [PATCH 05/13] move tools definition file for chatgpt script to the server --- project-bin/generate_openai_toolsdefinition.sh | 2 +- .../codeveloperengine-chatgptscript-toolsdefinition.json | 0 ...codeveloperengine-chatgptscript-toolsdefinition.json.version | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {chatgptscriptrun => src/main/resources/static}/codeveloperengine-chatgptscript-toolsdefinition.json (100%) rename {chatgptscriptrun => src/main/resources/static}/codeveloperengine-chatgptscript-toolsdefinition.json.version (100%) diff --git a/project-bin/generate_openai_toolsdefinition.sh b/project-bin/generate_openai_toolsdefinition.sh index 51e242f3..c1e6d891 100755 --- a/project-bin/generate_openai_toolsdefinition.sh +++ b/project-bin/generate_openai_toolsdefinition.sh @@ -5,5 +5,5 @@ cd $(dirname $0)/../ aigenpipeline -wvf -o src/main/resources/static/codeveloperengine-toolsdefinition.json \ -p $scriptdir/generate_openai_toolsdefinition.prompt src/test/resources/test-expected/codeveloperengine.yaml -aigenpipeline -wvf -o chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json \ +aigenpipeline -wvf -o src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json \ -p $scriptdir/generate_chatgpt_script_toolsdefinition.prompt src/main/resources/static/codeveloperengine-toolsdefinition.json diff --git a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json similarity index 100% rename from chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json rename to src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json diff --git a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version similarity index 100% rename from chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json.version rename to src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version From 4a3af9795c1ef7beb263a40782c33a12bc637b3e Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Thu, 19 Sep 2024 22:42:39 +0200 Subject: [PATCH 06/13] add hidden option --aitoolsdef that prints the tools definition --- .../stoerr/chatgpt/codevengine/CoDeveloperEngine.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java index 7e23d362..33274397 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java @@ -224,6 +224,14 @@ private static String getMainUrl(HttpServletRequest request) { } public static void main(String[] args) throws Exception { + if (args.length == 1 && args[0].equals("--aitoolsdef")) { + // hidden option to output a description of the operations for use with my chatgpt script to stdout + try (InputStream in = CoDeveloperEngine.class.getResourceAsStream("/static/codeveloperengine-chatgptscript-toolsdefinition.json")) { + IOUtils.copy(in, System.out); + } + return; + } + TbUtils.logVersion(); parseOptions(args); @@ -293,6 +301,7 @@ private static void parseOptions(String[] args) { if (cmd.hasOption("q")) { TbUtils.setQuiet(true); + // todo change loglevels } } catch (ParseException e) { TbUtils.logError("Error parsing command line options: " + e); From b64fa6c0adcf975c110c3995d0e2e9ec8b6d2c80 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Thu, 19 Sep 2024 22:43:08 +0200 Subject: [PATCH 07/13] little script to run cat + engine on the command line --- bin/pmcodevgpt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 bin/pmcodevgpt diff --git a/bin/pmcodevgpt b/bin/pmcodevgpt new file mode 100755 index 00000000..52843c66 --- /dev/null +++ b/bin/pmcodevgpt @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# "Poor mans" co developer engine: run it with the chatgpt script from https://github.com/stoerr/chatGPTtools + +# start the co-developer-gpt-engine.jar in the directory this script is placed, following links +progfile=$(realpath $0) +progdir=$(dirname "$progfile") + +JAVA=java +# if jenv is in the path, we use the version that is set for the directory this script is in, +# since the current dir could use some ridiculously low version. +if which jenv >/dev/null 2>&1; then + JAVA=$(cd $progdir; jenv which java) +fi + +$JAVA -jar "$progdir/co-developer-gpt-engine.jar" -w & +pid=$! +trap "kill $pid" EXIT + +sleep 1 + +chatgpt -tf <($JAVA -jar "$progdir/co-developer-gpt-engine.jar" --aitoolsdef) -cr "$@" From f5fb41650115cac0aff4209e6415dc2b038c2ea0 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Fri, 20 Sep 2024 10:19:39 +0200 Subject: [PATCH 08/13] various fixes related to listing directories --- ...rengine-chatgptscript-toolsdefinition.json | 1 + .../codevengine/AbstractPluginAction.java | 31 +- .../codevengine/CoDeveloperEngine.java | 10 + .../ExecuteOpenAIToolCallAction.java | 9 +- .../chatgpt/codevengine/GrepAction.java | 8 +- .../chatgpt/codevengine/ListFilesAction.java | 26 +- .../chatgpt/codevengine/ReadFileAction.java | 2 +- .../chatgpt/codevengine/ReplaceAction.java | 2 +- .../codevengine/ReplaceRegexAction.java | 2 +- .../stoerr/chatgpt/codevengine/TbUtils.java | 2 + ...rengine-chatgptscript-toolsdefinition.json | 483 ++++++++++-------- 11 files changed, 321 insertions(+), 255 deletions(-) create mode 120000 chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json diff --git a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json new file mode 120000 index 00000000..18edf714 --- /dev/null +++ b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json @@ -0,0 +1 @@ +../src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json \ No newline at end of file diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java index 5986a795..f1879971 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java @@ -62,10 +62,9 @@ protected static ExecutionAbortedException sendError(HttpServletResponse respons protected static Stream findMatchingFiles( boolean suppressMessage, HttpServletResponse response, Path path, Pattern filePathPattern, - Pattern grepPattern, boolean recursive) { + Pattern grepPattern, boolean recursive, boolean listDirectories) { boolean haveFilePathPattern = filePathPattern != null && !filePathPattern.pattern().isEmpty(); List result = new ArrayList<>(); - boolean returnDirectories = !recursive; try { Files.walkFileTree(path, new SimpleFileVisitor() { @Override @@ -74,7 +73,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th (!recursive && !dir.equals(path))) { return FileVisitResult.SKIP_SUBTREE; } - if (returnDirectories) { + if (listDirectories || !recursive) { result.add(dir); } return super.preVisitDirectory(dir, attrs); @@ -83,7 +82,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { FileVisitResult res = super.visitFile(file, attrs); - if (!isIgnored(file)) { + if (!isIgnored(file) && !listDirectories) { result.add(file); } return res; @@ -95,11 +94,15 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO List matchingFiles = result.stream() .filter(p -> !haveFilePathPattern || filePathPattern.matcher(p.toString()).find()) - .filter(f -> returnDirectories || Files.isRegularFile(f)) + .filter(f -> listDirectories && Files.isDirectory(f) || Files.isRegularFile(f)) .collect(toList()); if (matchingFiles.isEmpty() && !suppressMessage) { - String similarFilesMessage = getSimilarFilesMessage(response, path, filePathPattern != null ? filePathPattern.toString() : ""); - throw sendError(response, 404, "No files found matching filePathRegex: " + filePathPattern + "\n\n" + similarFilesMessage); + if (filePathPattern != null) { + String similarFilesMessage = getSimilarFilesMessage(response, path, filePathPattern != null ? filePathPattern.toString() : "", listDirectories); + throw sendError(response, 404, "No files found matching filePathRegex: " + filePathPattern + "\n\n" + similarFilesMessage); + } else { + throw sendError(response, 404, "No files found in " + path); + } } Collections.sort(matchingFiles); // make it deterministic. @@ -184,7 +187,7 @@ protected Path getPath(HttpServletRequest request, HttpServletResponse response, if (mustExist && !Files.exists(resolved)) { String message = "Path " + path + " does not exist! Try to list files with /listFiles to find the right path."; String filename = resolved.getFileName().toString(); - String similarFilesMessage = getSimilarFilesMessage(response, CoDeveloperEngine.currentDir, filename); + String similarFilesMessage = getSimilarFilesMessage(response, CoDeveloperEngine.currentDir, filename, false); if (!similarFilesMessage.isEmpty()) { message += "\n\n" + similarFilesMessage; } @@ -193,17 +196,17 @@ protected Path getPath(HttpServletRequest request, HttpServletResponse response, return resolved; } - protected static String getSimilarFilesMessage(HttpServletResponse response, Path path, String filename) { + protected static String getSimilarFilesMessage(HttpServletResponse response, Path path, String filename, boolean listDirectories) { String similarFilesMessage = ""; - List matchingFiles = findMatchingFiles(true, response, path, null, null, true) + List matchingFiles = findMatchingFiles(true, response, path, null, null, true, listDirectories) .collect(toList()); List files = matchingFiles.stream() - .map(p -> CoDeveloperEngine.currentDir.relativize(p).toString()) + .map(CoDeveloperEngine::canonicalName) .filter(p -> p.contains("/" + filename)) .limit(5) .collect(toList()); matchingFiles.stream() - .map(p -> CoDeveloperEngine.currentDir.relativize(p).toString()) + .map(CoDeveloperEngine::canonicalName) .map(p -> Pair.of(p, StringUtils.getFuzzyDistance(p, filename, Locale.getDefault()))) .map(p -> Pair.of(p.getLeft(), -p.getRight())) .sorted(Comparator.comparingDouble(Pair::getRight)) @@ -243,10 +246,6 @@ protected String getBodyParameter(HttpServletResponse response, String json, Str return parameterValue; } - protected String mappedFilename(Path path) { - return CoDeveloperEngine.currentDir.relativize(path).toString(); - } - protected String abbreviate(String s, int max) { if (s == null || s.length() <= max) { return s; diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java index 33274397..1b3441b8 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -317,4 +318,13 @@ public static void execute(Runnable runnable) { server.getThreadPool().execute(runnable); } + public static final String canonicalName(Path path) { + if (!path.toAbsolutePath().startsWith(currentDir.toAbsolutePath())) { + throw new IllegalArgumentException("Bug: trying to return file not in current dir - " + path); + } + String file = currentDir.relativize(path).toString(); + file = file.isEmpty() ? "." : file; + return Files.isDirectory(path) ? file + "/" : file; + } + } diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java index 8252e26e..26ace490 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java @@ -8,6 +8,8 @@ import java.util.Map; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; + import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletRequestWrapper; @@ -45,11 +47,11 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S logInfo("Received tool call:\n" + json); String name = getBodyParameter(resp, json, "name", true); String arguments = getBodyParameter(resp, json, "arguments", true); - Map parsedArguments = gson.fromJson(arguments, Map.class); + Map parsedArguments = gson.fromJson(arguments, Map.class); logInfo("Executing tool call: " + name + " " + parsedArguments); Object requestBody = parsedArguments.get("requestBody"); String body = requestBody != null ? gson.toJson(requestBody) : null; - logInfo("Body: " + body); + if (StringUtils.isNotBlank(body)) logInfo("Body: " + body); AbstractPluginAction handler = handlers.get("/" + name); if (null == handler) { sendError(resp, HttpServletResponse.SC_BAD_REQUEST, "No handler for tool call: " + name); @@ -58,7 +60,8 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S HttpServletRequest requestWrapper = new HttpServletRequestWrapper(req) { @Override public String getParameter(String name) { - return String.valueOf(parsedArguments.get(name)); + Object value = parsedArguments.get(name); + return value != null ? value.toString() : null; } @Override diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/GrepAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/GrepAction.java index e62a4249..41084f73 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/GrepAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/GrepAction.java @@ -98,7 +98,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO throw sendError(resp, 404, "Path is not readable: " + startPath); } - List matchingFiles = findMatchingFiles(false, resp, startPath, filePattern, grepPattern, true) + List matchingFiles = findMatchingFiles(false, resp, startPath, filePattern, grepPattern, true, false) .collect(Collectors.toList()); if (!matchingFiles.isEmpty()) { StringBuilder buf = new StringBuilder(); @@ -136,7 +136,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO resp.setContentType("text/plain;charset=UTF-8"); resp.getWriter().write(buf.toString()); } else { - long filePathFileCount = findMatchingFiles(true, resp, startPath, filePattern, null, true).count(); + long filePathFileCount = findMatchingFiles(true, resp, startPath, filePattern, null, true, false).count(); if (filePathFileCount > 0) throw sendError(resp, 404, "Found " + filePathFileCount + " files whose name is matching the filePathRegex but none of them contain a line matching the grepRegex."); else if (Files.isDirectory(startPath)) { @@ -151,9 +151,9 @@ else if (Files.isDirectory(startPath)) { private void appendBlock(List lines, StringBuilder buf, Path path, int start, int end) { if (start == end - 1) { - buf.append("======================== ").append(mappedFilename(path)).append(" line ").append(start + 1).append('\n'); + buf.append("======================== ").append(CoDeveloperEngine.canonicalName(path)).append(" line ").append(start + 1).append('\n'); } else { - buf.append("======================== ").append(mappedFilename(path)).append(" lines ").append(start + 1).append(" to ").append(end).append('\n'); + buf.append("======================== ").append(CoDeveloperEngine.canonicalName(path)).append(" lines ").append(start + 1).append(" to ").append(end).append('\n'); } for (int j = start; j < end; j++) { buf.append(lines.get(j)).append('\n'); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ListFilesAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ListFilesAction.java index 7eca9dea..9ff25da7 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ListFilesAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ListFilesAction.java @@ -4,7 +4,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Comparator; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -76,7 +75,8 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO Path path = getPath(req, resp, true, true); String filePathRegex = getQueryParam(req, "filePathRegex"); String grepRegex = getQueryParam(req, "grepRegex"); - String listDirectories = getQueryParam(req, "listDirectories"); + String listDirectoriesRaw = getQueryParam(req, "listDirectories"); + boolean listDirectories = Boolean.parseBoolean(listDirectoriesRaw); String recursiveRaw = getQueryParam(req, "recursive"); boolean recursive = recursiveRaw == null || Boolean.parseBoolean(recursiveRaw); RepeatedRequestChecker.CHECKER.checkRequestRepetition(resp, this, path, filePathRegex, grepRegex, recursiveRaw, listDirectories); @@ -95,28 +95,24 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO if (Files.isDirectory(path)) { resp.setContentType("text/plain;charset=UTF-8"); - List paths = findMatchingFiles(false, resp, path, filePathPattern, grepPattern, recursive) + List paths = findMatchingFiles(false, resp, path, filePathPattern, grepPattern, recursive, listDirectories) .collect(Collectors.toList()); List files = paths.stream() - .map(this::mappedFilename) + .map(path1 -> CoDeveloperEngine.canonicalName(path1)) .filter(StringUtils::isNotBlank) .collect(Collectors.toList()); if (files.isEmpty()) { - long filePathFileCount = findMatchingFiles(false, resp, path, filePathPattern, null, recursive).count(); - if (filePathFileCount > 0) - throw sendError(resp, 404, "Found " + filePathFileCount + " files whose name is matching the filePathRegex but none of them contain a line matching the grepRegex."); - else if (Files.newDirectoryStream(path).iterator().hasNext()) { - String similarFilesMessage = getSimilarFilesMessage(resp, path, filePathPattern != null ? filePathPattern.toString() : ""); + if (grepPattern != null) { + long filePathFileCount = findMatchingFiles(false, resp, path, filePathPattern, null, recursive, listDirectories).count(); + if (filePathFileCount > 0) + throw sendError(resp, 404, "Found " + filePathFileCount + " files whose name is matching the filePathRegex but none of them contain a line matching the grepRegex."); + } + if (Files.newDirectoryStream(path).iterator().hasNext()) { + String similarFilesMessage = getSimilarFilesMessage(resp, path, filePathPattern != null ? filePathPattern.toString() : "", listDirectories); throw sendError(resp, 404, "No files found matching filePathRegex: " + filePathRegex + "\n\n" + similarFilesMessage); } else { throw sendError(resp, 404, "No files found in directory: " + path); } - } else if ("TRUE".equalsIgnoreCase(listDirectories)) { - files = paths.stream().map(Path::getParent).distinct() - .sorted(Comparator.comparing(Path::toString)) - .map(f -> mappedFilename(f)) - .map(f -> StringUtils.defaultIfEmpty(f, ".") + "/") - .collect(Collectors.toList()); } else if (files.size() > 100) { long directoryCount = paths.stream().map(Path::getParent).distinct().count(); throw sendError(resp, 404, "Found " + files.size() + " files in " + directoryCount + " directories - please use a more specific path or filePathRegex, or use listDirectories instead to get an overview and then list specific directories you're interested in."); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ReadFileAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ReadFileAction.java index eb194348..be938f27 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ReadFileAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ReadFileAction.java @@ -111,7 +111,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO } if (maxLines <= fulllinecount || startLine > 1 || dropped != 0) { content = "CAUTION: Lines " + startLine + " to " + (startLine + lines.size() - 1) + " of " + fulllinecount + - " lines of file " + CoDeveloperEngine.currentDir.relativize(path) + " start now. " + + " lines of file " + CoDeveloperEngine.canonicalName(path) + " start now. " + "To get more of the file content repeat read request with startLine=" + (startLine + lines.size()) + " , or use the grepAction with enough contextLines if you are searching for something specific.\n\n" + content + "\n\n" + diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceAction.java index ddcacace..b9f62a39 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceAction.java @@ -152,7 +152,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I resp.getWriter().write(totalReplacementCount + " replacement; modified line(s) " + String.join(", ", modifiedLineDescr)); } catch (NoSuchFileException e) { - throw sendError(resp, 404, "File not found: " + CoDeveloperEngine.currentDir.relativize(path)); + throw sendError(resp, 404, "File not found: " + CoDeveloperEngine.canonicalName(path)); } catch (IOException e) { throw sendError(resp, 500, "Error reading or writing file : " + e); } catch (IllegalArgumentException e) { diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceRegexAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceRegexAction.java index 24f5dba8..bc28f8a1 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceRegexAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceRegexAction.java @@ -158,7 +158,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I resp.getWriter().write("Replaced " + replacementCount + " occurrences of pattern; modified lines " + String.join(", ", modifiedLineDescr)); } catch (NoSuchFileException e) { - throw sendError(resp, 404, "File not found: " + CoDeveloperEngine.currentDir.relativize(path)); + throw sendError(resp, 404, "File not found: " + CoDeveloperEngine.canonicalName(path)); } catch (IOException e) { throw sendError(resp, 500, "Error reading or writing file : " + e); } catch (PatternSyntaxException e) { diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java b/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java index d7195fe4..00838489 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java @@ -66,10 +66,12 @@ protected static void logBody(String parameterName, String parameterValue) { static void logStacktrace(Exception e) { e.printStackTrace(ERRLOG); + ERRLOG.println(); } static void logError(String msg) { ERRLOG.println(msg); + ERRLOG.println(); } static void logInfo(String msg) { diff --git a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json index 0d3e6350..ee8c74e0 100644 --- a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json +++ b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json @@ -1,234 +1,289 @@ [ - { - "function": { - "name": "executeExternalAction", - "description": "Runs a specified external action (given as parameter actionName), optionally with additional arguments and input. Run \"listActions\" to get a list of all available actions. Only on explicit user request.", - "parameters": { - "type": "object", - "properties": { - "actionName": { - "type": "string", - "description": "The name of the action to execute." - }, - "arguments": { - "type": "string", - "description": "Optional additional arguments for the action." - }, - "requestBody": { - "type": "object", - "properties": { - "actionInput": { - "type": "string", - "description": "Input for the action." - } - } - } - }, - "required": [ - "actionName", - "requestBody" - ] + { + "function": { + "name": "executeExternalAction", + "description": "Runs a specified external action (given as parameter actionName), optionally with additional arguments and input. Run \"listActions\" to get a list of all available actions. Only on explicit user request.", + "parameters": { + "type": "object", + "properties": { + "actionName": { + "type": "string", + "description": "The name of the action to execute." + }, + "arguments": { + "type": "string", + "description": "Optional additional arguments for the action." + }, + "requestBody": { + "type": "object", + "properties": { + "actionInput": { + "type": "string", + "description": "Input for the action." + } } + } }, - "commandline": [ - "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" + "required": [ + "actionName", + "requestBody" + ] + } }, - { - "function": { - "name": "fetchUrlTextContent", - "description": "Fetch text content from a given URL.", - "parameters": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "The URL to fetch content from." - }, - "raw": { - "type": "boolean", - "description": "return raw html or pdf content without converting to markdown" - } - }, - "required": ["url"] - } + "commandline": [ + "curl", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + "@-", + "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "fetchUrlTextContent", + "description": "Fetch text content from a given URL.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to fetch content from." + }, + "raw": { + "type": "boolean", + "description": "return raw html or pdf content without converting to markdown" + } }, - "commandline": [ - "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" + "required": [ + "url" + ] + } }, - { - "function": { - "name": "grepAction", - "description": "Search for lines in text files matching the given regex.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to the directory to search in or the file to search. default is the root directory = '.'." - }, - "fileRegex": { - "type": "string", - "description": "optional regex to filter file names" - }, - "grepRegex": { - "type": "string", - "description": "regex to filter lines in the files" - }, - "contextLines": { - "type": "integer", - "description": "number of context lines to include with each match (not yet used)" - } - }, - "required": ["grepRegex"] - } + "commandline": [ + "curl", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + "@-", + "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "grepAction", + "description": "Search for lines in text files matching the given regex.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to the directory to search in or the file to search. default is the root directory = '.'." + }, + "fileRegex": { + "type": "string", + "description": "optional regex to filter file names" + }, + "grepRegex": { + "type": "string", + "description": "regex to filter lines in the files" + }, + "contextLines": { + "type": "integer", + "description": "number of context lines to include with each match (not yet used)" + } }, - "commandline": [ - "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" + "required": [ + "grepRegex" + ] + } }, - { - "function": { - "name": "listFiles", - "description": "Recursively lists files in a directory. Optionally filters by filename and content.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to directory to list. default is the root directory = '.'." - }, - "recursive": { - "type": "boolean", - "description": "if true (default) lists files recursively, else only in that directory. In that case we will also list directories." - }, - "filePathRegex": { - "type": "string", - "description": "regex to filter file paths - use for search by file name" - }, - "grepRegex": { - "type": "string", - "description": "an optional regex that lists only files that contain a line matching this pattern" - }, - "listDirectories": { - "type": "boolean", - "description": "if true, lists directories instead of files" - } - }, - "required": [] - } + "commandline": [ + "curl", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + "@-", + "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "listFiles", + "description": "Recursively lists files in a directory. Optionally filters by filename and content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to directory to list. default is the root directory = '.'." + }, + "recursive": { + "type": "boolean", + "description": "if true (default) lists files recursively, else only in that directory. In that case we will also list directories." + }, + "filePathRegex": { + "type": "string", + "description": "regex to filter file paths - use for search by file name" + }, + "grepRegex": { + "type": "string", + "description": "an optional regex that lists only files that contain a line matching this pattern" + }, + "listDirectories": { + "type": "boolean", + "description": "if true, lists directories instead of files" + } }, - "commandline": [ - "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" + "required": [] + } }, - { - "function": { - "name": "readFile", - "description": "Read a files content.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to file" - }, - "maxLines": { - "type": "integer", - "description": "maximum number of lines to read from the file" - }, - "startLine": { - "type": "integer", - "description": "line number to start reading from; 1 is the first line" - } - }, - "required": ["path"] - } + "commandline": [ + "curl", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + "@-", + "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "readFile", + "description": "Read a files content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "maxLines": { + "type": "integer", + "description": "maximum number of lines to read from the file" + }, + "startLine": { + "type": "integer", + "description": "line number to start reading from; 1 is the first line" + } }, - "commandline": [ - "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" + "required": [ + "path" + ] + } }, - { - "function": { - "name": "replaceInFile", - "description": "Replaces the single occurrence of one or more literal strings in a file. The whole file content is matched, not line by line.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to file" + "commandline": [ + "curl", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + "@-", + "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "replaceInFile", + "description": "Replaces the single occurrence of one or more literal strings in a file. The whole file content is matched, not line by line.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "requestBody": { + "type": "object", + "properties": { + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "The literal string to be replaced - can contain many lines, but please take care to find a small number of lines to replace. Everything that is replaced must be here. Prefer to match the whole line / several whole lines." }, - "requestBody": { - "type": "object", - "properties": { - "replacements": { - "type": "array", - "items": { - "type": "object", - "properties": { - "search": { - "type": "string", - "description": "The literal string to be replaced - can contain many lines, but please take care to find a small number of lines to replace. Everything that is replaced must be here. Prefer to match the whole line / several whole lines." - }, - "replace": { - "type": "string", - "description": "Literal replacement, can contain several lines. Please observe the correct indentation." - } - } - } - } - } + "replace": { + "type": "string", + "description": "Literal replacement, can contain several lines. Please observe the correct indentation." } - }, - "required": [ - "path", - "requestBody" - ] + } + } + } } + } }, - "commandline": [ - "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" + "required": [ + "path", + "requestBody" + ] + } }, - { - "function": { - "name": "writeFile", - "description": "Overwrite a small file with the complete content given in one step. You cannot append to a file or write parts or write parts - use replaceInFile for inserting parts.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to file" - }, - "requestBody": { - "type": "object", - "properties": { - "content": { - "type": "string", - "description": "Content to write to the file." - } - } - } - }, - "required": [ - "path", - "requestBody" - ] + "commandline": [ + "curl", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + "@-", + "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "writeFile", + "description": "Overwrite a small file with the complete content given in one step. You cannot append to a file or write parts or write parts - use replaceInFile for inserting parts.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "requestBody": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Content to write to the file." + } } + } }, - "commandline": [ - "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" - } + "required": [ + "path", + "requestBody" + ] + } + }, + "commandline": [ + "curl", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "-d", + "@-", + "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + } ] From 1bff7dfd4084ef182b7f5eb2558d762b8fe1178d Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Fri, 20 Sep 2024 12:36:03 +0200 Subject: [PATCH 09/13] various fixes related to pmcodevgpt usage --- bin/pmcodevgpt | 4 ++-- project-bin/generate_openai_toolsdefinition.sh | 4 ++-- .../chatgpt/codevengine/CoDeveloperEngine.java | 3 --- .../ExecuteOpenAIToolCallAction.java | 18 ++++++++++++++++-- .../stoerr/chatgpt/codevengine/TbUtils.java | 18 +++++++++++++++++- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/bin/pmcodevgpt b/bin/pmcodevgpt index 52843c66..a1d6613a 100755 --- a/bin/pmcodevgpt +++ b/bin/pmcodevgpt @@ -12,10 +12,10 @@ if which jenv >/dev/null 2>&1; then JAVA=$(cd $progdir; jenv which java) fi -$JAVA -jar "$progdir/co-developer-gpt-engine.jar" -w & +$JAVA -jar "$progdir/co-developer-gpt-engine.jar" -w -q & pid=$! trap "kill $pid" EXIT -sleep 1 +sleep 2 chatgpt -tf <($JAVA -jar "$progdir/co-developer-gpt-engine.jar" --aitoolsdef) -cr "$@" diff --git a/project-bin/generate_openai_toolsdefinition.sh b/project-bin/generate_openai_toolsdefinition.sh index c1e6d891..1d6b798b 100755 --- a/project-bin/generate_openai_toolsdefinition.sh +++ b/project-bin/generate_openai_toolsdefinition.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # generates a json schema for using the engine within other tools -scriptdir=$(dirname $(realpath $0)) -cd $(dirname $0)/../ +scriptdir=$(dirname "$(realpath $0)") +cd "$(dirname $0)/../" aigenpipeline -wvf -o src/main/resources/static/codeveloperengine-toolsdefinition.json \ -p $scriptdir/generate_openai_toolsdefinition.prompt src/test/resources/test-expected/codeveloperengine.yaml diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java index 1b3441b8..dd71f89b 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java @@ -57,8 +57,6 @@ public class CoDeveloperEngine { */ public static final Pattern OVERRIDE_IGNORE_PATTERN = Pattern.compile(".*/.github/.*|.*/.content.xml|(.*/)?\\.chatgpt.*.md|.*\\.htaccess"); - // private static final Gson GSON = new Gson(); - static Path currentDir = Paths.get(".").normalize().toAbsolutePath(); private static int port; @@ -302,7 +300,6 @@ private static void parseOptions(String[] args) { if (cmd.hasOption("q")) { TbUtils.setQuiet(true); - // todo change loglevels } } catch (ParseException e) { TbUtils.logError("Error parsing command line options: " + e); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java index 26ace490..e1d3c85e 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java @@ -55,13 +55,21 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S AbstractPluginAction handler = handlers.get("/" + name); if (null == handler) { sendError(resp, HttpServletResponse.SC_BAD_REQUEST, "No handler for tool call: " + name); + return; } // call handler with a request that has parsedArguments as parameters and body as request body (JSON request) HttpServletRequest requestWrapper = new HttpServletRequestWrapper(req) { @Override public String getParameter(String name) { Object value = parsedArguments.get(name); - return value != null ? value.toString() : null; + if (value == null) return null; + if (value instanceof String) return (String) value; + if (value instanceof Double) { + // check whether it's an integer + double d = (Double) value; + if (Math.abs(d - Math.round(d)) < 0.001) return "" + Math.round(d); + } + return String.valueOf(value); } @Override @@ -74,7 +82,13 @@ public String getMethod() { return requestBody != null ? "POST" : "GET"; } }; - handler.service(requestWrapper, resp); + try { + handler.service(requestWrapper, resp); + } catch (ServletException | IOException | RuntimeException e) { + TbUtils.logError("Error executing tool call: " + name + "\n" + arguments); + TbUtils.logStacktrace(e); + throw e; + } } } diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java b/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java index 00838489..5eef3076 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java @@ -14,8 +14,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.LoggerFactory; + import com.google.common.collect.Range; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -30,6 +35,10 @@ public class TbUtils { static boolean isLoggingEnabled = true; static boolean quiet; + private TbUtils() { + throw new IllegalStateException("Utility class"); + } + /** * If there is a file named .cgptcodeveloper/.requestlog.txt, we append the request data to it. */ @@ -71,7 +80,6 @@ static void logStacktrace(Exception e) { static void logError(String msg) { ERRLOG.println(msg); - ERRLOG.println(); } static void logInfo(String msg) { @@ -190,5 +198,13 @@ private static String rangeDescription(Range lastRange) { public static void setQuiet(boolean quiet) { TbUtils.quiet = quiet; + // change logback root logger and org.eclipse.jetty logger to WARN level + if (quiet) { + LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); + Logger rootLogger = loggerContext.getLogger(Logger.ROOT_LOGGER_NAME); + rootLogger.setLevel(Level.WARN); + Logger jettyLogger = loggerContext.getLogger("org.eclipse.jetty"); + jettyLogger.setLevel(Level.WARN); + } } } From f829bedbca3f6b942b50a022cc317de244824dc2 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Fri, 20 Sep 2024 19:41:54 +0200 Subject: [PATCH 10/13] be quieter on quiet --- .../net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java | 8 +++++--- .../net/stoerr/chatgpt/codevengine/UserGlobalConfig.java | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java index dd71f89b..18b8b39b 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java @@ -231,9 +231,11 @@ public static void main(String[] args) throws Exception { return; } - TbUtils.logVersion(); - - parseOptions(args); + try { + parseOptions(args); + } finally { + TbUtils.logVersion(); + } server = new Server(new InetSocketAddress("127.0.0.1", port)); context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); context.setContextPath("/"); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/UserGlobalConfig.java b/src/main/java/net/stoerr/chatgpt/codevengine/UserGlobalConfig.java index fb553210..8167a937 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/UserGlobalConfig.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/UserGlobalConfig.java @@ -119,7 +119,7 @@ public boolean readAndCheckConfiguration(@Nullable String globalConfigDir) throw externport = Integer.parseInt(config.getProperty("externport", "443")); if (httpsPort == null || httpsPort <= 0) { - TbUtils.logError("httpsport property in " + configFile + " is not set - our own https is disabled, relying on a tunnel."); + TbUtils.logInfo("httpsport property in " + configFile + " is not set - our own https is disabled, relying on a tunnel."); // that is OK if we use a tunnel instead of doing https ourselves return true; } else { From 4c546b35a1485c7a0e7c46a71ca96e4d4b5ba097 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Fri, 20 Sep 2024 19:47:50 +0200 Subject: [PATCH 11/13] remove unneccesary content type header on requests --- .../generate_chatgpt_script_toolsdefinition.prompt | 2 +- ...eloperengine-chatgptscript-toolsdefinition.json | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/project-bin/generate_chatgpt_script_toolsdefinition.prompt b/project-bin/generate_chatgpt_script_toolsdefinition.prompt index 176533eb..df938589 100644 --- a/project-bin/generate_chatgpt_script_toolsdefinition.prompt +++ b/project-bin/generate_chatgpt_script_toolsdefinition.prompt @@ -6,7 +6,7 @@ Transform the retrieved OpenAI tools definition file into the following format: // here comes the function definition of the tool }, "commandline: [ - "curl", "-X", "POST", "-H", "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" ], "stdin": "$toolcall" }, diff --git a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json index ee8c74e0..7646e28c 100644 --- a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json +++ b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json @@ -34,8 +34,6 @@ "curl", "-X", "POST", - "-H", - "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" @@ -67,8 +65,6 @@ "curl", "-X", "POST", - "-H", - "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" @@ -108,8 +104,6 @@ "curl", "-X", "POST", - "-H", - "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" @@ -151,8 +145,6 @@ "curl", "-X", "POST", - "-H", - "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" @@ -188,8 +180,6 @@ "curl", "-X", "POST", - "-H", - "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" @@ -239,8 +229,6 @@ "curl", "-X", "POST", - "-H", - "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" @@ -278,8 +266,6 @@ "curl", "-X", "POST", - "-H", - "Content-Type: application/json", "-d", "@-", "http://localhost:3002/executetool" From 38933f54f8860076d8432c800d50ef0cb0e8057b Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Fri, 20 Sep 2024 21:51:44 +0200 Subject: [PATCH 12/13] documentation of pmcodevgpt --- README.md | 4 +++- bin/pmcodevgpt | 8 +++++++- chatgptscriptrun/README.md | 12 ------------ src/site/markdown/pmcodevgpt.md | 21 +++++++++++++++++++++ src/site/site.xml | 1 + 5 files changed, 32 insertions(+), 14 deletions(-) delete mode 100644 chatgptscriptrun/README.md create mode 100644 src/site/markdown/pmcodevgpt.md diff --git a/README.md b/README.md index 6848ba57..db0fa3ef 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ and have it execute (e.g. build and test) actions locally to support you in your development processes? Then this might be for you. The CoDeveloperGPTengine provide the actions for a OpenAI [GPT](https://openai.com/blog/introducing-gpts) for read or even write access to the files in the local directory it is started in. -It can also work as a [ChatGPT plugin](https://openai.com/blog/chatgpt-plugins). +It can also work as a [ChatGPT plugin](https://openai.com/blog/chatgpt-plugins) (OK, that's rather obsolete now) and +as a chat on the command line with the chatgpt script from my +[ChatGPT Toolsuite](https://github.com/stoerr/chatGPTtools). In contrast to other approaches like [AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) this is not meant to autonomously execute extensive changes (which would likely require a lot of prompt engineering), but to enable the diff --git a/bin/pmcodevgpt b/bin/pmcodevgpt index a1d6613a..2506b866 100755 --- a/bin/pmcodevgpt +++ b/bin/pmcodevgpt @@ -18,4 +18,10 @@ trap "kill $pid" EXIT sleep 2 -chatgpt -tf <($JAVA -jar "$progdir/co-developer-gpt-engine.jar" --aitoolsdef) -cr "$@" +ARGS="" +# if $* doesn't contain -cr or -ca we add -cr to ARGS +if [[ ! "$*" =~ -cr ]] && [[ ! "$*" =~ -ca ]]; then + ARGS="-cr" +fi + +chatgpt -tf <($JAVA -jar "$progdir/co-developer-gpt-engine.jar" --aitoolsdef) $ARGS "$@" diff --git a/chatgptscriptrun/README.md b/chatgptscriptrun/README.md deleted file mode 100644 index 12c5980a..00000000 --- a/chatgptscriptrun/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Running the codeveloper engine with the chatgpt command line tool - -As an alternative to running the CoDeveloper GPT Engine from ChatGPT you can run it with the chatgpt -script from my [ChatGPT Toolsuite](https://github.com/stoerr/chatGPTtools). -This directory contains a tools definition file for `chatgpt` - -[codeveloperengine-chatgptscript-toolsdefinition.json](codeveloperengine-chatgptscript-toolsdefinition.json) . - -## Examples - -chatgpt -tf codeveloperengine-chatgptscript-toolsdefinition.json "run the tool list files for the directory '.'" - -chatgpt -tf codeveloperengine-chatgptscript-toolsdefinition.json "write 'hallo' to file foo.bar" diff --git a/src/site/markdown/pmcodevgpt.md b/src/site/markdown/pmcodevgpt.md new file mode 100644 index 00000000..fd90dcbf --- /dev/null +++ b/src/site/markdown/pmcodevgpt.md @@ -0,0 +1,21 @@ +# Running the codeveloper engine with the chatgpt command line tool + +As an alternative to running the CoDeveloper GPT Engine from ChatGPT you can run it with the chatgpt +script from my [ChatGPT Toolsuite](https://github.com/stoerr/chatGPTtools). This not a full fledged chat interface, +(hence the script name `pmcodevgpt`= "Poor Mans CoDEVeloper GPT"), +but starts up within a second and can be used for quick tasks. You'll need an +[OpenAI API key](https://platform.openai.com/api-keys) to use it, though - +either in an environment variable `OPENAI_API_KEY` or in a file `~/.openai-api-key.txt` . +The `pmcodevgpt` script starts the CoDeveloper GPT Engine in the background and the +[chatgpt](https://github.com/stoerr/chatGPTtools/blob/develop/bin/chatgpt) script with a tools definition +that is generated from the OpenAPI description of the CoDeveloper GPT Engine. + +If you call pmcodevgpt, you can type your prompt and end it with `/end` on a line of its own, or press Ctrl-D to end +each message. After processing and printing the response this starts again - abort the program with Ctrl-C. + +If you like you could also use the audio chat feature of the chatgpt script to talk to the CoDeveloper GPT Engine. +You can call `pmcodevgpt -ca` to start the audio chat - follow the instructions it prints to the console. You can +dictate your prompts, but the output will be written to the console. + +BTW: if you know any open source models / interfaces that support a function calling / tools interface like +OpenAI does, please let me know! I'd like to try that / integrate that, too. diff --git a/src/site/site.xml b/src/site/site.xml index fc68bee5..85e58f88 100644 --- a/src/site/site.xml +++ b/src/site/site.xml @@ -48,6 +48,7 @@ + From 6e905db9cc200bb3582c929fca2e6a12e7c889f9 Mon Sep 17 00:00:00 2001 From: "Dr. Hans-Peter Stoerr" Date: Fri, 20 Sep 2024 21:51:51 +0200 Subject: [PATCH 13/13] misc --- ...rengine-chatgptscript-toolsdefinition.json | 1 - .../ExecuteOpenAIToolCallAction.java | 2 + ...rengine-chatgptscript-toolsdefinition.json | 471 ++++++++---------- ...chatgptscript-toolsdefinition.json.version | 2 +- 4 files changed, 218 insertions(+), 258 deletions(-) delete mode 120000 chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json diff --git a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json b/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json deleted file mode 120000 index 18edf714..00000000 --- a/chatgptscriptrun/codeveloperengine-chatgptscript-toolsdefinition.json +++ /dev/null @@ -1 +0,0 @@ -../src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json \ No newline at end of file diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java index e1d3c85e..217e9f9f 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java @@ -84,6 +84,8 @@ public String getMethod() { }; try { handler.service(requestWrapper, resp); + } catch (ExecutionAbortedException e) { + // is already sufficiently handled. Just ignore. } catch (ServletException | IOException | RuntimeException e) { TbUtils.logError("Error executing tool call: " + name + "\n" + arguments); TbUtils.logStacktrace(e); diff --git a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json index 7646e28c..9ad1c334 100644 --- a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json +++ b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json @@ -1,275 +1,234 @@ [ - { - "function": { - "name": "executeExternalAction", - "description": "Runs a specified external action (given as parameter actionName), optionally with additional arguments and input. Run \"listActions\" to get a list of all available actions. Only on explicit user request.", - "parameters": { - "type": "object", - "properties": { - "actionName": { - "type": "string", - "description": "The name of the action to execute." - }, - "arguments": { - "type": "string", - "description": "Optional additional arguments for the action." - }, - "requestBody": { - "type": "object", - "properties": { - "actionInput": { - "type": "string", - "description": "Input for the action." - } + { + "function": { + "name": "executeExternalAction", + "description": "Runs a specified external action (given as parameter actionName), optionally with additional arguments and input. Run \"listActions\" to get a list of all available actions. Only on explicit user request.", + "parameters": { + "type": "object", + "properties": { + "actionName": { + "type": "string", + "description": "The name of the action to execute." + }, + "arguments": { + "type": "string", + "description": "Optional additional arguments for the action." + }, + "requestBody": { + "type": "object", + "properties": { + "actionInput": { + "type": "string", + "description": "Input for the action." + } + } + } + }, + "required": [ + "actionName", + "requestBody" + ] } - } }, - "required": [ - "actionName", - "requestBody" - ] - } + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" }, - "commandline": [ - "curl", - "-X", - "POST", - "-d", - "@-", - "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" - }, - { - "function": { - "name": "fetchUrlTextContent", - "description": "Fetch text content from a given URL.", - "parameters": { - "type": "object", - "properties": { - "url": { - "type": "string", - "description": "The URL to fetch content from." - }, - "raw": { - "type": "boolean", - "description": "return raw html or pdf content without converting to markdown" - } + { + "function": { + "name": "fetchUrlTextContent", + "description": "Fetch text content from a given URL.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to fetch content from." + }, + "raw": { + "type": "boolean", + "description": "return raw html or pdf content without converting to markdown" + } + }, + "required": ["url"] + } }, - "required": [ - "url" - ] - } + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" }, - "commandline": [ - "curl", - "-X", - "POST", - "-d", - "@-", - "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" - }, - { - "function": { - "name": "grepAction", - "description": "Search for lines in text files matching the given regex.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to the directory to search in or the file to search. default is the root directory = '.'." - }, - "fileRegex": { - "type": "string", - "description": "optional regex to filter file names" - }, - "grepRegex": { - "type": "string", - "description": "regex to filter lines in the files" - }, - "contextLines": { - "type": "integer", - "description": "number of context lines to include with each match (not yet used)" - } + { + "function": { + "name": "grepAction", + "description": "Search for lines in text files matching the given regex.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to the directory to search in or the file to search. default is the root directory = '.'." + }, + "fileRegex": { + "type": "string", + "description": "optional regex to filter file names" + }, + "grepRegex": { + "type": "string", + "description": "regex to filter lines in the files" + }, + "contextLines": { + "type": "integer", + "description": "number of context lines to include with each match (not yet used)" + } + }, + "required": ["grepRegex"] + } }, - "required": [ - "grepRegex" - ] - } + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" }, - "commandline": [ - "curl", - "-X", - "POST", - "-d", - "@-", - "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" - }, - { - "function": { - "name": "listFiles", - "description": "Recursively lists files in a directory. Optionally filters by filename and content.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to directory to list. default is the root directory = '.'." - }, - "recursive": { - "type": "boolean", - "description": "if true (default) lists files recursively, else only in that directory. In that case we will also list directories." - }, - "filePathRegex": { - "type": "string", - "description": "regex to filter file paths - use for search by file name" - }, - "grepRegex": { - "type": "string", - "description": "an optional regex that lists only files that contain a line matching this pattern" - }, - "listDirectories": { - "type": "boolean", - "description": "if true, lists directories instead of files" - } + { + "function": { + "name": "listFiles", + "description": "Recursively lists files in a directory. Optionally filters by filename and content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to directory to list. default is the root directory = '.'." + }, + "recursive": { + "type": "boolean", + "description": "if true (default) lists files recursively, else only in that directory. In that case we will also list directories." + }, + "filePathRegex": { + "type": "string", + "description": "regex to filter file paths - use for search by file name" + }, + "grepRegex": { + "type": "string", + "description": "an optional regex that lists only files that contain a line matching this pattern" + }, + "listDirectories": { + "type": "boolean", + "description": "if true, lists directories instead of files" + } + }, + "required": [] + } }, - "required": [] - } + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" }, - "commandline": [ - "curl", - "-X", - "POST", - "-d", - "@-", - "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" - }, - { - "function": { - "name": "readFile", - "description": "Read a files content.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to file" - }, - "maxLines": { - "type": "integer", - "description": "maximum number of lines to read from the file" - }, - "startLine": { - "type": "integer", - "description": "line number to start reading from; 1 is the first line" - } + { + "function": { + "name": "readFile", + "description": "Read a files content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "maxLines": { + "type": "integer", + "description": "maximum number of lines to read from the file" + }, + "startLine": { + "type": "integer", + "description": "line number to start reading from; 1 is the first line" + } + }, + "required": ["path"] + } }, - "required": [ - "path" - ] - } + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" }, - "commandline": [ - "curl", - "-X", - "POST", - "-d", - "@-", - "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" - }, - { - "function": { - "name": "replaceInFile", - "description": "Replaces the single occurrence of one or more literal strings in a file. The whole file content is matched, not line by line.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to file" - }, - "requestBody": { - "type": "object", - "properties": { - "replacements": { - "type": "array", - "items": { - "type": "object", - "properties": { - "search": { - "type": "string", - "description": "The literal string to be replaced - can contain many lines, but please take care to find a small number of lines to replace. Everything that is replaced must be here. Prefer to match the whole line / several whole lines." + { + "function": { + "name": "replaceInFile", + "description": "Replaces the single occurrence of one or more literal strings in a file. The whole file content is matched, not line by line.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" }, - "replace": { - "type": "string", - "description": "Literal replacement, can contain several lines. Please observe the correct indentation." + "requestBody": { + "type": "object", + "properties": { + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "The literal string to be replaced - can contain many lines, but please take care to find a small number of lines to replace. Everything that is replaced must be here. Prefer to match the whole line / several whole lines." + }, + "replace": { + "type": "string", + "description": "Literal replacement, can contain several lines. Please observe the correct indentation." + } + } + } + } + } } - } - } - } + }, + "required": [ + "path", + "requestBody" + ] } - } }, - "required": [ - "path", - "requestBody" - ] - } + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" }, - "commandline": [ - "curl", - "-X", - "POST", - "-d", - "@-", - "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" - }, - { - "function": { - "name": "writeFile", - "description": "Overwrite a small file with the complete content given in one step. You cannot append to a file or write parts or write parts - use replaceInFile for inserting parts.", - "parameters": { - "type": "object", - "properties": { - "path": { - "type": "string", - "description": "relative path to file" - }, - "requestBody": { - "type": "object", - "properties": { - "content": { - "type": "string", - "description": "Content to write to the file." - } + { + "function": { + "name": "writeFile", + "description": "Overwrite a small file with the complete content given in one step. You cannot append to a file or write parts or write parts - use replaceInFile for inserting parts.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "requestBody": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Content to write to the file." + } + } + } + }, + "required": [ + "path", + "requestBody" + ] } - } }, - "required": [ - "path", - "requestBody" - ] - } - }, - "commandline": [ - "curl", - "-X", - "POST", - "-d", - "@-", - "http://localhost:3002/executetool" - ], - "stdin": "$toolcall" - } -] + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + } +] \ No newline at end of file diff --git a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version index 437aa162..176e34a3 100644 --- a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version +++ b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version @@ -1 +1 @@ -AIGenVersion(7ec718e6, generate_chatgpt_script_toolsdefinition.prompt-7e1bc539, codeveloperengine-toolsdefinition.json-2762cfb1) \ No newline at end of file +AIGenVersion(211f7fb7, generate_chatgpt_script_toolsdefinition.prompt-1a9be05b, codeveloperengine-toolsdefinition.json-2762cfb1) \ No newline at end of file