diff --git a/README.md b/README.md index 6848ba57..db0fa3ef 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ and have it execute (e.g. build and test) actions locally to support you in your development processes? Then this might be for you. The CoDeveloperGPTengine provide the actions for a OpenAI [GPT](https://openai.com/blog/introducing-gpts) for read or even write access to the files in the local directory it is started in. -It can also work as a [ChatGPT plugin](https://openai.com/blog/chatgpt-plugins). +It can also work as a [ChatGPT plugin](https://openai.com/blog/chatgpt-plugins) (OK, that's rather obsolete now) and +as a chat on the command line with the chatgpt script from my +[ChatGPT Toolsuite](https://github.com/stoerr/chatGPTtools). In contrast to other approaches like [AutoGPT](https://github.com/Significant-Gravitas/AutoGPT) this is not meant to autonomously execute extensive changes (which would likely require a lot of prompt engineering), but to enable the diff --git a/bin/pmcodevgpt b/bin/pmcodevgpt new file mode 100755 index 00000000..2506b866 --- /dev/null +++ b/bin/pmcodevgpt @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# "Poor mans" co developer engine: run it with the chatgpt script from https://github.com/stoerr/chatGPTtools + +# start the co-developer-gpt-engine.jar in the directory this script is placed, following links +progfile=$(realpath $0) +progdir=$(dirname "$progfile") + +JAVA=java +# if jenv is in the path, we use the version that is set for the directory this script is in, +# since the current dir could use some ridiculously low version. +if which jenv >/dev/null 2>&1; then + JAVA=$(cd $progdir; jenv which java) +fi + +$JAVA -jar "$progdir/co-developer-gpt-engine.jar" -w -q & +pid=$! +trap "kill $pid" EXIT + +sleep 2 + +ARGS="" +# if $* doesn't contain -cr or -ca we add -cr to ARGS +if [[ ! "$*" =~ -cr ]] && [[ ! "$*" =~ -ca ]]; then + ARGS="-cr" +fi + +chatgpt -tf <($JAVA -jar "$progdir/co-developer-gpt-engine.jar" --aitoolsdef) $ARGS "$@" diff --git a/project-bin/generate_chatgpt_script_toolsdefinition.prompt b/project-bin/generate_chatgpt_script_toolsdefinition.prompt new file mode 100644 index 00000000..df938589 --- /dev/null +++ b/project-bin/generate_chatgpt_script_toolsdefinition.prompt @@ -0,0 +1,14 @@ +Transform the retrieved OpenAI tools definition file into the following format: + +[ + { + "function": { + // here comes the function definition of the tool + }, + "commandline: [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + ... // more tools +] diff --git a/project-bin/generate_openai_toolsdefinition.prompt b/project-bin/generate_openai_toolsdefinition.prompt new file mode 100644 index 00000000..cd970e51 --- /dev/null +++ b/project-bin/generate_openai_toolsdefinition.prompt @@ -0,0 +1,4 @@ +Print an OpenAI tools definition file that describes the requests specified in the retrieved OpenAPI YAML. +Set "strict" to false. Include the descriptions unmodified. +Take care that the "required" attribute contains exactly the required attributes. +If the request is a POST with requestBody, then the body content should be contained in a parameter "requestBody". diff --git a/project-bin/generate_openai_toolsdefinition.sh b/project-bin/generate_openai_toolsdefinition.sh new file mode 100755 index 00000000..1d6b798b --- /dev/null +++ b/project-bin/generate_openai_toolsdefinition.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# generates a json schema for using the engine within other tools +scriptdir=$(dirname "$(realpath $0)") +cd "$(dirname $0)/../" +aigenpipeline -wvf -o src/main/resources/static/codeveloperengine-toolsdefinition.json \ + -p $scriptdir/generate_openai_toolsdefinition.prompt src/test/resources/test-expected/codeveloperengine.yaml + +aigenpipeline -wvf -o src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json \ + -p $scriptdir/generate_chatgpt_script_toolsdefinition.prompt src/main/resources/static/codeveloperengine-toolsdefinition.json diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java index d75be939..f1879971 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/AbstractPluginAction.java @@ -40,7 +40,7 @@ public abstract class AbstractPluginAction extends HttpServlet { */ public static final Pattern BINARY_FILES_PATTERN = Pattern.compile("(?i).*\\.(gif|png|mov|jpg|jpeg|mp4|mp3|pdf|zip|gz|tgz|tar|jar|class|war|ear|exe|dll|so|o|a|lib|bin|dat|dmg|iso)"); - private final transient Gson gson = new Gson(); + protected final transient Gson gson = new Gson(); /** * Logs an error and sends it to ChatGPT, always throws {@link ExecutionAbortedException}. @@ -62,10 +62,9 @@ protected static ExecutionAbortedException sendError(HttpServletResponse respons protected static Stream findMatchingFiles( boolean suppressMessage, HttpServletResponse response, Path path, Pattern filePathPattern, - Pattern grepPattern, boolean recursive) { + Pattern grepPattern, boolean recursive, boolean listDirectories) { boolean haveFilePathPattern = filePathPattern != null && !filePathPattern.pattern().isEmpty(); List result = new ArrayList<>(); - boolean returnDirectories = !recursive; try { Files.walkFileTree(path, new SimpleFileVisitor() { @Override @@ -74,7 +73,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th (!recursive && !dir.equals(path))) { return FileVisitResult.SKIP_SUBTREE; } - if (returnDirectories) { + if (listDirectories || !recursive) { result.add(dir); } return super.preVisitDirectory(dir, attrs); @@ -83,7 +82,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { FileVisitResult res = super.visitFile(file, attrs); - if (!isIgnored(file)) { + if (!isIgnored(file) && !listDirectories) { result.add(file); } return res; @@ -95,11 +94,15 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO List matchingFiles = result.stream() .filter(p -> !haveFilePathPattern || filePathPattern.matcher(p.toString()).find()) - .filter(f -> returnDirectories || Files.isRegularFile(f)) + .filter(f -> listDirectories && Files.isDirectory(f) || Files.isRegularFile(f)) .collect(toList()); if (matchingFiles.isEmpty() && !suppressMessage) { - String similarFilesMessage = getSimilarFilesMessage(response, path, filePathPattern != null ? filePathPattern.toString() : ""); - throw sendError(response, 404, "No files found matching filePathRegex: " + filePathPattern + "\n\n" + similarFilesMessage); + if (filePathPattern != null) { + String similarFilesMessage = getSimilarFilesMessage(response, path, filePathPattern != null ? filePathPattern.toString() : "", listDirectories); + throw sendError(response, 404, "No files found matching filePathRegex: " + filePathPattern + "\n\n" + similarFilesMessage); + } else { + throw sendError(response, 404, "No files found in " + path); + } } Collections.sort(matchingFiles); // make it deterministic. @@ -184,7 +187,7 @@ protected Path getPath(HttpServletRequest request, HttpServletResponse response, if (mustExist && !Files.exists(resolved)) { String message = "Path " + path + " does not exist! Try to list files with /listFiles to find the right path."; String filename = resolved.getFileName().toString(); - String similarFilesMessage = getSimilarFilesMessage(response, CoDeveloperEngine.currentDir, filename); + String similarFilesMessage = getSimilarFilesMessage(response, CoDeveloperEngine.currentDir, filename, false); if (!similarFilesMessage.isEmpty()) { message += "\n\n" + similarFilesMessage; } @@ -193,17 +196,17 @@ protected Path getPath(HttpServletRequest request, HttpServletResponse response, return resolved; } - protected static String getSimilarFilesMessage(HttpServletResponse response, Path path, String filename) { + protected static String getSimilarFilesMessage(HttpServletResponse response, Path path, String filename, boolean listDirectories) { String similarFilesMessage = ""; - List matchingFiles = findMatchingFiles(true, response, path, null, null, true) + List matchingFiles = findMatchingFiles(true, response, path, null, null, true, listDirectories) .collect(toList()); List files = matchingFiles.stream() - .map(p -> CoDeveloperEngine.currentDir.relativize(p).toString()) + .map(CoDeveloperEngine::canonicalName) .filter(p -> p.contains("/" + filename)) .limit(5) .collect(toList()); matchingFiles.stream() - .map(p -> CoDeveloperEngine.currentDir.relativize(p).toString()) + .map(CoDeveloperEngine::canonicalName) .map(p -> Pair.of(p, StringUtils.getFuzzyDistance(p, filename, Locale.getDefault()))) .map(p -> Pair.of(p.getLeft(), -p.getRight())) .sorted(Comparator.comparingDouble(Pair::getRight)) @@ -243,10 +246,6 @@ protected String getBodyParameter(HttpServletResponse response, String json, Str return parameterValue; } - protected String mappedFilename(Path path) { - return CoDeveloperEngine.currentDir.relativize(path).toString(); - } - protected String abbreviate(String s, int max) { if (s == null || s.length() <= max) { return s; diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java index b271f259..18b8b39b 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/CoDeveloperEngine.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -23,9 +24,7 @@ import org.apache.commons.cli.ParseException; import org.apache.commons.lang3.StringUtils; import org.apache.pdfbox.io.IOUtils; -import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.RequestLog; -import org.eclipse.jetty.server.Response; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ResourceHandler; import org.eclipse.jetty.servlet.FilterHolder; @@ -58,8 +57,6 @@ public class CoDeveloperEngine { */ public static final Pattern OVERRIDE_IGNORE_PATTERN = Pattern.compile(".*/.github/.*|.*/.content.xml|(.*/)?\\.chatgpt.*.md|.*\\.htaccess"); - // private static final Gson GSON = new Gson(); - static Path currentDir = Paths.get(".").normalize().toAbsolutePath(); private static int port; @@ -113,24 +110,21 @@ public class CoDeveloperEngine { chain.doFilter(rawRequest, rawResponse); }; - private static final RequestLog requestlog = new RequestLog() { - @Override - public void log(Request request, Response response) { - TbUtils.logInfo("Remote address: " + request.getRemoteAddr()); - TbUtils.logInfo("Remote host: " + request.getRemoteHost()); - TbUtils.logInfo("Remote port: " + request.getRemotePort()); - TbUtils.logInfo("Requestlog: " + request.getMethod() + " " + request.getRequestURL() + (request.getQueryString() != null && !request.getQueryString().isEmpty() ? "?" + request.getQueryString() : "") + " " + response.getStatus()); - // list all request headers - for (Enumeration e = request.getHeaderNames(); e.hasMoreElements(); ) { - String header = e.nextElement(); - TbUtils.logInfo("Request header: " + header + ": " + request.getHeader(header)); - } - // list all response headers - for (String header : response.getHeaderNames()) { - TbUtils.logInfo("Response header: " + header + ": " + response.getHeader(header)); - } - TbUtils.logInfo(""); + private static final RequestLog requestlog = (request, response) -> { + TbUtils.logInfo("Remote address: " + request.getRemoteAddr()); + TbUtils.logInfo("Remote host: " + request.getRemoteHost()); + TbUtils.logInfo("Remote port: " + request.getRemotePort()); + TbUtils.logInfo("Requestlog: " + request.getMethod() + " " + request.getRequestURL() + (request.getQueryString() != null && !request.getQueryString().isEmpty() ? "?" + request.getQueryString() : "") + " " + response.getStatus()); + // list all request headers + for (Enumeration e = request.getHeaderNames(); e.hasMoreElements(); ) { + String header = e.nextElement(); + TbUtils.logInfo("Request header: " + header + ": " + request.getHeader(header)); + } + // list all response headers + for (String header : response.getHeaderNames()) { + TbUtils.logInfo("Response header: " + header + ": " + response.getHeader(header)); } + TbUtils.logInfo(""); }; private static void addHandler(AbstractPluginAction handler) { @@ -148,6 +142,7 @@ protected static void initServlets() { addHandler(new ListFilesAction()); addHandler(new ReadFileAction()); addHandler(new GrepAction()); + addHandler(new ExecuteOpenAIToolCallAction(HANDLERS)); if (writingEnabled) { addHandler(new WriteFileAction()); ExecuteExternalAction executeExternalAction = new ExecuteExternalAction(); @@ -228,9 +223,19 @@ private static String getMainUrl(HttpServletRequest request) { } public static void main(String[] args) throws Exception { - TbUtils.logVersion(); + if (args.length == 1 && args[0].equals("--aitoolsdef")) { + // hidden option to output a description of the operations for use with my chatgpt script to stdout + try (InputStream in = CoDeveloperEngine.class.getResourceAsStream("/static/codeveloperengine-chatgptscript-toolsdefinition.json")) { + IOUtils.copy(in, System.out); + } + return; + } - parseOptions(args); + try { + parseOptions(args); + } finally { + TbUtils.logVersion(); + } server = new Server(new InetSocketAddress("127.0.0.1", port)); context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); context.setContextPath("/"); @@ -262,6 +267,7 @@ private static void parseOptions(String[] args) { options.addOption("h", "help", false, "Display this help message"); options.addOption("g", "globalconfigdir", true, "Directory for global configuration (default: ~/.cgptcodeveloperglobal/"); options.addOption("l", "local", false, "Only use local configuration via options - ignore any global configuration"); + options.addOption("q", "quiet", false, "Suppress info level output"); CommandLineParser parser = new DefaultParser(); @@ -293,6 +299,10 @@ private static void parseOptions(String[] args) { userGlobalConfigDir = null; ignoreGlobalConfig = true; } + + if (cmd.hasOption("q")) { + TbUtils.setQuiet(true); + } } catch (ParseException e) { TbUtils.logError("Error parsing command line options: " + e); System.exit(1); @@ -307,4 +317,13 @@ public static void execute(Runnable runnable) { server.getThreadPool().execute(runnable); } + public static final String canonicalName(Path path) { + if (!path.toAbsolutePath().startsWith(currentDir.toAbsolutePath())) { + throw new IllegalArgumentException("Bug: trying to return file not in current dir - " + path); + } + String file = currentDir.relativize(path).toString(); + file = file.isEmpty() ? "." : file; + return Files.isDirectory(path) ? file + "/" : file; + } + } diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java new file mode 100644 index 00000000..217e9f9f --- /dev/null +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ExecuteOpenAIToolCallAction.java @@ -0,0 +1,96 @@ +package net.stoerr.chatgpt.codevengine; + +import static net.stoerr.chatgpt.codevengine.TbUtils.logInfo; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; +import jakarta.servlet.http.HttpServletResponse; + +/** + * Executes an OpenAI tool call coming in as JSON - for usage outside ChatGPT. + */ +public class ExecuteOpenAIToolCallAction extends AbstractPluginAction { + + private final Map handlers; + + public ExecuteOpenAIToolCallAction(Map handlers) { + this.handlers = handlers; + } + + @Override + public String getUrl() { + return "/executetool"; + } + + /** + * This is not registered in the yaml description since it's not a normal action, but rather + * distributes to actions. + */ + @Override + public String openApiDescription() { + return ""; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + BufferedReader reader = req.getReader(); + String json = reader.lines().collect(Collectors.joining(System.lineSeparator())); + logInfo("Received tool call:\n" + json); + String name = getBodyParameter(resp, json, "name", true); + String arguments = getBodyParameter(resp, json, "arguments", true); + Map parsedArguments = gson.fromJson(arguments, Map.class); + logInfo("Executing tool call: " + name + " " + parsedArguments); + Object requestBody = parsedArguments.get("requestBody"); + String body = requestBody != null ? gson.toJson(requestBody) : null; + if (StringUtils.isNotBlank(body)) logInfo("Body: " + body); + AbstractPluginAction handler = handlers.get("/" + name); + if (null == handler) { + sendError(resp, HttpServletResponse.SC_BAD_REQUEST, "No handler for tool call: " + name); + return; + } + // call handler with a request that has parsedArguments as parameters and body as request body (JSON request) + HttpServletRequest requestWrapper = new HttpServletRequestWrapper(req) { + @Override + public String getParameter(String name) { + Object value = parsedArguments.get(name); + if (value == null) return null; + if (value instanceof String) return (String) value; + if (value instanceof Double) { + // check whether it's an integer + double d = (Double) value; + if (Math.abs(d - Math.round(d)) < 0.001) return "" + Math.round(d); + } + return String.valueOf(value); + } + + @Override + public BufferedReader getReader() throws IOException { + return body != null ? new BufferedReader(new StringReader(body)) : null; + } + + @Override + public String getMethod() { + return requestBody != null ? "POST" : "GET"; + } + }; + try { + handler.service(requestWrapper, resp); + } catch (ExecutionAbortedException e) { + // is already sufficiently handled. Just ignore. + } catch (ServletException | IOException | RuntimeException e) { + TbUtils.logError("Error executing tool call: " + name + "\n" + arguments); + TbUtils.logStacktrace(e); + throw e; + } + } + +} diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/GrepAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/GrepAction.java index e62a4249..41084f73 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/GrepAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/GrepAction.java @@ -98,7 +98,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO throw sendError(resp, 404, "Path is not readable: " + startPath); } - List matchingFiles = findMatchingFiles(false, resp, startPath, filePattern, grepPattern, true) + List matchingFiles = findMatchingFiles(false, resp, startPath, filePattern, grepPattern, true, false) .collect(Collectors.toList()); if (!matchingFiles.isEmpty()) { StringBuilder buf = new StringBuilder(); @@ -136,7 +136,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO resp.setContentType("text/plain;charset=UTF-8"); resp.getWriter().write(buf.toString()); } else { - long filePathFileCount = findMatchingFiles(true, resp, startPath, filePattern, null, true).count(); + long filePathFileCount = findMatchingFiles(true, resp, startPath, filePattern, null, true, false).count(); if (filePathFileCount > 0) throw sendError(resp, 404, "Found " + filePathFileCount + " files whose name is matching the filePathRegex but none of them contain a line matching the grepRegex."); else if (Files.isDirectory(startPath)) { @@ -151,9 +151,9 @@ else if (Files.isDirectory(startPath)) { private void appendBlock(List lines, StringBuilder buf, Path path, int start, int end) { if (start == end - 1) { - buf.append("======================== ").append(mappedFilename(path)).append(" line ").append(start + 1).append('\n'); + buf.append("======================== ").append(CoDeveloperEngine.canonicalName(path)).append(" line ").append(start + 1).append('\n'); } else { - buf.append("======================== ").append(mappedFilename(path)).append(" lines ").append(start + 1).append(" to ").append(end).append('\n'); + buf.append("======================== ").append(CoDeveloperEngine.canonicalName(path)).append(" lines ").append(start + 1).append(" to ").append(end).append('\n'); } for (int j = start; j < end; j++) { buf.append(lines.get(j)).append('\n'); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ListFilesAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ListFilesAction.java index 7eca9dea..9ff25da7 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ListFilesAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ListFilesAction.java @@ -4,7 +4,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Comparator; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -76,7 +75,8 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO Path path = getPath(req, resp, true, true); String filePathRegex = getQueryParam(req, "filePathRegex"); String grepRegex = getQueryParam(req, "grepRegex"); - String listDirectories = getQueryParam(req, "listDirectories"); + String listDirectoriesRaw = getQueryParam(req, "listDirectories"); + boolean listDirectories = Boolean.parseBoolean(listDirectoriesRaw); String recursiveRaw = getQueryParam(req, "recursive"); boolean recursive = recursiveRaw == null || Boolean.parseBoolean(recursiveRaw); RepeatedRequestChecker.CHECKER.checkRequestRepetition(resp, this, path, filePathRegex, grepRegex, recursiveRaw, listDirectories); @@ -95,28 +95,24 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO if (Files.isDirectory(path)) { resp.setContentType("text/plain;charset=UTF-8"); - List paths = findMatchingFiles(false, resp, path, filePathPattern, grepPattern, recursive) + List paths = findMatchingFiles(false, resp, path, filePathPattern, grepPattern, recursive, listDirectories) .collect(Collectors.toList()); List files = paths.stream() - .map(this::mappedFilename) + .map(path1 -> CoDeveloperEngine.canonicalName(path1)) .filter(StringUtils::isNotBlank) .collect(Collectors.toList()); if (files.isEmpty()) { - long filePathFileCount = findMatchingFiles(false, resp, path, filePathPattern, null, recursive).count(); - if (filePathFileCount > 0) - throw sendError(resp, 404, "Found " + filePathFileCount + " files whose name is matching the filePathRegex but none of them contain a line matching the grepRegex."); - else if (Files.newDirectoryStream(path).iterator().hasNext()) { - String similarFilesMessage = getSimilarFilesMessage(resp, path, filePathPattern != null ? filePathPattern.toString() : ""); + if (grepPattern != null) { + long filePathFileCount = findMatchingFiles(false, resp, path, filePathPattern, null, recursive, listDirectories).count(); + if (filePathFileCount > 0) + throw sendError(resp, 404, "Found " + filePathFileCount + " files whose name is matching the filePathRegex but none of them contain a line matching the grepRegex."); + } + if (Files.newDirectoryStream(path).iterator().hasNext()) { + String similarFilesMessage = getSimilarFilesMessage(resp, path, filePathPattern != null ? filePathPattern.toString() : "", listDirectories); throw sendError(resp, 404, "No files found matching filePathRegex: " + filePathRegex + "\n\n" + similarFilesMessage); } else { throw sendError(resp, 404, "No files found in directory: " + path); } - } else if ("TRUE".equalsIgnoreCase(listDirectories)) { - files = paths.stream().map(Path::getParent).distinct() - .sorted(Comparator.comparing(Path::toString)) - .map(f -> mappedFilename(f)) - .map(f -> StringUtils.defaultIfEmpty(f, ".") + "/") - .collect(Collectors.toList()); } else if (files.size() > 100) { long directoryCount = paths.stream().map(Path::getParent).distinct().count(); throw sendError(resp, 404, "Found " + files.size() + " files in " + directoryCount + " directories - please use a more specific path or filePathRegex, or use listDirectories instead to get an overview and then list specific directories you're interested in."); diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ReadFileAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ReadFileAction.java index eb194348..be938f27 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ReadFileAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ReadFileAction.java @@ -111,7 +111,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IO } if (maxLines <= fulllinecount || startLine > 1 || dropped != 0) { content = "CAUTION: Lines " + startLine + " to " + (startLine + lines.size() - 1) + " of " + fulllinecount + - " lines of file " + CoDeveloperEngine.currentDir.relativize(path) + " start now. " + + " lines of file " + CoDeveloperEngine.canonicalName(path) + " start now. " + "To get more of the file content repeat read request with startLine=" + (startLine + lines.size()) + " , or use the grepAction with enough contextLines if you are searching for something specific.\n\n" + content + "\n\n" + diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceAction.java index ddcacace..b9f62a39 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceAction.java @@ -152,7 +152,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I resp.getWriter().write(totalReplacementCount + " replacement; modified line(s) " + String.join(", ", modifiedLineDescr)); } catch (NoSuchFileException e) { - throw sendError(resp, 404, "File not found: " + CoDeveloperEngine.currentDir.relativize(path)); + throw sendError(resp, 404, "File not found: " + CoDeveloperEngine.canonicalName(path)); } catch (IOException e) { throw sendError(resp, 500, "Error reading or writing file : " + e); } catch (IllegalArgumentException e) { diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceRegexAction.java b/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceRegexAction.java index 24f5dba8..bc28f8a1 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceRegexAction.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/ReplaceRegexAction.java @@ -158,7 +158,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I resp.getWriter().write("Replaced " + replacementCount + " occurrences of pattern; modified lines " + String.join(", ", modifiedLineDescr)); } catch (NoSuchFileException e) { - throw sendError(resp, 404, "File not found: " + CoDeveloperEngine.currentDir.relativize(path)); + throw sendError(resp, 404, "File not found: " + CoDeveloperEngine.canonicalName(path)); } catch (IOException e) { throw sendError(resp, 500, "Error reading or writing file : " + e); } catch (PatternSyntaxException e) { diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java b/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java index 758c3873..5eef3076 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/TbUtils.java @@ -14,8 +14,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.LoggerFactory; + import com.google.common.collect.Range; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.LoggerContext; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -28,6 +33,11 @@ public class TbUtils { public static final PrintStream LOG = System.out; static boolean isLoggingEnabled = true; + static boolean quiet; + + private TbUtils() { + throw new IllegalStateException("Utility class"); + } /** * If there is a file named .cgptcodeveloper/.requestlog.txt, we append the request data to it. @@ -65,6 +75,7 @@ protected static void logBody(String parameterName, String parameterValue) { static void logStacktrace(Exception e) { e.printStackTrace(ERRLOG); + ERRLOG.println(); } static void logError(String msg) { @@ -72,7 +83,7 @@ static void logError(String msg) { } static void logInfo(String msg) { - if (isLoggingEnabled) { + if (isLoggingEnabled && !quiet) { LOG.println(msg); } } @@ -184,4 +195,16 @@ private static String rangeDescription(Range lastRange) { } return rangeDescr; } + + public static void setQuiet(boolean quiet) { + TbUtils.quiet = quiet; + // change logback root logger and org.eclipse.jetty logger to WARN level + if (quiet) { + LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); + Logger rootLogger = loggerContext.getLogger(Logger.ROOT_LOGGER_NAME); + rootLogger.setLevel(Level.WARN); + Logger jettyLogger = loggerContext.getLogger("org.eclipse.jetty"); + jettyLogger.setLevel(Level.WARN); + } + } } diff --git a/src/main/java/net/stoerr/chatgpt/codevengine/UserGlobalConfig.java b/src/main/java/net/stoerr/chatgpt/codevengine/UserGlobalConfig.java index fb553210..8167a937 100644 --- a/src/main/java/net/stoerr/chatgpt/codevengine/UserGlobalConfig.java +++ b/src/main/java/net/stoerr/chatgpt/codevengine/UserGlobalConfig.java @@ -119,7 +119,7 @@ public boolean readAndCheckConfiguration(@Nullable String globalConfigDir) throw externport = Integer.parseInt(config.getProperty("externport", "443")); if (httpsPort == null || httpsPort <= 0) { - TbUtils.logError("httpsport property in " + configFile + " is not set - our own https is disabled, relying on a tunnel."); + TbUtils.logInfo("httpsport property in " + configFile + " is not set - our own https is disabled, relying on a tunnel."); // that is OK if we use a tunnel instead of doing https ourselves return true; } else { diff --git a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json new file mode 100644 index 00000000..9ad1c334 --- /dev/null +++ b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json @@ -0,0 +1,234 @@ +[ + { + "function": { + "name": "executeExternalAction", + "description": "Runs a specified external action (given as parameter actionName), optionally with additional arguments and input. Run \"listActions\" to get a list of all available actions. Only on explicit user request.", + "parameters": { + "type": "object", + "properties": { + "actionName": { + "type": "string", + "description": "The name of the action to execute." + }, + "arguments": { + "type": "string", + "description": "Optional additional arguments for the action." + }, + "requestBody": { + "type": "object", + "properties": { + "actionInput": { + "type": "string", + "description": "Input for the action." + } + } + } + }, + "required": [ + "actionName", + "requestBody" + ] + } + }, + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "fetchUrlTextContent", + "description": "Fetch text content from a given URL.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to fetch content from." + }, + "raw": { + "type": "boolean", + "description": "return raw html or pdf content without converting to markdown" + } + }, + "required": ["url"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "grepAction", + "description": "Search for lines in text files matching the given regex.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to the directory to search in or the file to search. default is the root directory = '.'." + }, + "fileRegex": { + "type": "string", + "description": "optional regex to filter file names" + }, + "grepRegex": { + "type": "string", + "description": "regex to filter lines in the files" + }, + "contextLines": { + "type": "integer", + "description": "number of context lines to include with each match (not yet used)" + } + }, + "required": ["grepRegex"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "listFiles", + "description": "Recursively lists files in a directory. Optionally filters by filename and content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to directory to list. default is the root directory = '.'." + }, + "recursive": { + "type": "boolean", + "description": "if true (default) lists files recursively, else only in that directory. In that case we will also list directories." + }, + "filePathRegex": { + "type": "string", + "description": "regex to filter file paths - use for search by file name" + }, + "grepRegex": { + "type": "string", + "description": "an optional regex that lists only files that contain a line matching this pattern" + }, + "listDirectories": { + "type": "boolean", + "description": "if true, lists directories instead of files" + } + }, + "required": [] + } + }, + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "readFile", + "description": "Read a files content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "maxLines": { + "type": "integer", + "description": "maximum number of lines to read from the file" + }, + "startLine": { + "type": "integer", + "description": "line number to start reading from; 1 is the first line" + } + }, + "required": ["path"] + } + }, + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "replaceInFile", + "description": "Replaces the single occurrence of one or more literal strings in a file. The whole file content is matched, not line by line.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "requestBody": { + "type": "object", + "properties": { + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "The literal string to be replaced - can contain many lines, but please take care to find a small number of lines to replace. Everything that is replaced must be here. Prefer to match the whole line / several whole lines." + }, + "replace": { + "type": "string", + "description": "Literal replacement, can contain several lines. Please observe the correct indentation." + } + } + } + } + } + } + }, + "required": [ + "path", + "requestBody" + ] + } + }, + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + }, + { + "function": { + "name": "writeFile", + "description": "Overwrite a small file with the complete content given in one step. You cannot append to a file or write parts or write parts - use replaceInFile for inserting parts.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "requestBody": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Content to write to the file." + } + } + } + }, + "required": [ + "path", + "requestBody" + ] + } + }, + "commandline": [ + "curl", "-X", "POST", "-d", "@-", "http://localhost:3002/executetool" + ], + "stdin": "$toolcall" + } +] \ No newline at end of file diff --git a/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version new file mode 100644 index 00000000..176e34a3 --- /dev/null +++ b/src/main/resources/static/codeveloperengine-chatgptscript-toolsdefinition.json.version @@ -0,0 +1 @@ +AIGenVersion(211f7fb7, generate_chatgpt_script_toolsdefinition.prompt-1a9be05b, codeveloperengine-toolsdefinition.json-2762cfb1) \ No newline at end of file diff --git a/src/main/resources/static/codeveloperengine-toolsdefinition.json b/src/main/resources/static/codeveloperengine-toolsdefinition.json new file mode 100644 index 00000000..f6c62396 --- /dev/null +++ b/src/main/resources/static/codeveloperengine-toolsdefinition.json @@ -0,0 +1,195 @@ +{ + "tools": [ + { + "name": "executeExternalAction", + "description": "Runs a specified external action (given as parameter actionName), optionally with additional arguments and input. Run \"listActions\" to get a list of all available actions. Only on explicit user request.", + "parameters": { + "type": "object", + "properties": { + "actionName": { + "type": "string", + "description": "The name of the action to execute." + }, + "arguments": { + "type": "string", + "description": "Optional additional arguments for the action." + }, + "requestBody": { + "type": "object", + "properties": { + "actionInput": { + "type": "string", + "description": "Input for the action." + } + } + } + }, + "required": [ + "actionName", + "requestBody" + ] + } + }, + { + "name": "fetchUrlTextContent", + "description": "Fetch text content from a given URL.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to fetch content from." + }, + "raw": { + "type": "boolean", + "description": "return raw html or pdf content without converting to markdown" + } + }, + "required": ["url"] + } + }, + { + "name": "grepAction", + "description": "Search for lines in text files matching the given regex.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to the directory to search in or the file to search. default is the root directory = '.'." + }, + "fileRegex": { + "type": "string", + "description": "optional regex to filter file names" + }, + "grepRegex": { + "type": "string", + "description": "regex to filter lines in the files" + }, + "contextLines": { + "type": "integer", + "description": "number of context lines to include with each match (not yet used)" + } + }, + "required": ["grepRegex"] + } + }, + { + "name": "listFiles", + "description": "Recursively lists files in a directory. Optionally filters by filename and content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to directory to list. default is the root directory = '.'." + }, + "recursive": { + "type": "boolean", + "description": "if true (default) lists files recursively, else only in that directory. In that case we will also list directories." + }, + "filePathRegex": { + "type": "string", + "description": "regex to filter file paths - use for search by file name" + }, + "grepRegex": { + "type": "string", + "description": "an optional regex that lists only files that contain a line matching this pattern" + }, + "listDirectories": { + "type": "boolean", + "description": "if true, lists directories instead of files" + } + }, + "required": [] + } + }, + { + "name": "readFile", + "description": "Read a files content.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "maxLines": { + "type": "integer", + "description": "maximum number of lines to read from the file" + }, + "startLine": { + "type": "integer", + "description": "line number to start reading from; 1 is the first line" + } + }, + "required": ["path"] + } + }, + { + "name": "replaceInFile", + "description": "Replaces the single occurrence of one or more literal strings in a file. The whole file content is matched, not line by line.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "requestBody": { + "type": "object", + "properties": { + "replacements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "search": { + "type": "string", + "description": "The literal string to be replaced - can contain many lines, but please take care to find a small number of lines to replace. Everything that is replaced must be here. Prefer to match the whole line / several whole lines." + }, + "replace": { + "type": "string", + "description": "Literal replacement, can contain several lines. Please observe the correct indentation." + } + } + } + } + } + } + }, + "required": [ + "path", + "requestBody" + ] + } + }, + { + "name": "writeFile", + "description": "Overwrite a small file with the complete content given in one step. You cannot append to a file or write parts or write parts - use replaceInFile for inserting parts.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "relative path to file" + }, + "requestBody": { + "type": "object", + "properties": { + "content": { + "type": "string", + "description": "Content to write to the file." + } + } + } + }, + "required": [ + "path", + "requestBody" + ] + } + } + ], + "strict": false +} diff --git a/src/main/resources/static/codeveloperengine-toolsdefinition.json.version b/src/main/resources/static/codeveloperengine-toolsdefinition.json.version new file mode 100644 index 00000000..b052c46c --- /dev/null +++ b/src/main/resources/static/codeveloperengine-toolsdefinition.json.version @@ -0,0 +1 @@ +AIGenVersion(2762cfb1, generate_openai_toolsdefinition.prompt-54a86373, codeveloperengine.yaml-69a9e54b) \ No newline at end of file diff --git a/src/main/resources/static/index.html b/src/main/resources/static/index.html index bc927ea0..99a5d597 100644 --- a/src/main/resources/static/index.html +++ b/src/main/resources/static/index.html @@ -4,6 +4,11 @@

Co-Developer GPT Engine

-

See /.well-known/ai-plugin.json for the plugin description and - codeveloperengine.yaml for the available functions.

+

+ See /.well-known/ai-plugin.json for the plugin description and + codeveloperengine.yaml for the available functions. + codeveloperengine-toolsdefinition.json contains + the list of tools as appropriate for the OpenAI chat completion API. +

+ diff --git a/src/site/markdown/pmcodevgpt.md b/src/site/markdown/pmcodevgpt.md new file mode 100644 index 00000000..fd90dcbf --- /dev/null +++ b/src/site/markdown/pmcodevgpt.md @@ -0,0 +1,21 @@ +# Running the codeveloper engine with the chatgpt command line tool + +As an alternative to running the CoDeveloper GPT Engine from ChatGPT you can run it with the chatgpt +script from my [ChatGPT Toolsuite](https://github.com/stoerr/chatGPTtools). This not a full fledged chat interface, +(hence the script name `pmcodevgpt`= "Poor Mans CoDEVeloper GPT"), +but starts up within a second and can be used for quick tasks. You'll need an +[OpenAI API key](https://platform.openai.com/api-keys) to use it, though - +either in an environment variable `OPENAI_API_KEY` or in a file `~/.openai-api-key.txt` . +The `pmcodevgpt` script starts the CoDeveloper GPT Engine in the background and the +[chatgpt](https://github.com/stoerr/chatGPTtools/blob/develop/bin/chatgpt) script with a tools definition +that is generated from the OpenAPI description of the CoDeveloper GPT Engine. + +If you call pmcodevgpt, you can type your prompt and end it with `/end` on a line of its own, or press Ctrl-D to end +each message. After processing and printing the response this starts again - abort the program with Ctrl-C. + +If you like you could also use the audio chat feature of the chatgpt script to talk to the CoDeveloper GPT Engine. +You can call `pmcodevgpt -ca` to start the audio chat - follow the instructions it prints to the console. You can +dictate your prompts, but the output will be written to the console. + +BTW: if you know any open source models / interfaces that support a function calling / tools interface like +OpenAI does, please let me know! I'd like to try that / integrate that, too. diff --git a/src/site/site.xml b/src/site/site.xml index fc68bee5..85e58f88 100644 --- a/src/site/site.xml +++ b/src/site/site.xml @@ -48,6 +48,7 @@ +