From 6db7286139e666dc8d1f035d4b3d58a59dbb53ec Mon Sep 17 00:00:00 2001 From: Edison Fernandez Date: Tue, 9 Jul 2024 17:18:41 -0600 Subject: [PATCH] feat: First version --- .github/workflows/github-actions.yml | 62 +++++ .gitignore | 3 + .release/update_version_number.sh | 19 ++ .releaserc.json | 17 ++ CHANGELOG.md | 0 MANIFEST.in | 1 + README.md | 148 +++++++++++- aiagent/__init__.py | 0 aiagent/controllers/__init__.py | 0 aiagent/controllers/apidispatcher.py | 106 +++++++++ aiagent/controllers/controller.py | 36 +++ aiagent/controllers/prompt.py | 80 +++++++ aiagent/controllers/promptcontroller.py | 88 +++++++ aiagent/controllers/webcontroller.py | 38 +++ aiagent/main.py | 56 +++++ aiagent/server.py | 37 +++ aiagent/templates/index.html | 150 ++++++++++++ api/openapi.html | 49 ++++ api/openapi.yaml | 78 +++++++ docker/Dockerfile | 296 ++++++++++++++++++++++++ docs/Makefile | 20 ++ docs/build/.gitignore | 0 docs/make.bat | 35 +++ docs/source/_static/.gitignore | 0 docs/source/aiagent.controllers.rst | 53 +++++ docs/source/aiagent.rst | 37 +++ docs/source/api.rst | 23 ++ docs/source/conf.py | 65 ++++++ docs/source/index.rst | 24 ++ docs/source/modules.rst | 7 + docs/source/readme.rst | 4 + setup.py | 52 +++++ 32 files changed, 1583 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/github-actions.yml create mode 100644 .gitignore create mode 100755 .release/update_version_number.sh create mode 100644 .releaserc.json create mode 100644 CHANGELOG.md create mode 100644 MANIFEST.in create mode 100644 aiagent/__init__.py create mode 100644 aiagent/controllers/__init__.py create mode 100644 aiagent/controllers/apidispatcher.py create mode 100644 aiagent/controllers/controller.py create mode 100644 aiagent/controllers/prompt.py create mode 100644 aiagent/controllers/promptcontroller.py create mode 100644 aiagent/controllers/webcontroller.py create mode 100644 aiagent/main.py create mode 100644 aiagent/server.py create mode 100644 aiagent/templates/index.html create mode 100644 api/openapi.html create mode 100644 api/openapi.yaml create mode 100644 docker/Dockerfile create mode 100644 docs/Makefile create mode 100644 docs/build/.gitignore create mode 100644 docs/make.bat create mode 100644 docs/source/_static/.gitignore create mode 100644 docs/source/aiagent.controllers.rst create mode 100644 docs/source/aiagent.rst create mode 100644 docs/source/api.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/modules.rst create mode 100644 docs/source/readme.rst create mode 100644 setup.py diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml new file mode 100644 index 0000000..2882094 --- /dev/null +++ b/.github/workflows/github-actions.yml @@ -0,0 +1,62 @@ +name: Release + +on: + push: + branches: + - main + +permissions: + contents: write + pages: write + id-token: write + +jobs: + pages: + runs-on: ubuntu-latest + environment: + name: Development + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: 21 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y python3-pip python3-venv + npm install @semantic-release/github@10.0.0 + npm install @semantic-release/exec@6.0.3 + npm install @semantic-release/changelog@6.0.3 + npm install @semantic-release/git@10.0.1 + + - name: Make release + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + run: npx semantic-release --debug + + - name: Create documentation + run: | + python3 -m venv venv + source venv/bin/activate + pip install . + cd docs && make html && cd .. + mkdir public + mv docs/build/html/* public/ + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./public + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7084818 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*~ +ai_agent.egg-info +__pycache__ diff --git a/.release/update_version_number.sh b/.release/update_version_number.sh new file mode 100755 index 0000000..2086179 --- /dev/null +++ b/.release/update_version_number.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# This script has to be called from the root of the repository + +if [[ $# -eq 0 ]] ; then + echo "Usage: $0 " + exit 0 +fi + +VERSION=${1} + +# service-template a.b.c +sed -i "s/## AI Agent Microservice [[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+/## AI Agent Microservice ${VERSION}/g" README.md +#version="a.b.c" +sed -i "s/version=\"[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\"/version=\"${VERSION}\"/g" setup.py +# release = 'a.b.c' +sed -i "s/release = '[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+'/release = '${VERSION}'/g" docs/source/conf.py +# version: a.b.c +sed -i "s/version: [[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+/version: ${VERSION}/g" api/openapi.yaml diff --git a/.releaserc.json b/.releaserc.json new file mode 100644 index 0000000..6e1cb5f --- /dev/null +++ b/.releaserc.json @@ -0,0 +1,17 @@ +{ + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + ["@semantic-release/exec", { + "prepareCmd": "./.release/update_version_number.sh ${nextRelease.version}" + }], + ["@semantic-release/changelog", { + "changelogFile": "CHANGELOG.md" + }], + ["@semantic-release/git", { + "assets": ["CHANGELOG.md", "README.md", "api/openapi.yaml", "docs/source/conf.py", "setup.py", "docs/source/index.rst"], + "message": "Bump version number to ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" + }], + "@semantic-release/github" + ] + } diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e69de29 diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..ed6a3d3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include aiagent/templates/*.html \ No newline at end of file diff --git a/README.md b/README.md index f79c11f..f98603d 100644 --- a/README.md +++ b/README.md @@ -1 +1,147 @@ -# ai-agent-service \ No newline at end of file +## AI Agent Microservice 1.0.0 + +AI Agent Microservice allows natural communication between the user and other microservices. This service +uses the Hugging Face LLM Trelis/Llama-2-7b-chat-hf-function-calling-v3 to convert text commands into API +calls, process the LLM result, and call the corresponding API request. + +The service requires a prompt description defining the available functions and the behavior of the LLM and a +configuration file that defines the API calls and their mapping with the prompt function. See below for more details. + +### Prompt format + +Prompt should follow the model prompt format, please check model documentation at +https://huggingface.co/Trelis/Llama-2-7b-chat-hf-function-calling-v3 + +### API configuration + +The API configuration is a JSON file with the following structure: + +```json +{ + "function_name": { + "ip": "api_ip", + "port": "api_port", + "path": "api_request_path", + "method": "request_method", + "properties": { + "prop1": "prop1 value", + "prop2": "prop2 value" + }, + "body": { + "arg1": "value", + "arg2": "value" + } + } +} +``` +The JSON should have a function object per function described in the prompt since it will be used to +map the LLM reply function with the microservice API call. So the function_name should match one of the +functions defined in the prompt. + +The arguments port, path, and method are required. Port is the port of the microservice to call, path +is the route of the specific API request and method is the method for the request can be GET, POST, PUT. + +The argument ip is optional, it defines the IP of the microservice to call. If not defined localhost 127.0.0.0 +will be used. + +properties object defines the parameters of the API request. It is optional, add it only if the API request uses +parameters. The value of each property will be obtained from the LLM reply, so the string in the value should +match the argument name defined in the corresponding prompt function. + +body object defines the API request content. It is optional, add it only if the API request needs body description. +The value of each argument in the body will be obtained from the LLM reply, so the string in the value should +match the argument name defined in the corresponding prompt function. + + +Check the following example: + +```json +{ + "search_object": { + "ip": "192.168.86.25", + "port": 30080, + "path": "genai/prompt", + "method": "GET", + "properties": { + "objects": "input", + "thresholds": 0.2 + } + }, + "move_camera": { + "port": 1234, + "path": "position", + "method": "PUT", + "body": { + "pan": "pan_angle", + "tilt": "tilt_angle" + } + } +} +``` + +### Running the service + +The project is configured (via setup.py) to install the service with the name __ai-agent__. So to install it run: + +```bash +pip install . +``` + +Then you will have the service with the following options: + +```bash +usage: ai-agent [-h] [--port PORT] --system_prompt SYSTEM_PROMPT --api_map API_MAP + +options: + -h, --help show this help message and exit + --port PORT Port for server + --system_prompt SYSTEM_PROMPT + String with system prompt or path to a txt file with the prompt + --api_map API_MAP Path to a JSON file with API mapping configuration +usage: ai-agent [-h] [--port PORT] [--system_prompt SYSTEM_PROMPT] [--api_map API_MAP] +``` + +Notice that the system_prompt and api_map are required, so to run it use the following command: + +```bash +ai-agent --prompt PROMPT --api_map API_MAP +``` + +This will start the service in address 127.0.0.0 and port 5010. If you want to use a different port, use the __--port__ options. + + +## AI Agent Docker + + +### Build the container + +We can build the gen-ai microservice container using the Dockerfile in the docker directory. This includes a base tensort +image and the dependencies to run the ai-agent microservice application. + +First, we need to prepare the context directory for this build, you need to create a directory and include this repository +and the rrms-utils project. The Dockerfile will look for both packages in the context directory and copy them to the container. + +```bash +ai-agent-context/ +├── ai-agent +└── rrms-utils +``` + +Then build the container image with the following command: + +```bash +DOCKER_BUILDKIT=0 docker build --network=host --tag ridgerun/ai-agent-service --file ai-agent-context/ai-agent/docker/Dockerfile ai-agent-context/ +``` + +Change ai-agent-context/ to the path of your context and the tag to the name you want to give to your image. + + +### Launch the container + +The container can be launched by running the following command: + +```bash +docker run --runtime nvidia -it --network host --volume /home/nvidia/config:/ai-agent-config --name ai-agent ridgerun/ai-agent-service:latest ai-agent --system_prompt ai-agent-config/prompt.txt --api_map ai-agent-config/api_mapping.json +``` + +Here we are creating a container called ai-agent. Notice we are mounting the directory /home/nvidia/config into /ai-agent-config, this contains the prompt and API configuration files, you can change it to point to your configuration directory or any place where you have the required configs. Also, we are defining the ai-agent microservice application as entry point with its corresponding parameters. diff --git a/aiagent/__init__.py b/aiagent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aiagent/controllers/__init__.py b/aiagent/controllers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aiagent/controllers/apidispatcher.py b/aiagent/controllers/apidispatcher.py new file mode 100644 index 0000000..dec00fb --- /dev/null +++ b/aiagent/controllers/apidispatcher.py @@ -0,0 +1,106 @@ +""" + Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) + All Rights Reserved. + + The contents of this software are proprietary and confidential to RidgeRun, + LLC. No part of this program may be photocopied, reproduced or translated + into another programming language without prior written consent of + RidgeRun, LLC. The user is free to modify the source code after obtaining + a software license from RidgeRun. All source code changes must be provided + back to RidgeRun without any encumbrance. +""" +import json +import logging + +import requests +from rrmsutils.models.apiresponse import ApiResponse + +logger = logging.getLogger("ai-agent") + + +class ApiDispatcher: + """ + Call API corresponding to function calling request + """ + + def __init__(self, mapping_file): + with open(mapping_file, encoding="utf-8") as json_map: + self._api_mapping = json.load(json_map) + + def parse_request(self, request): + """ + Parse request and map to API call + """ + + # Get request parameters + request_json = json.loads(request) + function = request_json['name'] + arguments = request_json['arguments'] + + # Get mapping parameters + mapping = self._api_mapping[function] + if "ip" in mapping: + ip = mapping['ip'] + else: + ip = "127.0.0.1" + + port = mapping['port'] + path = mapping['path'] + method = mapping['method'] + + # Initialize URI + uri = 'http://' + ip + ':' + str(port) + '/' + path + + # Parse properties + if "properties" in mapping: + uri_arguments = '' + for prop in mapping['properties']: + if not uri_arguments: + uri_arguments = '?' + else: + uri_arguments += '&' + uri_arguments += prop + '=' + + key = str(mapping['properties'][prop]) + + if key in arguments: + uri_arguments += arguments[key] + else: + uri_arguments += key + + uri += uri_arguments + + # Parse body + json_body = None + if "body" in mapping: + json_body = mapping['body'].copy() + for prop in mapping['body']: + key = str(mapping['body'][prop]) + if key in arguments: + json_body[prop] = arguments[key] + + return method, uri, json_body + + def process_request(self, request): + """ + Process request and call corresponding API + """ + + try: + method, uri, json_body = self.parse_request(request) + except Exception as e: + logger.warning(f"Failed to parse request to api. {repr(e)}") + response = ApiResponse( + code=1, message="Missing mapping parameter. " + repr(e)) + return response.model_dump_json(), 200 + + # Send API request + logger.info(f"Sending API request uri: {uri}, body: {json_body}") + + try: + r = requests.request(method, uri, json=json_body) + except Exception as e: + response = ApiResponse(code=1, message=repr(e)) + return response.model_dump_json(), 400 + + return r.text, r.status_code diff --git a/aiagent/controllers/controller.py b/aiagent/controllers/controller.py new file mode 100644 index 0000000..3a52b7a --- /dev/null +++ b/aiagent/controllers/controller.py @@ -0,0 +1,36 @@ +""" + Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) + All Rights Reserved. + + The contents of this software are proprietary and confidential to RidgeRun, + LLC. No part of this program may be photocopied, reproduced or translated + into another programming language without prior written consent of + RidgeRun, LLC. The user is free to modify the source code after obtaining + a software license from RidgeRun. All source code changes must be provided + back to RidgeRun without any encumbrance. +""" + +from abc import ABC, abstractmethod + +from flask import Response + + +class Controller(ABC): + "Flask server method controller" + + @abstractmethod + def add_rules(self, app): + " Add rules to flask server" + + def response(self, data, code: int = 200, mimetype: str = "application/json"): + """Builds and returns Response for a request + + Args: + data: the data to be sent + code (int, optional): HTTPStatus code. Defaults to 200. + mimetype (str, optional): Response mimetype. Defaults to "application/json". + + Returns: + Flask.Response: A Flask Response object with the given data. + """ + return Response(data, code, mimetype=mimetype) diff --git a/aiagent/controllers/prompt.py b/aiagent/controllers/prompt.py new file mode 100644 index 0000000..bde6034 --- /dev/null +++ b/aiagent/controllers/prompt.py @@ -0,0 +1,80 @@ +""" + Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) + All Rights Reserved. + + The contents of this software are proprietary and confidential to RidgeRun, + LLC. No part of this program may be photocopied, reproduced or translated + into another programming language without prior written consent of + RidgeRun, LLC. The user is free to modify the source code after obtaining + a software license from RidgeRun. All source code changes must be provided + back to RidgeRun without any encumbrance. +""" +import logging + +from nano_llm import ChatHistory, NanoLLM +from nano_llm.utils import load_prompts + +logger = logging.getLogger("ai-agent") + + +class LLMPrompt: + """ + A class to process a prompt request with an LLM model + """ + + def __init__(self, system_prompt=None, model="Trelis/Llama-2-7b-chat-hf-function-calling-v3"): + if system_prompt: + system_prompt_list = load_prompts(system_prompt) + system_prompt = '' + system_prompt = system_prompt.join(system_prompt_list) + logger.info(f"system prompt: {system_prompt}") + + self._system_prompt = system_prompt + self._model_name = model + self._model = None + self._chat_history = None + + def start(self): + """ + Load pretrained LLM model and create chat history + """ + # load language model + self._model = NanoLLM.from_pretrained( + self._model_name, + api='mlc', + ) + + # create the chat history + self._chat_history = ChatHistory( + self._model, system_prompt=self._system_prompt) + + def process_prompt(self, prompt, role='user'): + """ + Process prompt with LLM model. + + Returns model reply + """ + # add prompt to the chat history + self._chat_history.append(role=role, msg=prompt) + + # get the latest embeddings (or tokens) from the chat + embedding, _ = self._chat_history.embed_chat( + max_tokens=self._model.config.max_length, + return_tokens=not self._model.has_embed, + ) + + # generate bot reply + reply = self._model.generate( + embedding, + kv_cache=self._chat_history.kv_cache, + stop_tokens=self._chat_history.template.stop, + ) + + # Get the reply text output + text = '' + text = text.join(reply) + text = text.removesuffix('') + self._chat_history.append(role='bot', text=text) + self._chat_history.kv_cache = reply.kv_cache + + return text diff --git a/aiagent/controllers/promptcontroller.py b/aiagent/controllers/promptcontroller.py new file mode 100644 index 0000000..deb7ca5 --- /dev/null +++ b/aiagent/controllers/promptcontroller.py @@ -0,0 +1,88 @@ +""" + Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) + All Rights Reserved. + + The contents of this software are proprietary and confidential to RidgeRun, + LLC. No part of this program may be photocopied, reproduced or translated + into another programming language without prior written consent of + RidgeRun, LLC. The user is free to modify the source code after obtaining + a software license from RidgeRun. All source code changes must be provided + back to RidgeRun without any encumbrance. +""" +import logging + +from flask import request +from flask_cors import cross_origin +from rrmsutils.models.aiagent.prompt import Prompt +from rrmsutils.models.apiresponse import ApiResponse + +from aiagent.controllers.apidispatcher import ApiDispatcher +from aiagent.controllers.controller import Controller +from aiagent.controllers.prompt import LLMPrompt + +logger = logging.getLogger("ai-agent") + + +class PromptController(Controller): + """ + Controller for prompt requests + """ + + def __init__(self, system_prompt, api_map_file): + self._llm_prompt = LLMPrompt(system_prompt) + self._llm_prompt.start() + self._dispatcher = ApiDispatcher(api_map_file) + self._prompt = None + + def add_rules(self, app): + """ + Add prompt update rule at /prompt uri + """ + app.add_url_rule('/prompt', 'update_prompt', + self.update_prompt, methods=['PUT', 'GET']) + + @cross_origin() + def update_prompt(self): + """ + Update prompt request and process it + """ + if request.method == 'PUT': + return self.put_prompt() + if request.method == 'GET': + return self.get_prompt() + + data = ApiResponse( + code=1, message=f'Method {request.method} not supported').model_dump_json() + return self.response(data, 400) + + def put_prompt(self): + """ + Update prompt request and process it + """ + content = request.json + prompt = None + try: + prompt = Prompt.model_validate(content) + except Exception as e: + response = ApiResponse(code=1, message=repr(e)) + return self.response(response.model_dump_json(), 400) + + logger.info(f"prompt: {prompt.prompt}") + self._prompt = prompt + + reply = self._llm_prompt.process_prompt(prompt.prompt) + + logger.info(f"reply: {reply}") + + response, code = self._dispatcher.process_request(reply) + return self.response(response, code) + + def get_prompt(self): + """Return current prompt + """ + if not self._prompt: + data = ApiResponse( + code=1, message='No prompt configured yet').model_dump_json() + return self.response(data, 400) + + return self.response(self._prompt.model_dump_json(), 200) diff --git a/aiagent/controllers/webcontroller.py b/aiagent/controllers/webcontroller.py new file mode 100644 index 0000000..b7d5c44 --- /dev/null +++ b/aiagent/controllers/webcontroller.py @@ -0,0 +1,38 @@ +""" + Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) + All Rights Reserved. + + The contents of this software are proprietary and confidential to RidgeRun, + LLC. No part of this program may be photocopied, reproduced or translated + into another programming language without prior written consent of + RidgeRun, LLC. The user is free to modify the source code after obtaining + a software license from RidgeRun. All source code changes must be provided + back to RidgeRun without any encumbrance. +""" +import logging + +from flask import render_template +from flask_cors import cross_origin + +from aiagent.controllers.controller import Controller + +logger = logging.getLogger("ai-agent") + + +class WebController(Controller): + """ + Controller for web interface + """ + + def add_rules(self, app): + """ + Add rules for web + """ + app.add_url_rule('/', 'home', self.home) + + @cross_origin() + def home(self): + """ + Renders home page + """ + return render_template('index.html') diff --git a/aiagent/main.py b/aiagent/main.py new file mode 100644 index 0000000..8b1d61e --- /dev/null +++ b/aiagent/main.py @@ -0,0 +1,56 @@ +""" + Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) + All Rights Reserved. + + The contents of this software are proprietary and confidential to RidgeRun, + LLC. No part of this program may be photocopied, reproduced or translated + into another programming language without prior written consent of + RidgeRun, LLC. The user is free to modify the source code after obtaining + a software license from RidgeRun. All source code changes must be provided + back to RidgeRun without any encumbrance. +""" + +import argparse +import logging + +from aiagent.controllers.promptcontroller import PromptController +from aiagent.controllers.webcontroller import WebController +from aiagent.server import Server + +logger = logging.getLogger("ai-agent") + + +def parse_args(): + """ Parse arguments """ + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=5010, + help="Port for server") + parser.add_argument("--system_prompt", type=str, default=None, required=True, + help="String with system prompt or path to a txt file with the prompt") + parser.add_argument("--api_map", type=str, default=None, required=True, + help="Path to a JSON file with API mapping configuration") + + args = parser.parse_args() + + return args + + +def main(): + """ + Main application + """ + args = parse_args() + logging.basicConfig(level=logging.INFO) + + controllers = [] + controllers.append(PromptController(args.system_prompt, args.api_map)) + controllers.append(WebController()) + + logger.info("Starting server") + # Launch flask server + server = Server(controllers, port=args.port) + server.start() + + +if __name__ == "__main__": + main() diff --git a/aiagent/server.py b/aiagent/server.py new file mode 100644 index 0000000..2aa05a0 --- /dev/null +++ b/aiagent/server.py @@ -0,0 +1,37 @@ +""" + Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) + All Rights Reserved. + + The contents of this software are proprietary and confidential to RidgeRun, + LLC. No part of this program may be photocopied, reproduced or translated + into another programming language without prior written consent of + RidgeRun, LLC. The user is free to modify the source code after obtaining + a software license from RidgeRun. All source code changes must be provided + back to RidgeRun without any encumbrance. +""" +import logging + +from flask import Flask + + +class Server: + """ + Flask server + """ + + def __init__(self, controllers: list, port=8550): + self._port = port + self._app = Flask(__name__) + + log = logging.getLogger('werkzeug') + log.setLevel(logging.ERROR) + + # Add rules + for controller in controllers: + controller.add_rules(self._app) + + def start(self): + """ + Run the server with given port. + """ + self._app.run(host='0.0.0.0', port=self._port, debug=False) diff --git a/aiagent/templates/index.html b/aiagent/templates/index.html new file mode 100644 index 0000000..67da0d5 --- /dev/null +++ b/aiagent/templates/index.html @@ -0,0 +1,150 @@ + + + + + + AI Agent + + + + + +
+
+ +
+
+
+
+

AI Agent

+
+
+
+
+ +
+ +
+ +
+
+
+ +
+
+
+
+
+
+
+
+
+ + + + + + + + + diff --git a/api/openapi.html b/api/openapi.html new file mode 100644 index 0000000..353c6b6 --- /dev/null +++ b/api/openapi.html @@ -0,0 +1,49 @@ + + + + + Swagger UI + + + + +
+ + + + + + diff --git a/api/openapi.yaml b/api/openapi.yaml new file mode 100644 index 0000000..591b403 --- /dev/null +++ b/api/openapi.yaml @@ -0,0 +1,78 @@ +openapi: 3.0.3 +info: + title: AI Agent Microservice + description: >- + Documentation fot the AI Agent Microservice. AI Agent Microservice allows a natural communication between the user and other microservices. + contact: + email: support@ridgerun.com + version: 1.0.0 +externalDocs: + description: Find out more about AI Agent Microservice + url: https://developer.ridgerun.com +servers: + - url: http://127.0.0.1:5010 +tags: + - name: prompt + description: Prompt +paths: + /prompt: + put: + tags: + - prompt + summary: Send a propmt request to the service + description: Update the service prompt + operationId: update_prompt + requestBody: + description: Update the service prompt + content: + application/json: + schema: + $ref: '#/components/schemas/Prompt' + required: true + responses: + '200': + description: Successful operation + '400': + description: Operation failed + content: + application/json: + schema: + $ref: '#/components/schemas/ApiResponse' + get: + tags: + - prompt + summary: Get the service prompt + description: Get the service prompt + operationId: get_prompt + responses: + '200': + description: Successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/Prompt' + '400': + description: Operation failed + content: + application/json: + schema: + $ref: '#/components/schemas/ApiResponse' +components: + schemas: + Prompt: + required: + - prompt + type: object + properties: + prompt: + type: string + format: string + example: 'Find a car' + ApiResponse: + type: object + properties: + code: + type: integer + format: int32 + message: + type: string diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..400e492 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,296 @@ +# Use NVIDIA L4T 36.2 with TensorRT as base +FROM nvcr.io/nvidia/l4t-tensorrt:r8.6.2-devel + +WORKDIR / + +# Install compilers and build tools +ENV DEBIAN_FRONTEND=noninteractive \ + LANGUAGE=en_US:en \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 + +RUN set -ex \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + locales \ + locales-all \ + tzdata \ + && locale-gen en_US $LANG \ + && update-locale LC_ALL=$LC_ALL LANG=$LANG \ + && locale \ + \ + && apt-get install -y --no-install-recommends \ + build-essential \ + software-properties-common \ + apt-transport-https \ + lsb-release \ + pkg-config \ + git \ + gdb \ + curl \ + wget \ + zip \ + unzip \ + time \ + sshpass \ + ssh-client \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + \ + && gcc --version \ + && g++ --version + +ARG TARPACK_TMPDIR="/tmp/tarpack" \ + TARPACK_PREFIX="/usr/local" \ + WGET_OPTIONS="--quiet --show-progress --progress=bar:force:noscroll" + +# Install python +ARG PYTHON_VERSION_ARG="3.10" + +ENV PYTHON_VERSION=${PYTHON_VERSION_ARG} \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + PYTHONFAULTHANDLER=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=utf-8 \ + PYTHONHASHSEED=random \ + PIP_NO_CACHE_DIR=off \ + PIP_CACHE_PURGE=true \ + PIP_ROOT_USER_ACTION=ignore \ + TWINE_NON_INTERACTIVE=1 \ + DEBIAN_FRONTEND=noninteractive + +RUN set -x \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} || \ + curl -sS https://bootstrap.pypa.io/pip/3.6/get-pip.py | python3.6 \ + && ln -s /usr/bin/python${PYTHON_VERSION} /usr/local/bin/python3 \ + && which python3 \ + && python3 --version \ + && which pip3 \ + && pip3 --version \ + && python3 -m pip install --upgrade pip --index-url https://pypi.org/simple \ + && pip3 install --no-cache-dir --verbose --no-binary :all: psutil \ + && pip3 install --upgrade --no-cache-dir \ + setuptools \ + packaging \ + 'Cython<3' \ + wheel \ + twine + +# Install cmake with pip +RUN set -ex \ + && pip3 install --upgrade --force-reinstall --no-cache-dir --verbose cmake \ + \ + && cmake --version \ + && which cmake + +ARG OPENCV_VERSION="4.8.1" \ + OPENCV_PYTHON="4.x" \ + CUDA_ARCH_BIN="8.7" \ + ARCH=$(uname -i) +RUN pip3 install --no-cache-dir \ + opencv-contrib-python~=${OPENCV_VERSION} + +# Installing gstreamer +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libgstreamer1.0-dev \ + gstreamer1.0-libav \ + gstreamer1.0-rtsp \ + gstreamer1.0-plugins-bad \ + libgstreamer-plugins-base1.0-dev \ + libgstreamer-plugins-good1.0-dev \ + libgstreamer-plugins-bad1.0-dev \ + libgstrtspserver-1.0-0 \ + python3-gi \ + python3-gst-1.0 \ + gstreamer1.0-plugins-rtp \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# Install onnx +ARG ONNX_VERSION="main" + +RUN pip3 install numpy==1.26.4 --upgrade --no-cache-dir --verbose + +RUN pip3 install --no-cache-dir --verbose onnx || \ + pip3 install --no-cache-dir --verbose git+https://github.com/onnx/onnx@${ONNX_VERSION} && \ + pip3 show onnx && \ + python3 -c 'import onnx; print(onnx.__version__)' + +# Installing onnxruntime +ARG ONNXRUNTIME_VERSION="1.17.0" \ + ONNXRUNTIME_BRANCH="v1.17.0" \ + ONNXRUNTIME_FLAGS="--allow_running_as_root" \ + TAR_INDEX_URL="http://jetson.webredirect.org:8000/jp6/cu122" \ + PIP_INDEX_REPO="http://jetson.webredirect.org/jp6/cu122" \ + PIP_UPLOAD_REPO="http://localhost/jp6/cu122" \ + PIP_UPLOAD_USER="jp6" \ + PIP_UPLOAD_PASS="none" \ + PIP_TRUSTED_HOSTS="jetson.webredirect.org" + +ENV TAR_INDEX_URL=${TAR_INDEX_URL} \ + PIP_INDEX_URL=${PIP_INDEX_REPO} \ + PIP_TRUSTED_HOST=${PIP_TRUSTED_HOSTS} + +RUN mkdir -p ${TARPACK_TMPDIR}/uploads \ + && wget ${WGET_OPTIONS} ${TAR_INDEX_URL}/onnxruntime-gpu-${ONNXRUNTIME_VERSION}.tar.gz \ + && wget ${WGET_OPTIONS} ${TAR_INDEX_URL}/onnxruntime-gpu-${ONNXRUNTIME_VERSION}.sha256 \ + && sha256sum --check onnxruntime-gpu-${ONNXRUNTIME_VERSION}.sha256 \ + && tar -xzvf onnxruntime-gpu-${ONNXRUNTIME_VERSION}.tar.gz -C ${TARPACK_PREFIX} \ + && rm onnxruntime-gpu-${ONNXRUNTIME_VERSION}.tar.gz \ + && rm onnxruntime-gpu-${ONNXRUNTIME_VERSION}.sha256 \ + && pip3 install --no-cache-dir --verbose onnxruntime-gpu==${ONNXRUNTIME_VERSION} \ + && python3 -c 'import onnxruntime; print(onnxruntime.__version__);' + +#Installing jetson-inference +RUN pip3 install torchvision==0.18.0a0+6043bc2 + +ADD https://api.github.com/repos/dusty-nv/jetson-inference/git/refs/heads/master /tmp/jetson_inference_version.json + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libglew-dev \ + glew-utils \ + libsoup2.4-dev \ + libjson-glib-dev \ + libgstrtspserver-1.0-dev \ + avahi-utils \ + libopenblas-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# build from source +RUN git clone --recursive --depth=1 https://github.com/dusty-nv/jetson-inference /opt/jetson-inference \ + && cd /opt/jetson-inference \ + && mkdir build \ + && cd build \ + && cmake ../ \ + && make -j$(nproc) \ + && make install \ + # clean build files + && /bin/bash -O extglob -c "cd /opt/jetson-inference/build; rm -rf -v !($(uname -m)|download-models.*)" + +# the jetson-inference installer calls apt-update +RUN rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# install optional dependencies +RUN pip3 install --no-cache-dir --ignore-installed blinker \ + && pip3 install --no-cache-dir --verbose -r /opt/jetson-inference/python/training/detection/ssd/requirements.txt \ + && pip3 install --no-cache-dir --verbose -r /opt/jetson-inference/python/www/flask/requirements.txt \ + && pip3 install --no-cache-dir --verbose -r /opt/jetson-inference/python/www/dash/requirements.txt + +ENV LD_PRELOAD=/lib/aarch64-linux-gnu/libGLdispatch.so.0:${LD_PRELOAD} + + +# Installing huggingface hub +# set the model cache dir +ENV TRANSFORMERS_CACHE=/data/models/huggingface \ + HUGGINGFACE_HUB_CACHE=/data/models/huggingface \ + HF_HOME=/data/models/huggingface + +# install huggingface_hub package (with CLI) +RUN set -ex \ + && pip3 install --no-cache-dir --verbose \ + huggingface_hub[cli] \ + dataclasses \ + \ + # make sure it loads \ + && huggingface-cli --help \ + && pip3 show huggingface_hub \ + && python3 -c 'import huggingface_hub; print(huggingface_hub.__version__)' \ + \ + # for benchmark timing \ + && apt-get update \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# Installing llm transformers +ARG TRANSFORMERS_PACKAGE=transformers==4.40.2 + +# if you want optimum[exporters,onnxruntime] see the optimum package +RUN pip3 install --no-cache-dir --verbose \ + accelerate \ + optimum \ + sentencepiece && \ + \ + # install from pypi, git, ect (sometimes other version got installed) + pip3 uninstall -y transformers && \ + pip3 install --no-cache-dir --verbose ${TRANSFORMERS_PACKAGE} && \ + \ + # "/usr/local/lib/python3.8/dist-packages/transformers/modeling_utils.py", line 118 + # AttributeError: module 'torch.distributed' has no attribute 'is_initialized' + PYTHON_ROOT=`pip3 show transformers | grep Location: | cut -d' ' -f2` && \ + sed -i 's|torch.distributed.is_initialized|torch.distributed.is_available|g' -i ${PYTHON_ROOT}/transformers/modeling_utils.py + +# make sure it loads +RUN pip3 show transformers && python3 -c 'import transformers; print(transformers.__version__)' + +# Installing mlc +ARG TVM_VERSION="0.15.0" \ + MLC_VERSION="0.1.0" \ + MLC_COMMIT="607dc5a" \ + MLC_PATCH="patches/607dc5a.diff" \ + LLVM_VERSION=17 + +ENV LD_LIBRARY_PATH="/usr/local/lib/python${PYTHON_VERSION}/dist-packages/tvm:${LD_LIBRARY_PATH}" \ + TVM_HOME=/opt/mlc-llm/3rdparty/tvm + +# install the wheels +RUN pip3 install --no-cache-dir --verbose tvm==${TVM_VERSION} mlc-llm==${MLC_VERSION} \ + && pip3 install --no-cache-dir --verbose mlc-chat==${MLC_VERSION} || echo "failed to pip install mlc-chat==${MLC_VERSION} (this is expected for mlc>=0.1.1)" \ + && pip3 install --no-cache-dir --verbose 'pydantic>2' + +# we need the source because the MLC model builder relies on it +RUN git clone https://github.com/mlc-ai/mlc-llm /opt/mlc-llm \ + && cd /opt/mlc-llm \ + && git checkout ${MLC_COMMIT} \ + && git submodule update --init --recursive + +RUN ln -s /opt/mlc-llm/3rdparty/tvm/3rdparty /usr/local/lib/python${PYTHON_VERSION}/dist-packages/tvm/3rdparty + +# make sure it loads +RUN pip3 show tvm mlc_llm + +# Installing nano llm + +ARG NANO_LLM_BRANCH=main \ + NANO_LLM_PATH=/opt/NanoLLM + +ENV PYTHONPATH=${PYTHONPATH}:${NANO_LLM_PATH} \ + SSL_KEY=/etc/ssl/private/localhost.key.pem \ + SSL_CERT=/etc/ssl/private/localhost.cert.pem + +ADD https://api.github.com/repos/dusty-nv/NanoLLM/git/refs/heads/${NANO_LLM_BRANCH} /tmp/nano_llm_version.json +RUN git clone --branch=${NANO_LLM_BRANCH} --recursive https://github.com/dusty-nv/NanoLLM ${NANO_LLM_PATH} \ + && cd ${NANO_LLM_PATH} \ + && git checkout 0c297a13bf45e90c1f7de640f58071a5e4a79fb7 + +RUN pip3 install --ignore-installed --no-cache-dir blinker && \ + pip3 install --no-cache-dir --verbose -r ${NANO_LLM_PATH}/requirements.txt && \ + openssl req \ + -new \ + -newkey rsa:4096 \ + -days 3650 \ + -nodes \ + -x509 \ + -keyout ${SSL_KEY} \ + -out ${SSL_CERT} \ + -subj '/CN=localhost' + +RUN mkdir -p /data/models/mlc/dist/models/ + +# Copy rrms-utils +COPY rrms-utils /rrms-utils +RUN cd /rrms-utils/ && pip3 install . + +# Copy ai-agent +COPY ai-agent /ai-agent +RUN cd /ai-agent && pip3 install . diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/build/.gitignore b/docs/build/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..747ffb7 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/_static/.gitignore b/docs/source/_static/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/docs/source/aiagent.controllers.rst b/docs/source/aiagent.controllers.rst new file mode 100644 index 0000000..cb1b349 --- /dev/null +++ b/docs/source/aiagent.controllers.rst @@ -0,0 +1,53 @@ +aiagent.controllers package +=========================== + +Submodules +---------- + +aiagent.controllers.apidispatcher module +---------------------------------------- + +.. automodule:: aiagent.controllers.apidispatcher + :members: + :undoc-members: + :show-inheritance: + +aiagent.controllers.controller module +------------------------------------- + +.. automodule:: aiagent.controllers.controller + :members: + :undoc-members: + :show-inheritance: + +aiagent.controllers.prompt module +--------------------------------- + +.. automodule:: aiagent.controllers.prompt + :members: + :undoc-members: + :show-inheritance: + +aiagent.controllers.promptcontroller module +------------------------------------------- + +.. automodule:: aiagent.controllers.promptcontroller + :members: + :undoc-members: + :show-inheritance: + +aiagent.controllers.webcontroller module +---------------------------------------- + +.. automodule:: aiagent.controllers.webcontroller + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: aiagent.controllers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/aiagent.rst b/docs/source/aiagent.rst new file mode 100644 index 0000000..cd5151b --- /dev/null +++ b/docs/source/aiagent.rst @@ -0,0 +1,37 @@ +aiagent package +=============== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + aiagent.controllers + +Submodules +---------- + +aiagent.main module +------------------- + +.. automodule:: aiagent.main + :members: + :undoc-members: + :show-inheritance: + +aiagent.server module +--------------------- + +.. automodule:: aiagent.server + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: aiagent + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 0000000..a756400 --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,23 @@ +API Documentation +================= +.. raw:: html + + + +
+ + diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..0e95501 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,65 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +# pylint: skip-file + +import os +import shutil +import sys + +project = 'AI Agent' +copyright = '2024, RidgeRun,LLC' +author = 'RidgeRun,LLC' +release = '1.0.0' + +sys.path.insert(0, os.path.abspath('../..')) + + +def copy_custom_files(source_dir, target_dir, file_list): + for file in file_list: + source_file = os.path.join(source_dir, file) + target_file = os.path.join(target_dir, file) + shutil.copyfile(source_file, target_file) + print(f"Copied {file} to {target_dir}") + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx_mdinclude' +] + +# Allow documentation to be built even with missing dependencies +autodoc_mock_imports = [] +try: + import nano_llm +except ImportError: + autodoc_mock_imports.append('nano_llm') + +try: + import rrmsutils +except ImportError: + autodoc_mock_imports.append('rrmsutils') + +templates_path = ['_templates'] +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_show_sourcelink = False +html_static_path = ['_static'] +files_to_copy = ['openapi.html', 'openapi.yaml'] + +# Copy swagger api +copy_custom_files(os.path.abspath('../../api'), '_static', files_to_copy) diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..616ff37 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,24 @@ +.. AI Agent documentation master file, created by + sphinx-quickstart on Wed Jun 26 14:01:02 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to AI Agent's documentation! +==================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + modules + api + readme + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 0000000..20e8f63 --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,7 @@ +aiagent +======= + +.. toctree:: + :maxdepth: 4 + + aiagent diff --git a/docs/source/readme.rst b/docs/source/readme.rst new file mode 100644 index 0000000..f824d32 --- /dev/null +++ b/docs/source/readme.rst @@ -0,0 +1,4 @@ +Service Usage Instructions +========================== + +.. mdinclude:: ../../README.md diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..bba75d2 --- /dev/null +++ b/setup.py @@ -0,0 +1,52 @@ +# Copyright (C) 2024 RidgeRun, LLC (http://www.ridgerun.com) +# All Rights Reserved. +# +# The contents of this software are proprietary and confidential to RidgeRun, +# LLC. No part of this program may be photocopied, reproduced or translated +# into another programming language without prior written consent of +# RidgeRun, LLC. The user is free to modify the source code after obtaining +# a software license from RidgeRun. All source code changes must be provided +# back to RidgeRun without any encumbrance. + +# Install with: python3 -m pip install . +# For developer mode install with: pip install -e . + +# pylint: skip-file + +import logging +import shlex +from subprocess import check_call + +import setuptools +from setuptools.command.develop import develop + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + + +setuptools.setup( + name="ai-agent", + version="1.0.0", + author='RidgeRun LLC', + author_email='support@ridgerun.com', + description="RidgeRun AI Agent Microservice", + long_description=long_description, + long_description_content_type="text/markdown", + url='https://ridgerun.com', + packages=setuptools.find_packages(), + include_package_data=True, + python_requires='>=3.0, <4', + install_requires=[ + 'pydantic', + 'flask', + 'flask-cors', + 'sphinx', + 'sphinx_rtd_theme', + 'sphinx-mdinclude' + ], + entry_points={ + 'console_scripts': [ + 'ai-agent=aiagent.main:main', + ], + }, +)