From 19a07d2dbf4c198471d6f14af05edb76fd228412 Mon Sep 17 00:00:00 2001 From: topce Date: Thu, 20 Jun 2024 11:08:39 +0200 Subject: [PATCH] fix: allow to have multiple document in local index (#85) --- packages/api/src/functions/documents-post.ts | 23 +++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/api/src/functions/documents-post.ts b/packages/api/src/functions/documents-post.ts index 077a5359..a9c189df 100644 --- a/packages/api/src/functions/documents-post.ts +++ b/packages/api/src/functions/documents-post.ts @@ -1,4 +1,5 @@ -import { HttpRequest, HttpResponseInit, InvocationContext, app } from '@azure/functions'; +import fs from 'node:fs/promises'; +import { type HttpRequest, type HttpResponseInit, type InvocationContext, app } from '@azure/functions'; import { AzureOpenAIEmbeddings } from '@langchain/openai'; import { PDFLoader } from 'langchain/document_loaders/fs/pdf'; import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; @@ -53,8 +54,15 @@ export async function postDocuments(request: HttpRequest, context: InvocationCon // If no environment variables are set, it means we are running locally context.log('No Azure OpenAI endpoint set, using Ollama models and local DB'); const embeddings = new OllamaEmbeddings({ model: ollamaEmbeddingsModel }); - const store = await FaissStore.fromDocuments(documents, embeddings, {}); - await store.save(faissStoreFolder); + const folderExists = await checkFolderExists(faissStoreFolder); + if (folderExists) { + const store = await FaissStore.load(faissStoreFolder, embeddings); + await store.addDocuments(documents); + await store.save(faissStoreFolder); + } else { + const store = await FaissStore.fromDocuments(documents, embeddings, {}); + await store.save(faissStoreFolder); + } } if (storageUrl && containerName) { @@ -81,6 +89,15 @@ export async function postDocuments(request: HttpRequest, context: InvocationCon } } +async function checkFolderExists(folderPath: string): Promise { + try { + const stats = await fs.stat(folderPath); + return stats.isDirectory(); + } catch { + return false; + } +} + app.http('documents-post', { route: 'documents', methods: ['POST'],