Skip to content

Commit

Permalink
Revert "refactor(tools): migrate to TypeScript and enhance build conf…
Browse files Browse the repository at this point in the history
…iguration"

This reverts commit 09887ef.
  • Loading branch information
anthonydevs17 committed Feb 20, 2025
1 parent dcac955 commit eea51a0
Show file tree
Hide file tree
Showing 49 changed files with 130 additions and 3,626 deletions.
872 changes: 42 additions & 830 deletions packages/tools/package-lock.json

Large diffs are not rendered by default.

44 changes: 13 additions & 31 deletions packages/tools/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,72 +5,58 @@
"private": false,
"main": "dist/index.cjs.js",
"module": "dist/index.esm.js",
"types": "dist/index.d.ts",
"exports": {
".": {
"import": "./dist/index.esm.js",
"require": "./dist/index.cjs.js",
"types": "./dist/index.d.ts"
"require": "./dist/index.cjs.js"
},
"./firecrawl": {
"import": "./dist/firecrawl/index.esm.js",
"require": "./dist/firecrawl/index.cjs.js",
"types": "./dist/firecrawl/index.d.ts"
"require": "./dist/firecrawl/index.cjs.js"
},
"./tavily": {
"import": "./dist/tavily/index.esm.js",
"require": "./dist/tavily/index.cjs.js",
"types": "./dist/tavily/index.d.ts"
"require": "./dist/tavily/index.cjs.js"
},
"./serper": {
"import": "./dist/serper/index.esm.js",
"require": "./dist/serper/index.cjs.js",
"types": "./dist/serper/index.d.ts"
"require": "./dist/serper/index.cjs.js"
},
"./exa": {
"import": "./dist/exa/index.esm.js",
"require": "./dist/exa/index.cjs.js",
"types": "./dist/exa/index.d.ts"
"require": "./dist/exa/index.cjs.js"
},
"./wolframalpha": {
"import": "./dist/wolframalpha/index.esm.js",
"require": "./dist/wolframalpha/index.cjs.js",
"types": "./dist/wolframalpha/index.d.ts"
"require": "./dist/wolframalpha/index.cjs.js"
},
"./simple-rag": {
"import": "./dist/simple-rag/index.esm.js",
"require": "./dist/simple-rag/index.cjs.js",
"types": "./dist/simple-rag/index.d.ts"
"require": "./dist/simple-rag/index.cjs.js"
},
"./website-search": {
"import": "./dist/website-search/index.esm.js",
"require": "./dist/website-search/index.cjs.js",
"types": "./dist/website-search/index.d.ts"
"require": "./dist/website-search/index.cjs.js"
},
"./pdf-search": {
"import": "./dist/pdf-search/index.esm.js",
"require": "./dist/pdf-search/index.cjs.js",
"types": "./dist/pdf-search/index.d.ts"
"require": "./dist/pdf-search/index.cjs.js"
},
"./textfile-search": {
"import": "./dist/textfile-search/index.esm.js",
"require": "./dist/textfile-search/index.cjs.js",
"types": "./dist/textfile-search/index.d.ts"
"require": "./dist/textfile-search/index.cjs.js"
},
"./zapier-webhook": {
"import": "./dist/zapier-webhook/index.esm.js",
"require": "./dist/zapier-webhook/index.cjs.js",
"types": "./dist/zapier-webhook/index.d.ts"
"require": "./dist/zapier-webhook/index.cjs.js"
},
"./make-webhook": {
"import": "./dist/make-webhook/index.esm.js",
"require": "./dist/make-webhook/index.cjs.js",
"types": "./dist/make-webhook/index.d.ts"
"require": "./dist/make-webhook/index.cjs.js"
},
"./jina-url-to-markdown": {
"import": "./dist/jina-url-to-markdown/index.esm.js",
"require": "./dist/jina-url-to-markdown/index.cjs.js",
"types": "./dist/jina-url-to-markdown/index.d.ts"
"require": "./dist/jina-url-to-markdown/index.cjs.js"
}
},
"files": [
Expand Down Expand Up @@ -133,7 +119,6 @@
"@rollup/plugin-node-resolve": "^15.3.0",
"@rollup/plugin-replace": "^6.0.1",
"@rollup/plugin-terser": "^0.4.4",
"@rollup/plugin-typescript": "^12.1.2",
"@storybook/addon-essentials": "^8.3.6",
"@storybook/addon-interactions": "^8.3.6",
"@storybook/addon-links": "^8.3.6",
Expand All @@ -142,7 +127,6 @@
"@storybook/react": "^8.3.6",
"@storybook/react-vite": "^8.3.6",
"@storybook/test": "^8.3.6",
"@types/node": "^22.13.4",
"babel-jest": "^29.7.0",
"eslint": "^8.4.1",
"eslint-plugin-storybook": "^0.10.1",
Expand All @@ -152,8 +136,6 @@
"rollup": "^4.24.0",
"rollup-plugin-node-polyfills": "^0.2.1",
"storybook": "^8.3.6",
"tslib": "^2.8.1",
"typescript": "^5.7.3",
"undici": "^6.20.1"
},
"eslintConfig": {
Expand Down
39 changes: 3 additions & 36 deletions packages/tools/rollup.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import terser from '@rollup/plugin-terser';
import json from '@rollup/plugin-json';
import nodePolyfills from 'rollup-plugin-node-polyfills'; // Correct plugin name
import replace from '@rollup/plugin-replace';
import typescript from '@rollup/plugin-typescript';

// Array of tool folder names
const toolFolders = [
Expand All @@ -25,7 +24,7 @@ const toolFolders = [
]; // Add more folder names as needed

const toolConfigs = toolFolders.map((tool) => {
const inputPath = `src/${tool}/index.ts`;
const inputPath = `src/${tool}/index.js`; // Adjusted for plain JavaScript

return defineConfig({
input: inputPath,
Expand All @@ -49,19 +48,9 @@ const toolConfigs = toolFolders.map((tool) => {
'pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js',
],
plugins: [
typescript({
tsconfig: './tsconfig.json',
declaration: true,
declarationDir: `dist/${tool}/`,
outDir: `./dist/${tool}`,
rootDir: `./src/${tool}/`,
}),
nodeResolve({
browser: true,
preferBuiltins: false, // Use polyfills for Node built-in modules
extensions: ['.ts', '.js', '.json'],
moduleDirectories: ['node_modules', 'src'],
preserveSymlinks: true,
}),
commonjs(),
json(),
Expand All @@ -81,7 +70,7 @@ const toolConfigs = toolFolders.map((tool) => {

// Main index config
const mainConfig = defineConfig({
input: 'src/index.ts',
input: 'src/index.js',
output: [
{
file: 'dist/index.cjs.js',
Expand All @@ -102,12 +91,6 @@ const mainConfig = defineConfig({
'pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js',
],
plugins: [
typescript({
tsconfig: './tsconfig.json',
declaration: true,
declarationDir: `./dist/`,
rootDir: './src/',
}),
nodeResolve({
browser: true,
preferBuiltins: false,
Expand All @@ -126,9 +109,8 @@ const mainConfig = defineConfig({
}),
],
});

const ragToolkitConfig = defineConfig({
input: 'src/_utils/rag/ragToolkit.ts',
input: 'src/_utils/rag/ragToolkit.js',
output: [
{
file: 'dist/rag-toolkit/index.cjs.js',
Expand All @@ -144,12 +126,6 @@ const ragToolkitConfig = defineConfig({
},
],
plugins: [
typescript({
tsconfig: './tsconfig.json',
declaration: true,
declarationDir: './dist/rag-toolkit/',
rootDir: './src/_utils/rag/',
}),
nodeResolve({
browser: true,
preferBuiltins: false,
Expand All @@ -158,15 +134,6 @@ const ragToolkitConfig = defineConfig({
json(),
nodePolyfills(),
terser(),
replace({
preventAssignment: true,
values: {
'node:fs/promises': 'fs/promises',
'Promise.withResolvers':
'(() => ({ promise: new Promise(() => {}), resolve: () => {}, reject: () => {} }))',
},
}),
],
});

export default [ragToolkitConfig, ...toolConfigs, mainConfig];
Original file line number Diff line number Diff line change
@@ -1,46 +1,15 @@
import { BaseDocumentLoader } from '@langchain/core/document_loaders/base';
import { Document as BaseDocument } from 'langchain/document';
// import { getDocument, GlobalWorkerOptions } from 'pdfjs-dist';

interface PDFMetadata {
[key: string]: any;
}

interface PDFTextItem {
str?: string;
[key: string]: any;
}

interface PDFPage {
getTextContent(): Promise<{ items: PDFTextItem[] }>;
}

interface PDFDocument {
numPages: number;
getPage(pageNum: number): Promise<PDFPage>;
}

interface PDFDocumentProxy {
promise: Promise<PDFDocument>;
}

interface PDFLib {
getDocument(data: Uint8Array): PDFDocumentProxy;
GlobalWorkerOptions: {
workerSrc: string;
};
}

export class BrowserPDFLoader extends BaseDocumentLoader {
private file: string | File;
private metadata: PDFMetadata;

constructor(file: string | File, metadata: PDFMetadata = {}) {
class BrowserPDFLoader extends BaseDocumentLoader {
constructor(file, metadata = {}) {
super();
this.file = file;
this.metadata = metadata;
}

async load(): Promise<BaseDocument[]> {
async load() {
if (!this.file) {
console.log('No file selected.');
return [];
Expand All @@ -53,28 +22,26 @@ export class BrowserPDFLoader extends BaseDocumentLoader {
}

try {
const { getDocument, GlobalWorkerOptions } = (await import(
'pdfjs-dist'
)) as PDFLib;
const { getDocument, GlobalWorkerOptions } = await import('pdfjs-dist');
GlobalWorkerOptions.workerSrc =
'https://cdnjs.cloudflare.com/ajax/libs/pdf.js/4.8.69/pdf.worker.min.mjs';

const fileReader = new FileReader();

const fileReaderPromise = new Promise<Uint8Array>((resolve, reject) => {
const fileReaderPromise = new Promise((resolve, reject) => {
fileReader.onload = () => {
resolve(new Uint8Array(fileReader.result as ArrayBuffer));
resolve(new Uint8Array(fileReader.result));
};
fileReader.onerror = (error) => {
reject(error);
};
});

fileReader.readAsArrayBuffer(this.file as File);
fileReader.readAsArrayBuffer(this.file);

const typedArray = await fileReaderPromise;
const pdf = await getDocument(typedArray).promise;
const pagesContent: string[] = [];
const pagesContent = [];

for (let pageNum = 1; pageNum <= pdf.numPages; pageNum++) {
const page = await pdf.getPage(pageNum);
Expand All @@ -94,8 +61,11 @@ export class BrowserPDFLoader extends BaseDocumentLoader {
);

return documents;
} catch (_error) {
} catch {
// console.log('Error loading PDF:', error);
throw new Error('Error loading PDF.');
}
}
}

export { BrowserPDFLoader };
Original file line number Diff line number Diff line change
@@ -1,25 +1,20 @@
import { BaseDocumentLoader } from '@langchain/core/document_loaders/base';
import { Document as BaseDocument } from 'langchain/document';

interface TextInputMetadata {
[key: string]: any;
}

export class TextInputLoader extends BaseDocumentLoader {
private text: string;
private metadata: TextInputMetadata;

constructor(text: string, metadata: TextInputMetadata = {}) {
class TextInputLoader extends BaseDocumentLoader {
constructor(text, metadata = {}) {
super();
this.text = text;
this.metadata = metadata;
}

async load(): Promise<BaseDocument[]> {
async load() {
const document = new BaseDocument({
pageContent: this.text,
metadata: this.metadata,
});
return [document];
}
}

export { TextInputLoader };
25 changes: 0 additions & 25 deletions packages/tools/src/_utils/rag/loaders/textInputLoader/index.ts

This file was deleted.

Loading

0 comments on commit eea51a0

Please sign in to comment.