From a4f9e5572ef017b4e7cf73dd87f4d9a9216c4540 Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Mon, 10 Feb 2025 17:09:03 -0500 Subject: [PATCH] fix(cu): linting --- servers/cu/src/app.js | 1 - servers/cu/src/bootstrap.js | 2 +- servers/cu/src/effects/ao-evaluation.js | 2 - servers/cu/src/effects/ao-evaluation.test.js | 378 +++++++++--------- .../cu/src/effects/worker/dumpEvaluations.js | 56 +-- .../effects/worker/dumpEvaluations.test.js | 118 +++--- .../cu/src/effects/worker/loadEvaluation.js | 3 +- .../src/effects/worker/loadEvaluation.test.js | 266 ++++++------ .../src/effects/worker/loadEvaluation/main.js | 24 +- .../worker/saveEvaluationToDir.test.js | 106 +++-- 10 files changed, 472 insertions(+), 484 deletions(-) diff --git a/servers/cu/src/app.js b/servers/cu/src/app.js index f937e328b..6d0ba4122 100644 --- a/servers/cu/src/app.js +++ b/servers/cu/src/app.js @@ -38,7 +38,6 @@ export const server = pipeP( logger('Currently Pending readState operations: %j', domain.apis.pendingReadStates()) }, config.MEM_MONITOR_INTERVAL) memMonitor.unref() - if (config.PROCESS_MEMORY_CACHE_CHECKPOINT_INTERVAL) { logger('Setting up Interval to Checkpoint all Processes every %s', ms(config.PROCESS_MEMORY_CACHE_CHECKPOINT_INTERVAL)) diff --git a/servers/cu/src/bootstrap.js b/servers/cu/src/bootstrap.js index dbafd13f5..f0af648c5 100644 --- a/servers/cu/src/bootstrap.js +++ b/servers/cu/src/bootstrap.js @@ -584,7 +584,7 @@ export const createApis = async (ctx) => { ctx.logger('Error in loadEvaluation worker while dumping evaluations', e) throw new Error(`Error in loadEvaluation worker: ${e}`) }) - ) + ) ) return { metrics, stats, pendingReadStates, readState, dryRun, readResult, readResults, readCronResults, checkpointWasmMemoryCache, healthcheck, dumpEvaluations } diff --git a/servers/cu/src/effects/ao-evaluation.js b/servers/cu/src/effects/ao-evaluation.js index 81aab4e28..ec813258a 100644 --- a/servers/cu/src/effects/ao-evaluation.js +++ b/servers/cu/src/effects/ao-evaluation.js @@ -173,7 +173,6 @@ export function saveEvaluationWith ({ DISABLE_PROCESS_EVALUATION_CACHE, db, logg const statements = [] if (!DISABLE_PROCESS_EVALUATION_CACHE) { - // If we have a directory and bucket, we need to save the evaluation to the directory, not sqlite if (EVALUATION_RESULT_DIR && EVALUATION_RESULT_BUCKET) { statements.push({ @@ -198,7 +197,6 @@ export function saveEvaluationWith ({ DISABLE_PROCESS_EVALUATION_CACHE, db, logg ] }) } else { - statements.push({ sql: ` INSERT OR IGNORE INTO ${EVALUATIONS_TABLE} diff --git a/servers/cu/src/effects/ao-evaluation.test.js b/servers/cu/src/effects/ao-evaluation.test.js index 8b4dccf23..ea76bb8a0 100644 --- a/servers/cu/src/effects/ao-evaluation.test.js +++ b/servers/cu/src/effects/ao-evaluation.test.js @@ -88,59 +88,58 @@ describe('ao-evaluation', () => { }) test('find evaluation from directory if dir/bucket are set', async () => { - const findEvaluation = findEvaluationSchema.implement( - findEvaluationWith({ - db: { - engine: 'sqlite', - query: async ({ parameters }) => { - assert.deepStrictEqual(parameters, ['process-124,1702677252111,1']) - return [{ - id: 'process-124,1702677252111,1', - processId: 'process-124', - messageId: 'message-123', - deepHash: 'deepHash-123', - nonce: 1, - epoch: 0, - timestamp: 1702677252111, - ordinate: '1', - blockHeight: 1234, - cron: undefined, - evaluatedAt: evaluatedAt.getTime(), - output: undefined - }] - } - }, - loadEvaluation: async ({ processId, messageId }) => { - assert.equal(processId, 'process-124') - assert.equal(messageId, 'message-123') - return { Messages: [{ foo: 'bar' }] } - }, - EVALUATION_RESULT_DIR: 'test-directory', - EVALUATION_RESULT_BUCKET: 'test-bucket' - }) - ) - - const res = await findEvaluation({ - processId: 'process-124', - to: 1702677252111, - ordinate: '1', - cron: undefined, - messageId: 'message-123' - }) - assert.deepStrictEqual(res, { - processId: 'process-124', - messageId: 'message-123', - deepHash: 'deepHash-123', - nonce: 1, - epoch: 0, - timestamp: 1702677252111, - ordinate: '1', - blockHeight: 1234, - cron: undefined, - evaluatedAt, - output: { Messages: [{ foo: 'bar' }] } + const findEvaluation = findEvaluationSchema.implement( + findEvaluationWith({ + db: { + engine: 'sqlite', + query: async ({ parameters }) => { + assert.deepStrictEqual(parameters, ['process-124,1702677252111,1']) + return [{ + id: 'process-124,1702677252111,1', + processId: 'process-124', + messageId: 'message-123', + deepHash: 'deepHash-123', + nonce: 1, + epoch: 0, + timestamp: 1702677252111, + ordinate: '1', + blockHeight: 1234, + cron: undefined, + evaluatedAt: evaluatedAt.getTime(), + output: undefined + }] + } + }, + loadEvaluation: async ({ processId, messageId }) => { + assert.equal(processId, 'process-124') + assert.equal(messageId, 'message-123') + return { Messages: [{ foo: 'bar' }] } + }, + EVALUATION_RESULT_DIR: 'test-directory', + EVALUATION_RESULT_BUCKET: 'test-bucket' }) - + ) + + const res = await findEvaluation({ + processId: 'process-124', + to: 1702677252111, + ordinate: '1', + cron: undefined, + messageId: 'message-123' + }) + assert.deepStrictEqual(res, { + processId: 'process-124', + messageId: 'message-123', + deepHash: 'deepHash-123', + nonce: 1, + epoch: 0, + timestamp: 1702677252111, + ordinate: '1', + blockHeight: 1234, + cron: undefined, + evaluatedAt, + output: { Messages: [{ foo: 'bar' }] } + }) }) test('return 404 if dir/bucket are set but AWS Credentials are not set', async () => { @@ -183,14 +182,14 @@ describe('ao-evaluation', () => { cron: undefined, messageId: 'message-123' }) - .catch(err => { - assert.equal(err.status, 404) - assert.equal(err.message, 'Could not find evaluation: AWS Credentials not set') - return { ok: true } - }) + .catch(err => { + assert.equal(err.status, 404) + assert.equal(err.message, 'Could not find evaluation: AWS Credentials not set') + return { ok: true } + }) assert(res.ok) - }) + }) }) describe('saveEvaluation', () => { @@ -243,7 +242,7 @@ describe('ao-evaluation', () => { logger, saveEvaluationToDir: async () => { assert.fail('saveEvaluationToDir should not be called') - }, + } }) ) @@ -283,7 +282,7 @@ describe('ao-evaluation', () => { logger, saveEvaluationToDir: async () => { assert.fail('saveEvaluationToDir should not be called') - }, + } }) ) @@ -327,7 +326,7 @@ describe('ao-evaluation', () => { logger, saveEvaluationToDir: async () => { assert.fail('saveEvaluationToDir should not be called') - }, + } }) ) @@ -338,7 +337,6 @@ describe('ao-evaluation', () => { }) }) - test('with bucket and dir set, save the evaluation and message', async () => { const saveEvaluation = saveEvaluationSchema.implement( saveEvaluationWith({ @@ -546,7 +544,6 @@ describe('ao-evaluation', () => { assert.equal(res.length, 2) }) - test("return the evaluations between 'from' and 'to', get outputs from dir/bucket", async () => { const evaluatedAt = new Date() const mockEval = { @@ -602,56 +599,17 @@ describe('ao-evaluation', () => { assert.equal(res.length, 2) assert.deepStrictEqual(res[0].output, { Messages: [{ foo: 'bar' }] }) assert.deepStrictEqual(res[1].output, { Messages: [{ foo: 'bar' }] }) - }) - - describe('findMessageBeforeWith', () => { - test('find the prior message by deepHash', async () => { - const findMessageBefore = findMessageBeforeSchema.implement( - findMessageBeforeWith({ - db: { - engine: 'sqlite', - query: async ({ parameters }) => { - assert.deepStrictEqual(parameters, [ - 'deepHash-123', - 'process-123', - 0, - 0, - 3 - ]) - - const mockAssigment = { - id: 'deepHash-123', - processId: 'process-123', - seq: '0:3' - } - - return [mockAssigment] - } - } - }) - ) - - const res = await findMessageBefore({ - processId: 'process-123', - messageId: 'message-123', - deepHash: 'deepHash-123', - isAssignment: false, - epoch: 0, - nonce: 3 - }) - - assert.deepStrictEqual(res, { id: 'deepHash-123' }) }) - describe('find the prior message by messageId', () => { - test('if no deepHash was calculated', async () => { + describe('findMessageBeforeWith', () => { + test('find the prior message by deepHash', async () => { const findMessageBefore = findMessageBeforeSchema.implement( findMessageBeforeWith({ db: { engine: 'sqlite', query: async ({ parameters }) => { assert.deepStrictEqual(parameters, [ - 'message-123', + 'deepHash-123', 'process-123', 0, 0, @@ -659,7 +617,7 @@ describe('ao-evaluation', () => { ]) const mockAssigment = { - id: 'message-123', + id: 'deepHash-123', processId: 'process-123', seq: '0:3' } @@ -673,79 +631,142 @@ describe('ao-evaluation', () => { const res = await findMessageBefore({ processId: 'process-123', messageId: 'message-123', - deepHash: undefined, + deepHash: 'deepHash-123', isAssignment: false, epoch: 0, nonce: 3 }) - assert.deepStrictEqual(res, { id: 'message-123' }) + assert.deepStrictEqual(res, { id: 'deepHash-123' }) }) - test('if it is an assignment', async () => { - const findMessageBefore = findMessageBeforeSchema.implement( - findMessageBeforeWith({ - db: { - engine: 'sqlite', - query: async ({ sql, parameters }) => { - // Only the postgres engine uses POSITION - assert.ok(!sql.includes('POSITION')) - assert.deepStrictEqual(parameters, [ - 'message-123', - 'process-123', - 0, - 0, - 3 - ]) - - const mockAssigment = { - id: 'message-123', - processId: 'process-123', - seq: '0:3' + describe('find the prior message by messageId', () => { + test('if no deepHash was calculated', async () => { + const findMessageBefore = findMessageBeforeSchema.implement( + findMessageBeforeWith({ + db: { + engine: 'sqlite', + query: async ({ parameters }) => { + assert.deepStrictEqual(parameters, [ + 'message-123', + 'process-123', + 0, + 0, + 3 + ]) + + const mockAssigment = { + id: 'message-123', + processId: 'process-123', + seq: '0:3' + } + + return [mockAssigment] } + } + }) + ) + + const res = await findMessageBefore({ + processId: 'process-123', + messageId: 'message-123', + deepHash: undefined, + isAssignment: false, + epoch: 0, + nonce: 3 + }) - return [mockAssigment] + assert.deepStrictEqual(res, { id: 'message-123' }) + }) + + test('if it is an assignment', async () => { + const findMessageBefore = findMessageBeforeSchema.implement( + findMessageBeforeWith({ + db: { + engine: 'sqlite', + query: async ({ sql, parameters }) => { + // Only the postgres engine uses POSITION + assert.ok(!sql.includes('POSITION')) + assert.deepStrictEqual(parameters, [ + 'message-123', + 'process-123', + 0, + 0, + 3 + ]) + + const mockAssigment = { + id: 'message-123', + processId: 'process-123', + seq: '0:3' + } + + return [mockAssigment] + } } - } + }) + ) + + const res = await findMessageBefore({ + processId: 'process-123', + messageId: 'message-123', + deepHash: 'deepHash-123', + isAssignment: true, + epoch: 0, + nonce: 3 }) - ) - const res = await findMessageBefore({ - processId: 'process-123', - messageId: 'message-123', - deepHash: 'deepHash-123', - isAssignment: true, - epoch: 0, - nonce: 3 + assert.deepStrictEqual(res, { id: 'message-123' }) }) + test('if it is an assignment, postgres', async () => { + const findMessageBefore = findMessageBeforeSchema.implement( + findMessageBeforeWith({ + db: { + engine: 'postgres', + query: async ({ sql, parameters }) => { + // Only the postgres engine uses POSITION + assert.ok(sql.includes('POSITION')) + assert.deepStrictEqual(parameters, [ + 'message-123', + 'process-123', + 0, + 0, + 3 + ]) + + const mockAssigment = { + id: 'message-123', + processId: 'process-123', + seq: '0:3' + } + + return [mockAssigment] + } + } + }) + ) + + const res = await findMessageBefore({ + processId: 'process-123', + messageId: 'message-123', + deepHash: 'deepHash-123', + isAssignment: true, + epoch: 0, + nonce: 3 + }) - assert.deepStrictEqual(res, { id: 'message-123' }) + assert.deepStrictEqual(res, { id: 'message-123' }) + }) }) - test('if it is an assignment, postgres', async () => { + + test('return 404 status if not found', async () => { const findMessageBefore = findMessageBeforeSchema.implement( findMessageBeforeWith({ db: { - engine: 'postgres', - query: async ({ sql, parameters }) => { - // Only the postgres engine uses POSITION - assert.ok(sql.includes('POSITION')) - assert.deepStrictEqual(parameters, [ - 'message-123', - 'process-123', - 0, - 0, - 3 - ]) - - const mockAssigment = { - id: 'message-123', - processId: 'process-123', - seq: '0:3' - } - - return [mockAssigment] - } - } + engine: 'sqlite', + query: async () => [] + }, + logger }) ) @@ -757,36 +778,13 @@ describe('ao-evaluation', () => { epoch: 0, nonce: 3 }) + .catch(err => { + assert.equal(err.status, 404) + return { ok: true } + }) - assert.deepStrictEqual(res, { id: 'message-123' }) - }) - }) - - test('return 404 status if not found', async () => { - const findMessageBefore = findMessageBeforeSchema.implement( - findMessageBeforeWith({ - db: { - engine: 'sqlite', - query: async () => [] - }, - logger - }) - ) - - const res = await findMessageBefore({ - processId: 'process-123', - messageId: 'message-123', - deepHash: 'deepHash-123', - isAssignment: true, - epoch: 0, - nonce: 3 + assert(res.ok) }) - .catch(err => { - assert.equal(err.status, 404) - return { ok: true } - }) - - assert(res.ok) }) }) -})}) +}) diff --git a/servers/cu/src/effects/worker/dumpEvaluations.js b/servers/cu/src/effects/worker/dumpEvaluations.js index 286682570..4eb3c0cff 100644 --- a/servers/cu/src/effects/worker/dumpEvaluations.js +++ b/servers/cu/src/effects/worker/dumpEvaluations.js @@ -2,33 +2,33 @@ import path from 'path' import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3' export function dumpEvaluationsWith ({ EVALUATION_RESULT_DIR, EVALUATION_RESULT_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION, uploadFileToS3With, globSync, logger }) { - return async (processId) => { - if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY || !AWS_REGION) { - return 'AWS Credentials not set' - } - const globPath = path.join(EVALUATION_RESULT_DIR, `${processId ?? ``}**.json`) - const files = globSync(globPath) - const s3Client = new S3Client({ - region: AWS_REGION, - credentials: { - accessKeyId: AWS_ACCESS_KEY_ID, - secretAccessKey: AWS_SECRET_ACCESS_KEY - } - }) - logger('Dumping evaluations to S3 Bucket: %s', EVALUATION_RESULT_BUCKET) - const uploadFileToS3 = uploadFileToS3With(s3Client) - for (const file of files) { - const bucketName = EVALUATION_RESULT_BUCKET - const key = file.split('/').pop() - logger('Uploading file %s to S3 Bucket: %s', key, bucketName) - try { - await uploadFileToS3(file, bucketName, key) - logger('Successfully uploaded file %s to S3 Bucket: %s', key, bucketName) - } catch (e) { - logger('Error uploading file %s to S3 Bucket: %s', key, bucketName) - throw new Error(`Error uploading file ${key} to S3: ${e}`) - } - } - logger('Successfully dumped evaluations to S3 Bucket: %s', EVALUATION_RESULT_BUCKET) + return async (processId) => { + if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY || !AWS_REGION) { + return 'AWS Credentials not set' } + const globPath = path.join(EVALUATION_RESULT_DIR, `${processId ?? ''}**.json`) + const files = globSync(globPath) + const s3Client = new S3Client({ + region: AWS_REGION, + credentials: { + accessKeyId: AWS_ACCESS_KEY_ID, + secretAccessKey: AWS_SECRET_ACCESS_KEY + } + }) + logger('Dumping evaluations to S3 Bucket: %s', EVALUATION_RESULT_BUCKET) + const uploadFileToS3 = uploadFileToS3With(s3Client) + for (const file of files) { + const bucketName = EVALUATION_RESULT_BUCKET + const key = file.split('/').pop() + logger('Uploading file %s to S3 Bucket: %s', key, bucketName) + try { + await uploadFileToS3(file, bucketName, key) + logger('Successfully uploaded file %s to S3 Bucket: %s', key, bucketName) + } catch (e) { + logger('Error uploading file %s to S3 Bucket: %s', key, bucketName) + throw new Error(`Error uploading file ${key} to S3: ${e}`) + } + } + logger('Successfully dumped evaluations to S3 Bucket: %s', EVALUATION_RESULT_BUCKET) + } } diff --git a/servers/cu/src/effects/worker/dumpEvaluations.test.js b/servers/cu/src/effects/worker/dumpEvaluations.test.js index 876f7f522..7c172af0d 100644 --- a/servers/cu/src/effects/worker/dumpEvaluations.test.js +++ b/servers/cu/src/effects/worker/dumpEvaluations.test.js @@ -5,70 +5,68 @@ import assert from 'node:assert' import { createTestLogger } from '../../domain/logger.js' import { dumpEvaluationsWith } from './dumpEvaluations.js' - const logger = createTestLogger({ name: 'ao-cu:worker' }) - describe('dumpEvaluations', async () => { - describe('dumpEvaluationsWith', async () => { - test('AWS Credentials not set', async () => { - const dumpEvaluations = dumpEvaluationsWith({ - EVALUATION_RESULT_DIR: './test/evaluation-results', - EVALUATION_RESULT_BUCKET: 'test-bucket' - }) - const result = await dumpEvaluations('test-process-id') - assert.strictEqual(result, 'AWS Credentials not set') - }) + describe('dumpEvaluationsWith', async () => { + test('AWS Credentials not set', async () => { + const dumpEvaluations = dumpEvaluationsWith({ + EVALUATION_RESULT_DIR: './test/evaluation-results', + EVALUATION_RESULT_BUCKET: 'test-bucket' + }) + const result = await dumpEvaluations('test-process-id') + assert.strictEqual(result, 'AWS Credentials not set') + }) - test('Correct glob path', async () => { - let globCalled = false - const dumpEvaluations = dumpEvaluationsWith({ - EVALUATION_RESULT_DIR: 'test-dir/', - EVALUATION_RESULT_BUCKET: 'test-bucket', - AWS_ACCESS_KEY_ID: 'test-access-key-id', - AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', - AWS_REGION: 'test-region', - globSync: (path) => { - globCalled = true - assert.strictEqual(path, 'test-dir/test-process-id**.json') - return [] - }, - uploadFileToS3With: async () => null, - logger - }) - await dumpEvaluations('test-process-id') - assert.ok(globCalled) - }) + test('Correct glob path', async () => { + let globCalled = false + const dumpEvaluations = dumpEvaluationsWith({ + EVALUATION_RESULT_DIR: 'test-dir/', + EVALUATION_RESULT_BUCKET: 'test-bucket', + AWS_ACCESS_KEY_ID: 'test-access-key-id', + AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', + AWS_REGION: 'test-region', + globSync: (path) => { + globCalled = true + assert.strictEqual(path, 'test-dir/test-process-id**.json') + return [] + }, + uploadFileToS3With: async () => null, + logger + }) + await dumpEvaluations('test-process-id') + assert.ok(globCalled) + }) - test('Upload file to S3', async () => { - let globCalled = false - let uploadFileToS3Called = false - const dumpEvaluations = dumpEvaluationsWith({ - EVALUATION_RESULT_DIR: 'test-dir/', - EVALUATION_RESULT_BUCKET: 'test-bucket', - AWS_ACCESS_KEY_ID: 'test-access-key-id', - AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', - AWS_REGION: 'test-region', - globSync: (path) => { - globCalled = true - assert.strictEqual(path, 'test-dir/test-process-id**.json') - return ['test-dir/test-process-id-test-message-1.json', 'test-dir/test-process-id-test-message-2.json'] - }, - uploadFileToS3With: () => { - uploadFileToS3Called = true - let i = 1 - return async (file, bucket, key) => { - assert.strictEqual(file, `test-dir/test-process-id-test-message-${i}.json`) - assert.strictEqual(bucket, 'test-bucket') - assert.strictEqual(key, `test-process-id-test-message-${i}.json`) - i++ - } - }, - logger - }) - await dumpEvaluations('test-process-id') - assert.ok(globCalled) - assert.ok(uploadFileToS3Called) - }) + test('Upload file to S3', async () => { + let globCalled = false + let uploadFileToS3Called = false + const dumpEvaluations = dumpEvaluationsWith({ + EVALUATION_RESULT_DIR: 'test-dir/', + EVALUATION_RESULT_BUCKET: 'test-bucket', + AWS_ACCESS_KEY_ID: 'test-access-key-id', + AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', + AWS_REGION: 'test-region', + globSync: (path) => { + globCalled = true + assert.strictEqual(path, 'test-dir/test-process-id**.json') + return ['test-dir/test-process-id-test-message-1.json', 'test-dir/test-process-id-test-message-2.json'] + }, + uploadFileToS3With: () => { + uploadFileToS3Called = true + let i = 1 + return async (file, bucket, key) => { + assert.strictEqual(file, `test-dir/test-process-id-test-message-${i}.json`) + assert.strictEqual(bucket, 'test-bucket') + assert.strictEqual(key, `test-process-id-test-message-${i}.json`) + i++ + } + }, + logger + }) + await dumpEvaluations('test-process-id') + assert.ok(globCalled) + assert.ok(uploadFileToS3Called) }) + }) }) diff --git a/servers/cu/src/effects/worker/loadEvaluation.js b/servers/cu/src/effects/worker/loadEvaluation.js index ec0efc884..9ad3936e0 100644 --- a/servers/cu/src/effects/worker/loadEvaluation.js +++ b/servers/cu/src/effects/worker/loadEvaluation.js @@ -36,7 +36,7 @@ export function loadEvaluationFromDirWith ({ logger, readFileSync }) { export function loadEvaluationFromS3With ({ EVALUATION_RESULT_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION, logger, getFileFromS3With }) { return async ({ fileName }) => { - const s3Client = new S3Client({ + const s3Client = new S3Client({ region: AWS_REGION, credentials: { accessKeyId: AWS_ACCESS_KEY_ID, @@ -54,4 +54,3 @@ export function loadEvaluationFromS3With ({ EVALUATION_RESULT_BUCKET, AWS_ACCESS return file } } - diff --git a/servers/cu/src/effects/worker/loadEvaluation.test.js b/servers/cu/src/effects/worker/loadEvaluation.test.js index 8c160f406..54068899d 100644 --- a/servers/cu/src/effects/worker/loadEvaluation.test.js +++ b/servers/cu/src/effects/worker/loadEvaluation.test.js @@ -5,148 +5,146 @@ import assert from 'node:assert' import { createTestLogger } from '../../domain/logger.js' import { loadEvaluationFromDirWith, loadEvaluationFromS3With, loadEvaluationWith } from './loadEvaluation.js' - const logger = createTestLogger({ name: 'ao-cu:worker' }) - describe('loadEvaluation', async () => { - describe('loadEvaluationWith', async () => { - test('Correct file name, file exists', async () => { - let existsCalled = false - let loadFromDirCalled = false - const loadEvaluation = loadEvaluationWith({ - EVALUATION_RESULT_DIR: 'test-directory', - EVALUATION_RESULT_BUCKET: 'test-bucket', - AWS_ACCESS_KEY_ID: 'test-access-key-id', - AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', - AWS_REGION: 'test-region', - logger, - existsSync: (path) => { - existsCalled = true - assert.equal(path, 'test-directory/test-process-id-test-message-id.json') - return true - }, - writeFileSync: () => { - assert.fail('writeFileSync should not be called') - }, - loadEvaluationFromDir: async ({ fileName }) => { - loadFromDirCalled = true - assert.equal(fileName, 'test-directory/test-process-id-test-message-id.json') - return { foo: 'bar' } - }, - loadEvaluationFromS3: async () => { - assert.fail('loadEvaluationFromS3 should not be called') - } - }) - const result = await loadEvaluation({ - messageId: 'test-message-id', - processId: 'test-process-id' - }) - assert.deepStrictEqual(result, { foo: 'bar' }) - assert.strictEqual(existsCalled, true) - assert.strictEqual(loadFromDirCalled, true) - }) + describe('loadEvaluationWith', async () => { + test('Correct file name, file exists', async () => { + let existsCalled = false + let loadFromDirCalled = false + const loadEvaluation = loadEvaluationWith({ + EVALUATION_RESULT_DIR: 'test-directory', + EVALUATION_RESULT_BUCKET: 'test-bucket', + AWS_ACCESS_KEY_ID: 'test-access-key-id', + AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', + AWS_REGION: 'test-region', + logger, + existsSync: (path) => { + existsCalled = true + assert.equal(path, 'test-directory/test-process-id-test-message-id.json') + return true + }, + writeFileSync: () => { + assert.fail('writeFileSync should not be called') + }, + loadEvaluationFromDir: async ({ fileName }) => { + loadFromDirCalled = true + assert.equal(fileName, 'test-directory/test-process-id-test-message-id.json') + return { foo: 'bar' } + }, + loadEvaluationFromS3: async () => { + assert.fail('loadEvaluationFromS3 should not be called') + } + }) + const result = await loadEvaluation({ + messageId: 'test-message-id', + processId: 'test-process-id' + }) + assert.deepStrictEqual(result, { foo: 'bar' }) + assert.strictEqual(existsCalled, true) + assert.strictEqual(loadFromDirCalled, true) + }) - test('AWS Credentials not set', async () => { - const loadEvaluation = loadEvaluationWith({ - EVALUATION_RESULT_DIR: './test/evaluation-results', - EVALUATION_RESULT_BUCKET: 'test-bucket', - AWS_ACCESS_KEY_ID: null, - AWS_SECRET_ACCESS_KEY: null, - AWS_REGION: null, - existsSync: () => false, - writeFileSync: () => {}, - loadEvaluationFromDir: async () => null, - loadEvaluationFromS3: async () => null, - logger - }) - const result = await loadEvaluation({ - messageId: 'test-message-id', - processId: 'test-process-id' - }) - assert.strictEqual(result, 'AWS Credentials not set') - }) + test('AWS Credentials not set', async () => { + const loadEvaluation = loadEvaluationWith({ + EVALUATION_RESULT_DIR: './test/evaluation-results', + EVALUATION_RESULT_BUCKET: 'test-bucket', + AWS_ACCESS_KEY_ID: null, + AWS_SECRET_ACCESS_KEY: null, + AWS_REGION: null, + existsSync: () => false, + writeFileSync: () => {}, + loadEvaluationFromDir: async () => null, + loadEvaluationFromS3: async () => null, + logger + }) + const result = await loadEvaluation({ + messageId: 'test-message-id', + processId: 'test-process-id' + }) + assert.strictEqual(result, 'AWS Credentials not set') + }) - test('File does not exist', async () => { - let existsCalled = false - let writeFileSyncCalled = false - const loadEvaluation = loadEvaluationWith({ - EVALUATION_RESULT_DIR: 'test-directory', - EVALUATION_RESULT_BUCKET: 'test-bucket', - AWS_ACCESS_KEY_ID: 'test-access-key-id', - AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', - AWS_REGION: 'test-region', - logger, - existsSync: (path) => { - existsCalled = true - assert.equal(path, 'test-directory/test-process-id-test-message-id.json') - return false - }, - writeFileSync: (path, data ) => { - writeFileSyncCalled = true - assert.equal(path, 'test-directory/test-process-id-test-message-id.json') - assert.equal(data, JSON.stringify({ foo: 'bar' })) - }, - loadEvaluationFromDir: async () => { - assert.fail('loadEvaluationFromDir should not be called') - }, - loadEvaluationFromS3: async ({ fileName }) => { - assert.equal(fileName, 'test-directory/test-process-id-test-message-id.json') - return { foo: 'bar' } - } - }) - const result = await loadEvaluation({ - messageId: 'test-message-id', - processId: 'test-process-id' - }) - assert.deepStrictEqual(result, { foo: 'bar' }) - assert.strictEqual(existsCalled, true) - }) + test('File does not exist', async () => { + let existsCalled = false + let writeFileSyncCalled = false + const loadEvaluation = loadEvaluationWith({ + EVALUATION_RESULT_DIR: 'test-directory', + EVALUATION_RESULT_BUCKET: 'test-bucket', + AWS_ACCESS_KEY_ID: 'test-access-key-id', + AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', + AWS_REGION: 'test-region', + logger, + existsSync: (path) => { + existsCalled = true + assert.equal(path, 'test-directory/test-process-id-test-message-id.json') + return false + }, + writeFileSync: (path, data) => { + writeFileSyncCalled = true + assert.equal(path, 'test-directory/test-process-id-test-message-id.json') + assert.equal(data, JSON.stringify({ foo: 'bar' })) + }, + loadEvaluationFromDir: async () => { + assert.fail('loadEvaluationFromDir should not be called') + }, + loadEvaluationFromS3: async ({ fileName }) => { + assert.equal(fileName, 'test-directory/test-process-id-test-message-id.json') + return { foo: 'bar' } + } + }) + const result = await loadEvaluation({ + messageId: 'test-message-id', + processId: 'test-process-id' + }) + assert.deepStrictEqual(result, { foo: 'bar' }) + assert.strictEqual(existsCalled, true) }) + }) - describe('loadEvaluationFromDirWith', async () => { - test('Load from dir', async () => { - let readCalled = false - const loadEvaluationFromDir = loadEvaluationFromDirWith({ - logger, - readFileSync: (path, encoding) => { - readCalled = true - assert.equal(path, 'test-directory/test-process-id-test-message-id.json') - assert.equal(encoding, 'utf8') - return JSON.stringify({ foo: 'bar' }) - } - }) - const result = await loadEvaluationFromDir({ - fileName: 'test-directory/test-process-id-test-message-id.json' - }) - assert.deepStrictEqual(result, { foo: 'bar' }) - assert.strictEqual(readCalled, true) - }) + describe('loadEvaluationFromDirWith', async () => { + test('Load from dir', async () => { + let readCalled = false + const loadEvaluationFromDir = loadEvaluationFromDirWith({ + logger, + readFileSync: (path, encoding) => { + readCalled = true + assert.equal(path, 'test-directory/test-process-id-test-message-id.json') + assert.equal(encoding, 'utf8') + return JSON.stringify({ foo: 'bar' }) + } + }) + const result = await loadEvaluationFromDir({ + fileName: 'test-directory/test-process-id-test-message-id.json' + }) + assert.deepStrictEqual(result, { foo: 'bar' }) + assert.strictEqual(readCalled, true) }) + }) - describe('loadEvaluationFromS3With', async () => { - test('Load from S3', async () => { - let getFileFromS3Called = false - const loadEvaluationFromS3 = loadEvaluationFromS3With({ - EVALUATION_RESULT_BUCKET: 'test-bucket', - AWS_ACCESS_KEY_ID: 'test-access-key-id', - AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', - AWS_REGION: 'test-region', - logger, - getFileFromS3With: (bucketName, s3Client) => { - getFileFromS3Called = true - assert.equal(bucketName, 'test-bucket') - return async (fileName) => { - assert.equal(fileName, 'test-directory/test-process-id-test-message-id.json') - return JSON.stringify({ foo: 'bar' }) - } - } - }) - const result = await loadEvaluationFromS3({ - fileName: 'test-directory/test-process-id-test-message-id.json' - }) - assert.deepStrictEqual(result, { foo: 'bar' }) - assert.strictEqual(getFileFromS3Called, true) - }) + describe('loadEvaluationFromS3With', async () => { + test('Load from S3', async () => { + let getFileFromS3Called = false + const loadEvaluationFromS3 = loadEvaluationFromS3With({ + EVALUATION_RESULT_BUCKET: 'test-bucket', + AWS_ACCESS_KEY_ID: 'test-access-key-id', + AWS_SECRET_ACCESS_KEY: 'test-secret-access-key', + AWS_REGION: 'test-region', + logger, + getFileFromS3With: (bucketName, s3Client) => { + getFileFromS3Called = true + assert.equal(bucketName, 'test-bucket') + return async (fileName) => { + assert.equal(fileName, 'test-directory/test-process-id-test-message-id.json') + return JSON.stringify({ foo: 'bar' }) + } + } + }) + const result = await loadEvaluationFromS3({ + fileName: 'test-directory/test-process-id-test-message-id.json' + }) + assert.deepStrictEqual(result, { foo: 'bar' }) + assert.strictEqual(getFileFromS3Called, true) }) + }) }) diff --git a/servers/cu/src/effects/worker/loadEvaluation/main.js b/servers/cu/src/effects/worker/loadEvaluation/main.js index cb42a3f8d..a5dbdf05d 100644 --- a/servers/cu/src/effects/worker/loadEvaluation/main.js +++ b/servers/cu/src/effects/worker/loadEvaluation/main.js @@ -6,14 +6,14 @@ import { GetObjectCommand, PutObjectCommand } from '@aws-sdk/client-s3' export const createApis = async (ctx) => { const uploadFileToS3With = (s3Client) => { return async (filePath, bucketName, key) => { - const fileStream = fs.createReadStream(filePath); - const params = { - Bucket: bucketName, - Key: key, - Body: fileStream - }; - const command = new PutObjectCommand(params); - await s3Client.send(command); + const fileStream = fs.createReadStream(filePath) + const params = { + Bucket: bucketName, + Key: key, + Body: fileStream + } + const command = new PutObjectCommand(params) + await s3Client.send(command) } } @@ -37,17 +37,17 @@ export const createApis = async (ctx) => { logger: ctx.logger, existsSync: fs.existsSync, writeFileSync: fs.writeFileSync, - loadEvaluationFromDir: loadEvaluationFromDirWith({ + loadEvaluationFromDir: loadEvaluationFromDirWith({ logger: ctx.logger, - readFileSync: fs.readFileSync + readFileSync: fs.readFileSync }), - loadEvaluationFromS3: loadEvaluationFromS3With({ + loadEvaluationFromS3: loadEvaluationFromS3With({ EVALUATION_RESULT_BUCKET: ctx.EVALUATION_RESULT_BUCKET, AWS_ACCESS_KEY_ID: ctx.AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY: ctx.AWS_SECRET_ACCESS_KEY, AWS_REGION: ctx.AWS_REGION, logger: ctx.logger, - getFileFromS3With: getFileFromS3With + getFileFromS3With }) }) const dumpEvaluations = dumpEvaluationsWith({ diff --git a/servers/cu/src/effects/worker/saveEvaluationToDir.test.js b/servers/cu/src/effects/worker/saveEvaluationToDir.test.js index 33207a68b..2043663a0 100644 --- a/servers/cu/src/effects/worker/saveEvaluationToDir.test.js +++ b/servers/cu/src/effects/worker/saveEvaluationToDir.test.js @@ -5,63 +5,61 @@ import assert from 'node:assert' import { createTestLogger } from '../../domain/logger.js' import { saveEvaluationToDirWith } from './saveEvaluationToDir.js' - const logger = createTestLogger({ name: 'ao-cu:worker' }) - describe('saveEvaluationToDir', async () => { - describe('saveEvaluationToDirWith', async () => { - test('File exists (exit)', async () => { - let existsSyncCalled = false - let writeFileSyncCalled = false - const saveEvaluationToDir = saveEvaluationToDirWith({ - EVALUATION_RESULT_DIR: 'test-directory', - existsSync: (path) => { - existsSyncCalled = true - assert.equal(path, 'test-directory/test-process-id-test-message-id.json') - return true - }, - writeFileSync: (path, data) => { - writeFileSyncCalled = true - assert.fail('writeFileSync should not be called') - }, - logger - }) - const result = await saveEvaluationToDir({ - messageId: 'test-message-id', - processId: 'test-process-id', - output: { foo: 'bar' } - }) - assert.deepStrictEqual(result, { foo: 'bar' }) - assert.ok(existsSyncCalled) - assert.ok(!writeFileSyncCalled) - }) + describe('saveEvaluationToDirWith', async () => { + test('File exists (exit)', async () => { + let existsSyncCalled = false + let writeFileSyncCalled = false + const saveEvaluationToDir = saveEvaluationToDirWith({ + EVALUATION_RESULT_DIR: 'test-directory', + existsSync: (path) => { + existsSyncCalled = true + assert.equal(path, 'test-directory/test-process-id-test-message-id.json') + return true + }, + writeFileSync: (path, data) => { + writeFileSyncCalled = true + assert.fail('writeFileSync should not be called') + }, + logger + }) + const result = await saveEvaluationToDir({ + messageId: 'test-message-id', + processId: 'test-process-id', + output: { foo: 'bar' } + }) + assert.deepStrictEqual(result, { foo: 'bar' }) + assert.ok(existsSyncCalled) + assert.ok(!writeFileSyncCalled) + }) - test('File does not exist (write)', async () => { - let existsSyncCalled = false - let writeFileSyncCalled = false - const saveEvaluationToDir = saveEvaluationToDirWith({ - EVALUATION_RESULT_DIR: 'test-directory', - existsSync: (path) => { - existsSyncCalled = true - assert.equal(path, 'test-directory/test-process-id-test-message-id.json') - return false - }, - writeFileSync: (path, data) => { - writeFileSyncCalled = true - assert.equal(path, 'test-directory/test-process-id-test-message-id.json') - assert.equal(data, JSON.stringify({ foo: 'bar' })) - }, - logger - }) - const result = await saveEvaluationToDir({ - messageId: 'test-message-id', - processId: 'test-process-id', - output: { foo: 'bar' } - }) - assert.deepStrictEqual(result, { foo: 'bar' }) - assert.ok(existsSyncCalled) - assert.ok(writeFileSyncCalled) - }) + test('File does not exist (write)', async () => { + let existsSyncCalled = false + let writeFileSyncCalled = false + const saveEvaluationToDir = saveEvaluationToDirWith({ + EVALUATION_RESULT_DIR: 'test-directory', + existsSync: (path) => { + existsSyncCalled = true + assert.equal(path, 'test-directory/test-process-id-test-message-id.json') + return false + }, + writeFileSync: (path, data) => { + writeFileSyncCalled = true + assert.equal(path, 'test-directory/test-process-id-test-message-id.json') + assert.equal(data, JSON.stringify({ foo: 'bar' })) + }, + logger + }) + const result = await saveEvaluationToDir({ + messageId: 'test-message-id', + processId: 'test-process-id', + output: { foo: 'bar' } + }) + assert.deepStrictEqual(result, { foo: 'bar' }) + assert.ok(existsSyncCalled) + assert.ok(writeFileSyncCalled) }) + }) })