Ecarton/cumulus 3751 from 18.5.2

CHANGELOG.md

@@ -23,6 +23,8 @@ included in the future will have a corresponding CHANGELOG entry in future relea
 
 ### Changed
 
+- **CUMULUS-3751**
+  - lowered memory reservation of python_processing_workflow and hello_world_workflow to share resources with ecs_move_granule_collections workflow
 - **CUMULUS-3967**
   - Pinned @aws-sdk/client-s3 in @cumulus/aws-client to 3.726.0 to address breaking changes/incompatibility in releases > 3.726.0
   - Pinned @aws-sdk/client-s3 in @cumulus/lib-storage to 3.726.0 to address breaking changes/incompatibility in releases > 3.726.0
@@ -44,6 +46,17 @@ included in the future will have a corresponding CHANGELOG entry in future relea
 - **CUMULUS-3981**
   - Added required $metadata field when creating new instance of ServiceException.
 
+### Added
+
+- **CUMULUS-3751**
+  - Added `move-granule-collections` to move granules to a different collection.
+    - expects a list of granules along with a new (target) collection
+    - transfers those granules to that collection in cumulus data stores
+    - moves those granule files in S3 according to pathing of target collection
+- **CUMULUS-3757**
+  - Added a `/granules` [endpoint](https://nasa.github.io/cumulus-api/#bulk-update-granules-collectionId) `PATCH/bulkPatchGranuleCollection` which updates a batch of granule records collectionId to a new collectionId. This endpoint takes a list of granules, a collectionId, and an `esConcurrency` variable, updating the granules' to the collectionId passed with the payload in both postgres and elasticsearch, while providing concurrency for updating elasticsearch to tailor for performance and database needs.
+  - Added a `/granules` [endpoint](https://nasa.github.io/cumulus-api/#bulk-update-granules) `PATCH/bulkPatch` which applies PATCH to a list of granules. For its payload, this endpoint takes a list of granules (the updates to be made to the granule, similar to the pre-existing `PATCH`), a `dbConcurrency` and `dbMaxPool` variables for configuring concurrency and database thoroughput for postgres to tailor to performance and database needs.
+
 ## [v18.5.2] 2024-12-12
 
 ### Breaking Changes

example/cumulus-tf/ecs_hello_world_workflow.tf

@@ -21,7 +21,7 @@ module "hello_world_service" {
   image                                 = "${data.aws_ecr_repository.ecs_task_image.repository_url}:${var.ecs_task_image_version}"
 
   cpu                = 400
-  memory_reservation = 700
+  memory_reservation = 400
   default_log_retention_days                     = var.default_log_retention_days
   cloudwatch_log_retention_periods               = var.cloudwatch_log_retention_periods
 

example/cumulus-tf/ecs_move_granule_collections.asl.json

@@ -0,0 +1,31 @@
+{
+  "Comment": "Moves granules across collections",
+  "StartAt": "EcsTaskMoveGranuleCollections",
+  "States": {
+    "EcsTaskMoveGranuleCollections": {
+      "Parameters": {
+        "cma": {
+          "event.$": "$",
+          "task_config": {
+            "buckets": "{$.meta.buckets}",
+            "provider": "{$.meta.provider}",
+            "sourceCollection": "{$.meta.sourceCollection}",
+            "targetCollection": "{$.meta.targetCollection}"
+          }
+        }
+      },
+      "Type": "Task",
+      "Resource": "${ecs_task_move_granule_collections_id}",
+      "TimeoutSeconds": 60,
+      "Retry": [
+        {
+          "ErrorEquals": [
+            "States.Timeout"
+          ],
+          "MaxAttempts": 1
+        }
+      ],
+      "End": true
+    }
+  }
+}

example/cumulus-tf/ecs_move_granule_collections.tf

@@ -0,0 +1,54 @@
+resource "aws_sfn_activity" "ecs_task_move_granule_collections" {
+  name = "${var.prefix}-EcsTaskMoveGranuleCollections"
+  tags = local.tags
+}
+
+module "move_granule_collections_services" {
+  source = "../../tf-modules/cumulus_ecs_service"
+
+  prefix = var.prefix
+  name   = "MoveGranuleCollections"
+  tags   = local.tags
+
+  cluster_arn                           = module.cumulus.ecs_cluster_arn
+  desired_count                         = 1
+  image                                 = "${data.aws_ecr_repository.ecs_task_image.repository_url}:${var.ecs_task_image_version}"
+
+  cpu                = 400
+  memory_reservation = 700
+  default_log_retention_days                     = var.default_log_retention_days
+  cloudwatch_log_retention_periods               = var.cloudwatch_log_retention_periods
+
+  environment = {
+    AWS_DEFAULT_REGION = data.aws_region.current.name
+    stackName = var.prefix
+  }
+  command = [
+    "cumulus-ecs-task",
+    "--activityArn",
+    aws_sfn_activity.ecs_task_move_granule_collections.id,
+    "--lambdaArn",
+    module.cumulus.move_granule_collections_task.task_arn,
+    "--lastModified",
+    module.cumulus.move_granule_collections_task.last_modified_date
+  ]
+}
+
+
+module "ecs_move_granule_collections" {
+  source = "../../tf-modules/workflow"
+
+  prefix          = var.prefix
+  name            = "ECSMoveGranuleCollectionsWorkflow"
+  workflow_config = module.cumulus.workflow_config
+  system_bucket   = var.system_bucket
+  tags            = local.tags
+
+
+  state_machine_definition = templatefile(
+    "${path.module}/ecs_move_granule_collections.asl.json",
+    {
+      ecs_task_move_granule_collections_id: aws_sfn_activity.ecs_task_move_granule_collections.id
+    }
+  )
+}

example/cumulus-tf/python_processing_workflow.tf

@@ -21,7 +21,7 @@ module "python_test_ingest_processing_service" {
   image                                 = "${data.aws_ecr_repository.cumulus_test_ingest_process.repository_url}:${var.cumulus_test_ingest_image_version}"
 
   cpu                = 400
-  memory_reservation = 700
+  memory_reservation = 400
 
   environment = {
     AWS_DEFAULT_REGION = data.aws_region.current.name

example/cumulus-tf/python_reference_workflow.tf

@@ -20,7 +20,7 @@ module "python_processing_service" {
   image                                 = "${data.aws_ecr_repository.cumulus_process_activity.repository_url}:${var.cumulus_process_activity_version}"
 
   cpu                = 400
-  memory_reservation = 700
+  memory_reservation = 400
 
   environment = {
     AWS_DEFAULT_REGION = data.aws_region.current.name

example/spec/parallel/moveGranuleCollections/MoveGranuleCollectionWorkflowSpec.js

@@ -0,0 +1,134 @@
+const { deleteExecution } = require('@cumulus/api-client/executions');
+const { ActivityStep } = require('@cumulus/integration-tests/sfnStep');
+const fs = require('fs');
+const { waitForListObjectsV2ResultCount } = require('@cumulus/integration-tests');
+const {
+  deleteS3Object,
+} = require('@cumulus/aws-client/S3');
+const { createProvider } = require('@cumulus/api-client/providers');
+const { deleteGranule } = require('@cumulus/api-client/granules');
+const { buildAndStartWorkflow } = require('../../helpers/workflowUtils');
+const { loadConfig, createTestSuffix, createTimestampedTestId } = require('../../helpers/testUtils');
+const { setupInitialState, getTargetFiles, getTargetCollection, getSourceCollection } = require('./move-granule-collection-spec-utils');
+
+const activityStep = new ActivityStep();
+
+describe('The MoveGranuleCollection workflow using ECS', () => {
+  let workflowExecutionArn;
+  let config;
+  let finalFiles;
+  let beforeAllFailed = false;
+  const granuleIds = ['MOD11A1.A2017200.h19v04.006.2017201090724'];
+  afterAll(async () => {
+    try {
+      await Promise.all(finalFiles.map((fileObj) => deleteS3Object(
+        fileObj.bucket,
+        fileObj.key
+      )));
+    } catch {
+      console.log('no need to delete s3 objects');
+    }
+    try {
+      await Promise.all(granuleIds.map((granuleId) => deleteGranule({ prefix: config.stackName, granuleId})));
+    } catch {
+      console.log('no need to delete granules');
+    }
+
+    try {
+      await deleteExecution({ prefix: config.stackName, executionArn: workflowExecutionArn });
+    } catch {
+      console.log('no need to delete execution');
+    }
+  });
+  beforeAll(async () => {
+
+    const sourceUrlPrefix = 'move-granule-collection-testing';
+    const targetUrlPrefix = 'move-granule-collection-testing-target';
+    config = await loadConfig();
+    const providersDir = './data/providers/s3/';
+    const stackName = config.stackName;
+    const testId = createTimestampedTestId(config.stackName, 'MoveGranuleCollections');
+    const testSuffix = createTestSuffix(testId);
+    const provider = { id: `s3_provider${testSuffix}` };
+
+    const providerJson = JSON.parse(fs.readFileSync(`${providersDir}/s3_provider.json`, 'utf8'));
+    const providerData = {
+      ...providerJson,
+      id: provider.id,
+      host: config.bucket,
+    };
+    await createProvider({ prefix: config.stackName, provider: providerData })
+
+    finalFiles = getTargetFiles(targetUrlPrefix, config);
+    //upload to cumulus
+
+    try {
+      await setupInitialState(stackName, sourceUrlPrefix, targetUrlPrefix, config);
+
+      workflowExecutionArn = await buildAndStartWorkflow(
+        config.stackName,
+        config.bucket,
+        'ECSMoveGranuleCollectionsWorkflow',
+        getSourceCollection(sourceUrlPrefix),
+        provider,
+        { granules: ['MOD11A1.A2017200.h19v04.006.2017201090724'] },
+        {
+          targetCollection: getTargetCollection(targetUrlPrefix),
+          sourceCollection: getSourceCollection(sourceUrlPrefix),
+          buckets: config.buckets
+        }
+      );
+      await Promise.all(finalFiles.map((file) => expectAsync(
+        waitForListObjectsV2ResultCount({
+          bucket: file.bucket,
+          prefix: file.key,
+          desiredCount: 1,
+          interval: 5 * 1000,
+          timeout: 60 * 1000,
+        })
+      ).toBeResolved()));
+    } catch (error) {
+      console.log(`files do not appear to have been moved: error: ${error}`);
+      beforeAllFailed = true;
+    }
+  });
+
+  it('executes successfully', () => {
+    expect(beforeAllFailed).toEqual(false);
+  });
+
+  it('outputs the updated granules', async () => {
+    expect(beforeAllFailed).toEqual(false);
+    const activityOutput = await activityStep.getStepOutput(
+      workflowExecutionArn,
+      'EcsTaskMoveGranuleCollections'
+    );
+    expect(activityOutput.payload.granules[0].files).toEqual([
+      {
+        key: 'move-granule-collection-testing-target/MOD11A1.A2017200.h19v04.006.2017201090724.hdf',
+        bucket: config.buckets.protected.name,
+        type: 'data',
+        fileName: 'MOD11A1.A2017200.h19v04.006.2017201090724.hdf',
+      },
+      {
+        key: 'move-granule-collection-testing-target/jpg/example2/MOD11A1.A2017200.h19v04.006.2017201090724_1.jpg',
+        bucket: config.buckets.public.name,
+        type: 'browse',
+        fileName: 'MOD11A1.A2017200.h19v04.006.2017201090724_1.jpg',
+      },
+      {
+        key: 'move-granule-collection-testing-target/MOD11A1.A2017200.h19v04.006.2017201090724_2.jpg',
+        bucket: config.buckets.public.name,
+        type: 'browse',
+        fileName: 'MOD11A1.A2017200.h19v04.006.2017201090724_2.jpg',
+      },
+      {
+        key: 'move-granule-collection-testing-target/MOD11A1.A2017200.h19v04.006.2017201090724.cmr.xml',
+        bucket: config.buckets.public.name,
+        type: 'metadata',
+        fileName: 'MOD11A1.A2017200.h19v04.006.2017201090724.cmr.xml',
+      },
+    ]);
+    expect(activityOutput.payload.granules[0].granuleId).toEqual('MOD11A1.A2017200.h19v04.006.2017201090724');
+  });
+});

example/spec/parallel/moveGranuleCollections/MoveGranuleCollectionsSpec.js

@@ -0,0 +1,90 @@
+'use strict';
+
+const { InvokeCommand } = require('@aws-sdk/client-lambda');
+const { lambda } = require('@cumulus/aws-client/services');
+const {
+  deleteS3Object,
+} = require('@cumulus/aws-client/S3');
+const { waitForListObjectsV2ResultCount } = require('@cumulus/integration-tests');
+const {
+  granules,
+} = require('@cumulus/api-client');
+
+const { v4: uuidv4 } = require('uuid');
+const { loadConfig } = require('../../helpers/testUtils');
+const { constructCollectionId } = require('../../../../packages/message/Collections');
+const { getTargetCollection, getProcessGranule, setupInitialState, getPayload, getTargetFiles } = require('./move-granule-collection-spec-utils');
+describe('when moveGranulesCollection is called', () => {
+  let stackName;
+  const sourceUrlPrefix = `source_path/${uuidv4()}`;
+  const targetUrlPrefix = `target_path/${uuidv4()}`;
+  const targetCollection = getTargetCollection(targetUrlPrefix);
+  let processGranule;
+  let config;
+  // let systemBucket;
+  beforeAll(async () => {
+    config = await loadConfig();
+    stackName = config.stackName;
+    processGranule = getProcessGranule(sourceUrlPrefix, config);
+  });
+
+  describe('under normal circumstances', () => {
+    let beforeAllFailed = false;
+    let finalFiles;
+    afterAll(async () => {
+      await Promise.all(finalFiles.map((fileObj) => deleteS3Object(
+        fileObj.bucket,
+        fileObj.key
+      )));
+    });
+    beforeAll(async () => {
+      finalFiles = getTargetFiles(targetUrlPrefix, config);
+      const payload = getPayload(sourceUrlPrefix, targetUrlPrefix, config);
+      //upload to cumulus
+      try {
+        await setupInitialState(stackName, sourceUrlPrefix, targetUrlPrefix, config);
+        const { $metadata } = await lambda().send(new InvokeCommand({
+          FunctionName: `${stackName}-MoveGranuleCollections`,
+          InvocationType: 'RequestResponse',
+          Payload: JSON.stringify({
+            cma: {
+              meta: payload.meta,
+              task_config: payload.config,
+              event: {
+                payload: payload.input,
+              },
+            },
+          }),
+        }));
+        if ($metadata.httpStatusCode >= 400) {
+          console.log(`lambda invocation to set up failed, code ${$metadata.httpStatusCode}`);
+          beforeAllFailed = true;
+        }
+        await Promise.all(finalFiles.map((file) => expectAsync(
+          waitForListObjectsV2ResultCount({
+            bucket: file.bucket,
+            prefix: file.key,
+            desiredCount: 1,
+            interval: 5 * 1000,
+            timeout: 60 * 1000,
+          })
+        ).toBeResolved()));
+      } catch (error) {
+        console.log(`files do not appear to have been moved: error: ${error}`);
+        beforeAllFailed = true;
+      }
+    });
+    it('moves the granule data in s3', () => {
+      if (beforeAllFailed) fail('beforeAllFailed');
+    });
+    it('updates the granule data in cumulus', async () => {
+      if (beforeAllFailed) fail('beforeAllFailed');
+      const cumulusGranule = await granules.getGranule({
+        prefix: stackName,
+        granuleId: processGranule.granuleId,
+      });
+      expect(cumulusGranule.granuleId).toEqual(processGranule.granuleId);
+      expect(cumulusGranule.collectionId).toEqual(constructCollectionId(targetCollection.name, targetCollection.version));
+    });
+  });
+});