Merge pull request #14 from italia/fix/error_jobs

Fix/error jobs
italia · Mar 15, 2023 · e861225 · e861225
2 parents 5988849 + c337b2d
commit e861225
Show file tree

Hide file tree

Showing 5 changed files with 79 additions and 11 deletions.
diff --git a/command/queueManager.ts b/command/queueManager.ts
@@ -18,6 +18,7 @@ import {
   generateJobs,
   getForcedRescanEntitiesToBeAnalyzed,
   getFirstTimeForcedEntityToBeAnalyzed,
+  manageEntitiesInErrorJobs,
 } from "../controller/queueManagerController";
 
 const command = yargs(hideBin(process.argv))
@@ -85,7 +86,10 @@ dbQM
     const inProgressJobInError = await new jobController(
       dbQM
     ).manageInProgressJobInError();
-    console.log('MANAGE JOB IN "ERROR": ', inProgressJobInError.length);
+    console.log(
+      'MANAGE IN PROGRESS JOB IN "ERROR": ',
+      inProgressJobInError.length
+    );
 
     const inPendingJob = await new jobController(dbQM).managePendingJobs(
       crawlerQueue
@@ -94,6 +98,14 @@ dbQM
 
     const manualScanLogic = command.manualScanLogic;
 
+    if (manualScanLogic) {
+      const entityInErrorJob = await manageEntitiesInErrorJobs();
+      console.log(
+        'MANAGE ENTITY WITH LAST JOB IN "ERROR": ',
+        entityInErrorJob.length
+      );
+    }
+
     let gapLimit: number = parseInt(command.maxItems);
 
     let firstTimeEntityToBeAnalyzed = [];

diff --git a/command/scanManager.ts b/command/scanManager.ts
@@ -204,7 +204,6 @@ const uploadFiles = async (
       };
     }
 
-    //TODO: Integrazione completata - In attesa di bucket S3 per testing
     const htmlLocationUrl = await s3Upload(
       htmlReport,
       entityId + "/" + jobId + "/" + "report.html"

diff --git a/controller/PA2026/integrationController.ts b/controller/PA2026/integrationController.ts
@@ -100,6 +100,8 @@ const callPatch = async (body: object, path: string, retry = 3) => {
       return result?.data ?? {};
     } else if (result?.statusCode === 401) {
       await new tokenController(dbWS).create();
+    } else {
+      throw new Error(JSON.stringify(result));
     }
   } catch (e) {
     console.log("CALL PATCH EXCEPTION: ", e.toString());

diff --git a/controller/queueManagerController.ts b/controller/queueManagerController.ts
@@ -1,9 +1,11 @@
-import { dbQM } from "../database/connection";
+import { dbQM, dbWS } from "../database/connection";
 import { QueryTypes } from "sequelize";
 import { Entity } from "../types/models";
 import { define as entityDefine } from "../database/models/entity";
 import { define as jobDefine } from "../database/models/job";
 import dateFormat from "dateformat";
+import { entityController } from "./entityController";
+import { callPatch } from "./PA2026/integrationController";
 
 const getFirstTimeEntityToBeAnalyzed = async (limit: number) => {
   let returnValues = [];
@@ -34,6 +36,58 @@ const getFirstTimeEntityToBeAnalyzed = async (limit: number) => {
   }
 };
 
+const manageEntitiesInErrorJobs = async () => {
+  const returnValues = [];
+
+  try {
+    const querySql = `SELECT J."entity_id"
+            FROM "Jobs" as J
+            WHERE J.status = 'ERROR'
+            AND J.id IN (
+                SELECT MAX(J2.id)
+                FROM "Jobs" as J2
+                GROUP BY J2."entity_id"
+            )
+            `;
+
+    const inErrorEntities = await dbQM.query(querySql, {
+      type: QueryTypes.RAW,
+    });
+
+    for (let i = 0; i < inErrorEntities[0].length; i++) {
+      const entity: Entity | null = await new entityController(
+        dbWS
+      ).retrieveById(inErrorEntities[0][i]["entity_id"]);
+
+      if (!entity) {
+        continue;
+      }
+
+      try {
+        await callPatch(
+          {
+            Da_Scansionare_Data_Scansione__c: new Date().getTime(),
+          },
+          process.env.PA2026_UPDATE_RECORDS_PATH.replace(
+            "{external_entity_id}",
+            entity.external_id
+          )
+        );
+      } catch (e) {
+        console.log("FOR: ENTITY IN ERROR JOBS: ", e.toString());
+        continue;
+      }
+
+      returnValues.push(entity.id);
+    }
+
+    return returnValues;
+  } catch (e) {
+    console.log("ERROR: MANAGE IN ERROR JOB");
+    return returnValues;
+  }
+};
+
 const getFirstTimeForcedEntityToBeAnalyzed = async (limit: number) => {
   let returnValues = [];
 
@@ -250,6 +304,7 @@ const generateJobs = async (
 
 export {
   getFirstTimeEntityToBeAnalyzed,
+  manageEntitiesInErrorJobs,
   getFirstTimeForcedEntityToBeAnalyzed,
   getForcedRescanEntitiesToBeAnalyzed,
   getRescanEntityToBeAnalyzed,

diff --git a/package.json b/package.json
@@ -8,14 +8,14 @@
     "node": ">=18.0.0"
   },
   "scripts": {
-    "dist-webserver": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/server.js",
-    "dist-queue-manager": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/command/queueManager.js",
-    "dist-scan-manager": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/command/scanManager.js",
-    "dist-PA2026-manager": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/command/PA2026Manager.js",
-    "ts-webserver": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./server.ts",
-    "ts-queue-manager": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./command/queueManager.ts",
-    "ts-scan-manager": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./command/scanManager.ts",
-    "ts-PA2026-manager": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./command/PA2026Manager.ts",
+    "dist-webserver": "node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/server.js",
+    "dist-queue-manager": "node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/command/queueManager.js",
+    "dist-scan-manager": "node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/command/scanManager.js",
+    "dist-PA2026-manager": "node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node ./dist/command/PA2026Manager.js",
+    "ts-webserver": "node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./server.ts",
+    "ts-queue-manager": "node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./command/queueManager.ts",
+    "ts-scan-manager": "node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./command/scanManager.ts",
+    "ts-PA2026-manager": "node --max-old-space-size=8192 --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./command/PA2026Manager.ts",
     "migrate": "node --no-warnings --experimental-modules --es-module-specifier-resolution=node  --loader ts-node/esm ./database/migrate.ts --tablename all",
     "build": "tsc && cp -rp ./.env ./dist && cp -rp ./.env.example ./dist && cp -rp ./package.json ./dist",
     "format:check": "prettier -c .",