Merge pull request #13 from italia/evo-release-3-forcedScan

Evo release 3 forced scan

.env.example

@@ -38,6 +38,7 @@ PA2026_HOST=https://test.salesforce.com
 PA2026_AUTH_PATH=/services/oauth2/token
 PA2026_QUERY_PATH=/services/data/v53.0/query
 PA2026_UPDATE_RECORDS_PATH=/services/data/v53.0/sobjects/outfunds__Funding_Request__c/{external_entity_id}
+PA2026_UPLOAD_FILE_PATH=/services/apexrest/ReportCrawler/v1/{external_entity_id}-{scan_number} # {scan_number} = 1 o N
 PA2026_AUTH_GRANT_TYPE=password
 PA2026_AUTH_CLIENT_ID=
 PA2026_AUTH_CLIENT_SECRET=

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-FROM node:16.17.0-buster-slim
+FROM node:18.15.0-buster-slim
 
 ARG GEOIP_LICENSE
 
@@ -16,7 +16,7 @@ RUN apt-get install -y git
 RUN apt-get install -y nano
 RUN apt-get install -y chromium
 
-RUN npm install -g npm@8.19.2
+RUN npm install -g npm@9.5.0
 
 # Bundle app source
 COPY . /usr/src/app/

README.md

@@ -24,7 +24,7 @@ PA Website Validator handler utilizza le seguenti tecnologie
 
 ## Requirements
 
-PA Website Validator necessita [Node.js](https://nodejs.org/it/) v16+ (LTS), [npm], [PostgreSQL] e [Redis].
+PA Website Validator necessita [Node.js](https://nodejs.org/it/) v18+ (LTS), [npm], [PostgreSQL] e [Redis].
 
 ## Plugins
 
@@ -52,7 +52,7 @@ npm run build
 Comando di **creazione coda**:
 
 ```bash
-npm run dist-queue-manager --maxItems <number> --passedOlderThanDays <number> --failedOlderThanDays <number> --asservationOlderThanDays <number>
+npm run dist-queue-manager --maxItems <number> --passedOlderThanDays <number> --failedOlderThanDays <number> --asservationOlderThanDays <number> --manualScanLogic <boolean>
 ```
 
 Mappa opzioni comando
@@ -62,6 +62,7 @@ Mappa opzioni comando
 | - -passedOlderThanDays | Giorni dopo i quali le entity con Job che ha fornito risultato PASSED vengono riaccodate per essere scansionate | ❌ |28
 | - -failedOlderThanDays | Giorni dopo i quali le entity con Job che ha fornito risultato FAILED vengono riaccodate per essere scansionate | ❌ | 14
 | - -asservationOlderThanDays | Giorni dopo i quali le entity asseverate vengono riaccodate per essere scansionate | ❌ | 28
+| - -manualScanLogic | Flag per permettere solo alle entity flaggate come 'da scansionare' su PA2026 di entrare in coda di scansione | ❌ | false
 
 Comando di prelievo Job da coda e **start scansione**:
 

command/PA2026Manager.ts

@@ -12,6 +12,7 @@ import { Entity, Job } from "../types/models";
 import { jobController } from "../controller/jobController";
 import { preserveReasons, define as jobDefine } from "../database/models/job";
 import { Op } from "sequelize";
+import { getFile } from "../controller/s3Controller";
 
 dbRoot
   .authenticate()
@@ -25,6 +26,9 @@ dbRoot
       const updateResult = await update();
       console.log("[PA2026 MANAGER]: UPDATE RESULT - ", updateResult);
 
+      const forcedScanResult = await forcedScanEntities();
+      console.log("[PA2026 MANAGER]: FORCED SCAN RESULT - ", forcedScanResult);
+
       const asseverationResult = await asseveration();
       console.log(
         "[PA2026 MANAGER]: ASSEVERATION RESULT - ",
@@ -45,7 +49,12 @@ dbRoot
 
 const create = async () => {
   const createQuery =
-    "SELECT id,Url_Sito_Internet__c, Pacchetto_1_4_1__c, ID_Crawler__c FROM outfunds__Funding_Request__c  WHERE outfunds__Status__c ='Finanziata' AND outfunds__FundingProgram__r.RecordType.DeveloperName='Misura_141'  AND Url_Sito_Internet__c !=null AND ID_Crawler__c=null";
+    "SELECT id,Url_Sito_Internet__c, Pacchetto_1_4_1__c, ID_Crawler__c " +
+    "FROM outfunds__Funding_Request__c " +
+    "WHERE outfunds__Status__c ='Finanziata' " +
+    "AND outfunds__FundingProgram__r.RecordType.DeveloperName='Misura_141' " +
+    "AND Url_Sito_Internet__c !=null " +
+    "AND ID_Crawler__c=null";
   const returnIds = [];
 
   try {
@@ -79,6 +88,7 @@ const create = async () => {
             enable: true,
             type: type,
             subtype: subtype,
+            forcedScan: false,
           });
 
           if (!entity) {
@@ -117,6 +127,7 @@ const update = async () => {
     "AND Url_Sito_Internet__c !=null " +
     "AND ID_Crawler__c!=null " +
     "AND Controllo_URL__c=false ";
+
   const returnIds = [];
 
   try {
@@ -260,11 +271,14 @@ const asseveration = async () => {
 
         returnIds.push(entityUpdated.id);
       } catch (e) {
-        console.log("UPDATE QUERY FOR-STATEMENT EXCEPTION: ", e.toString());
+        console.log(
+          "ASSEVERATION QUERY FOR-STATEMENT EXCEPTION: ",
+          e.toString()
+        );
       }
     }
   } catch (e) {
-    console.log("UPDATE QUERY EXCEPTION: ", e.toString());
+    console.log("ASSEVERATION QUERY EXCEPTION: ", e.toString());
   }
   return returnIds;
 };
@@ -315,9 +329,93 @@ const sendRetryJobInSendError = async () => {
     });
 
     for (const job of jobs) {
-      await pushResult(job, job.json_result, job.status === "PASSED");
+      try {
+        let s3FilePath = job.s3_html_url;
+        if (!s3FilePath) {
+          continue;
+        }
+
+        if (s3FilePath.startsWith("http")) {
+          s3FilePath = new URL(s3FilePath).pathname;
+        }
+
+        s3FilePath = s3FilePath.substring(1);
+
+        const file: string = await getFile(s3FilePath);
+        if (!file) {
+          continue;
+        }
+
+        await pushResult(job, job.json_result, job.status === "PASSED", file);
+      } catch (e) {
+        console.log("SEND RETRY JOB FOR-STATEMENT EXCEPTION: ", e.toString());
+      }
     }
   } catch (e) {
     console.log("SEND RETRY JOB EXCEPTION: ", e.toString());
   }
 };
+
+const forcedScanEntities = async () => {
+  const forcedScanEntitiesQuery =
+    "SELECT id,Url_Sito_Internet__c, Pacchetto_1_4_1__c, ID_Crawler__c " +
+    "FROM outfunds__Funding_Request__c  " +
+    "WHERE outfunds__Status__c ='Finanziata' " +
+    "AND outfunds__FundingProgram__r.RecordType.DeveloperName='Misura_141'  " +
+    "AND Url_Sito_Internet__c !=null " +
+    "AND Da_Scansionare__c=true";
+
+  const returnIds = [];
+
+  try {
+    const scanFromPA2026Result = await callQuery(forcedScanEntitiesQuery);
+
+    if (!scanFromPA2026Result) {
+      throw new Error("Empty values from create query");
+    }
+
+    const records = scanFromPA2026Result?.records;
+    if (records.length <= 0) {
+      return [];
+    }
+
+    for (const record of records) {
+      try {
+        const externalId = record.Id ?? "";
+        const entity: Entity = await new entityController(dbWS).retrieve(
+          externalId
+        );
+
+        if (!entity) {
+          throw new Error("Entity not found: " + externalId);
+        }
+
+        const updateEntity = await entity.update({
+          forcedScan: true,
+        });
+
+        returnIds.push(updateEntity.id);
+
+        await callPatch(
+          {
+            Da_Scansionare__c: false,
+            Da_Scansionare_Data_Scansione__c: new Date().getTime(),
+          },
+          process.env.PA2026_UPDATE_RECORDS_PATH.replace(
+            "{external_entity_id}",
+            entity.external_id
+          )
+        );
+      } catch (e) {
+        console.log(
+          "FORCED SCAN QUERY FOR-STATEMENT EXCEPTION: ",
+          e.toString()
+        );
+      }
+    }
+  } catch (e) {
+    console.log("FORCED SCAN QUERY EXCEPTION: ", e.toString());
+  }
+
+  return returnIds;
+};

command/queueManager.ts

@@ -16,11 +16,17 @@ import {
   getRescanEntityToBeAnalyzed,
   getRescanEntityAsseveratedToBeAnalyzed,
   generateJobs,
+  getForcedRescanEntitiesToBeAnalyzed,
+  getFirstTimeForcedEntityToBeAnalyzed,
 } from "../controller/queueManagerController";
 
 const command = yargs(hideBin(process.argv))
   .usage(
-    "Usage: --maxItems <maxItems> --passedOlderThanDays <passedOlderThanDays> --failedOlderThanDays <failedOlderThanDays>"
+    "Usage: " +
+      "--maxItems <maxItems> " +
+      "--passedOlderThanDays <passedOlderThanDays> " +
+      "--failedOlderThanDays <failedOlderThanDays> " +
+      "--manualScanLogic"
   )
   .option("maxItems", {
     describe: "Numero massimo di entity da analizzare",
@@ -48,6 +54,13 @@ const command = yargs(hideBin(process.argv))
     type: "integer",
     demandOption: true,
     default: 28,
+  })
+  .option("manualScanLogic", {
+    describe:
+      "Flag per permettere solo alle entity flaggate come 'da scansionare' di entrare in coda di scansione",
+    type: "bool",
+    demandOption: true,
+    default: false,
   }).argv;
 
 dbQM
@@ -79,35 +92,64 @@ dbQM
     );
     console.log('MANAGE JOB IN "PENDING": ', inPendingJob.length);
 
-    const maxItems: number = parseInt(command.maxItems);
+    const manualScanLogic = command.manualScanLogic;
 
-    const firstTimeEntityToBeAnalyzed = await getFirstTimeEntityToBeAnalyzed(
-      maxItems
-    );
+    let gapLimit: number = parseInt(command.maxItems);
+
+    let firstTimeEntityToBeAnalyzed = [];
+    if (command.passedOlderThanDays > 0 || command.failedOlderThanDays) {
+      firstTimeEntityToBeAnalyzed = await getFirstTimeEntityToBeAnalyzed(
+        gapLimit
+      );
+
+      gapLimit = gapLimit - firstTimeEntityToBeAnalyzed.length;
+    }
+
+    let firstTimeForcedEntityToBeAnalyzed = [];
+    if (gapLimit > 0 && manualScanLogic) {
+      firstTimeForcedEntityToBeAnalyzed =
+        await getFirstTimeForcedEntityToBeAnalyzed(gapLimit);
+
+      gapLimit = gapLimit - firstTimeForcedEntityToBeAnalyzed.length;
+    }
+
+    let forcedRescanEntitiesToBeAnalyzed = [];
+    if (gapLimit > 0 && manualScanLogic) {
+      forcedRescanEntitiesToBeAnalyzed =
+        await getForcedRescanEntitiesToBeAnalyzed(gapLimit);
+
+      gapLimit = gapLimit - forcedRescanEntitiesToBeAnalyzed.length;
+    }
 
     let rescanEntityToBeAnalyzed = [];
-    let gapLimit: number = maxItems - firstTimeEntityToBeAnalyzed.length;
-    if (gapLimit > 0) {
+    if (
+      gapLimit > 0 &&
+      (command.passedOlderThanDays > 0 || command.failedOlderThanDays)
+    ) {
       rescanEntityToBeAnalyzed = await getRescanEntityToBeAnalyzed(
         command.passedOlderThanDays,
         command.failedOlderThanDays,
         gapLimit
       );
+
+      gapLimit = gapLimit - rescanEntityToBeAnalyzed.length;
     }
 
     let rescanEntityAsseveratedToBeAnalyzed = [];
-    gapLimit = gapLimit - rescanEntityToBeAnalyzed.length;
-    if (gapLimit > 0) {
+    if (gapLimit > 0 && command.asservationOlderThanDays > 0) {
       rescanEntityAsseveratedToBeAnalyzed =
         await getRescanEntityAsseveratedToBeAnalyzed(
           command.asservationOlderThanDays,
           gapLimit
         );
     }
 
-    if (firstTimeEntityToBeAnalyzed.length > 0) {
+    if (
+      firstTimeEntityToBeAnalyzed.length > 0 ||
+      firstTimeForcedEntityToBeAnalyzed.length > 0
+    ) {
       await generateJobs(
-        firstTimeEntityToBeAnalyzed,
+        [...firstTimeEntityToBeAnalyzed, ...firstTimeForcedEntityToBeAnalyzed],
         crawlerQueue,
         true,
         preserveReasons[0]
@@ -125,16 +167,31 @@ dbQM
 
     if (
       rescanEntityToBeAnalyzed.length > 0 ||
-      rescanEntityAsseveratedToBeAnalyzed.length > 0
+      rescanEntityAsseveratedToBeAnalyzed.length > 0 ||
+      forcedRescanEntitiesToBeAnalyzed.length > 0
     ) {
       await generateJobs(
-        [...rescanEntityToBeAnalyzed, ...rescanEntityAsseveratedToBeAnalyzed],
+        [
+          ...rescanEntityToBeAnalyzed,
+          ...rescanEntityAsseveratedToBeAnalyzed,
+          ...forcedRescanEntitiesToBeAnalyzed,
+        ],
         crawlerQueue,
         false
       );
     }
 
     console.log("FIRST TIME ENTITIES", firstTimeEntityToBeAnalyzed.length);
+
+    console.log(
+      "FIRST TIME FORCED ENTITIES",
+      firstTimeForcedEntityToBeAnalyzed.length
+    );
+
+    console.log(
+      "FORCED RESCAN ENTITIES",
+      forcedRescanEntitiesToBeAnalyzed.length
+    );
     console.log(
       "RESCAN ENTITIES TO BE ANALYZED",
       rescanEntityToBeAnalyzed.length