From cfed926c6701bb137c33df4839ad0dd5105a31b5 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Fri, 16 Dec 2022 16:57:24 +0100 Subject: [PATCH 01/15] feat: add crawler version in pa2026 body --- utils/utils.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/utils.ts b/utils/utils.ts index 4b026fc..ee60c7f 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -1,8 +1,11 @@ "use strict"; +import { readFileSync } from "fs"; import { ValidationError } from "jsonschema"; import { Job } from "../types/models"; import { auditDictionary } from "pa-website-validator/dist/storage/auditDictionary"; +const packageJSON = JSON.parse(readFileSync('../package.json').toString()) ?? {}; + const arrayChunkify = async ( inputArray: [], numberOfDesiredChuck: number, @@ -92,6 +95,8 @@ const mapPA2026Body = async ( const key = isFirstScan ? "1" : "n"; const initialBody = []; + initialBody[`Versione_Crawler_${key}__c`] = + packageJSON?.dependencies["pa-website-validator"]?.split("#")[1] ?? ""; initialBody[`Status_Generale_${key}__c`] = generalStatus; initialBody[`Data_Job_Crawler_${key}__c`] = new Date(job.end_at).getTime(); (initialBody[`URL_Scansione_${key}__c`] = job.scan_url), From 0cddb12c25cb2f366f5ec4b9bc340b71a3780a6e Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Fri, 16 Dec 2022 17:36:27 +0100 Subject: [PATCH 02/15] feat: add passed audit percentage calculation & field in pa2026 body --- controller/PA2026/integrationController.ts | 16 ++++-- utils/utils.ts | 62 ++++++++++++++++++++-- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/controller/PA2026/integrationController.ts b/controller/PA2026/integrationController.ts index 2d9ea84..712a117 100644 --- a/controller/PA2026/integrationController.ts +++ b/controller/PA2026/integrationController.ts @@ -14,7 +14,10 @@ import { define as jobDefine, preserveReasons, } from "../../database/models/job"; -import { mapPA2026Body } from "../../utils/utils"; +import { + calculatePassedAuditPercentage, + mapPA2026Body, +} from "../../utils/utils"; const retrieveToken = async () => { try { @@ -115,11 +118,17 @@ const pushResult = async ( const isFirstScan = job.preserve && job.preserve_reason === preserveReasons[0]; + const passedAuditsPercentage = await calculatePassedAuditPercentage( + job, + cleanJsonReport + ); + let scanBody = await mapPA2026Body( job, cleanJsonReport, generalStatus, - false + false, + passedAuditsPercentage ); if (isFirstScan) { @@ -127,7 +136,8 @@ const pushResult = async ( job, cleanJsonReport, generalStatus, - true + true, + passedAuditsPercentage ); scanBody = { diff --git a/utils/utils.ts b/utils/utils.ts index ee60c7f..cdfef97 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -4,7 +4,8 @@ import { ValidationError } from "jsonschema"; import { Job } from "../types/models"; import { auditDictionary } from "pa-website-validator/dist/storage/auditDictionary"; -const packageJSON = JSON.parse(readFileSync('../package.json').toString()) ?? {}; +const packageJSON = + JSON.parse(readFileSync("../package.json").toString()) ?? {}; const arrayChunkify = async ( inputArray: [], @@ -77,7 +78,8 @@ const mapPA2026Body = async ( job: Job, cleanJsonResult, generalStatus: boolean, - isFirstScan: boolean + isFirstScan: boolean, + passedAuditsPercentage: string ) => { try { const mainObjKey = @@ -97,6 +99,8 @@ const mapPA2026Body = async ( const initialBody = []; initialBody[`Versione_Crawler_${key}__c`] = packageJSON?.dependencies["pa-website-validator"]?.split("#")[1] ?? ""; + initialBody[`Criteri_Superati_Crawler_${key}__c`] = + passedAuditsPercentage ?? 0; initialBody[`Status_Generale_${key}__c`] = generalStatus; initialBody[`Data_Job_Crawler_${key}__c`] = new Date(job.end_at).getTime(); (initialBody[`URL_Scansione_${key}__c`] = job.scan_url), @@ -153,4 +157,56 @@ const mapPA2026Body = async ( } }; -export { arrayChunkify, mapValidationErrors, mapPA2026Body }; +const calculatePassedAuditPercentage = async ( + job: Job, + cleanJsonResult +): Promise => { + let passed = 0; + let total = 0; + let totalAudits = {}; + + const mainObjKey = + job.type === "municipality" ? "cittadino-informato" : "criteri-conformita"; + + const raccomandationAuditsObj = + cleanJsonResult["raccomandazioni"]["audits"] ?? {}; + const legislationAudits = + cleanJsonResult[mainObjKey]["groups"]["normativa"]["audits"] ?? {}; + const securityAudits = + cleanJsonResult[mainObjKey]["groups"]["sicurezza"]["audits"] ?? {}; + const userExperienceAudits = + cleanJsonResult[mainObjKey]["groups"]["esperienza-utente"]["audits"] ?? {}; + totalAudits = { + ...raccomandationAuditsObj, + ...legislationAudits, + ...securityAudits, + ...userExperienceAudits, + }; + + if (job.type === "municipality") { + const functionalityAudits = + cleanJsonResult[mainObjKey]["groups"]["funzionalita"]["audits"] ?? {}; + totalAudits = { ...totalAudits, ...functionalityAudits }; + } + + for (const auditResult of Object.values(totalAudits)) { + total++; + + if (auditResult > 0) { + passed++; + } + } + + if (total > 0) { + return (passed / total).toFixed(2); + } + + return "0"; +}; + +export { + arrayChunkify, + mapValidationErrors, + mapPA2026Body, + calculatePassedAuditPercentage, +}; From b848ce93959dd003c7686d9c1db5caa6d948e521 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Sat, 17 Dec 2022 17:06:40 +0100 Subject: [PATCH 03/15] feat: push result to pa2026 handling if scan url does not exist --- command/scanManager.ts | 16 +++++++-- controller/PA2026/integrationController.ts | 42 +++++++++++++++++++++- utils/utils.ts | 28 +++++++++++++++ 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/command/scanManager.ts b/command/scanManager.ts index bd7f434..c9c238c 100644 --- a/command/scanManager.ts +++ b/command/scanManager.ts @@ -21,7 +21,11 @@ import { isPassedReport, } from "../controller/auditController"; import { jobController } from "../controller/jobController"; -import { pushResult } from "../controller/PA2026/integrationController"; +import { + pushResult, + pushResultUrlNotExists, +} from "../controller/PA2026/integrationController"; +import { urlExists } from "../utils/utils"; dbSM .authenticate() @@ -76,8 +80,16 @@ const scan = async (jobId) => { }); const jobObjParsed = jobObj.toJSON(); + const urlToBeScanned = jobObjParsed.scan_url; + + const urlToBeScannedExists = await urlExists(urlToBeScanned); + if (!urlToBeScannedExists) { + await pushResultUrlNotExists(jobObj); + throw new Error("Scan URL does not exists"); + } + const lighthouseResult = await run( - jobObjParsed.scan_url, + urlToBeScanned, jobObjParsed.type, "online", logLevels.display_none, diff --git a/controller/PA2026/integrationController.ts b/controller/PA2026/integrationController.ts index 712a117..dbdc36d 100644 --- a/controller/PA2026/integrationController.ts +++ b/controller/PA2026/integrationController.ts @@ -17,6 +17,7 @@ import { import { calculatePassedAuditPercentage, mapPA2026Body, + mapPA2026BodyUrlNotExists, } from "../../utils/utils"; const retrieveToken = async () => { @@ -183,4 +184,43 @@ const pushResult = async ( } }; -export { retrieveToken, callQuery, callPatch, pushResult }; +const pushResultUrlNotExists = async (job: Job) => { + try { + const entity = await new entityController(dbSM).retrieveByPk(job.entity_id); + + const scanBody = await mapPA2026BodyUrlNotExists(); + + const result = await callPatch( + scanBody, + process.env.PA2026_UPDATE_RECORDS_PATH.replace( + "{external_entity_id}", + entity.external_id + ) + ); + + //Warn: API returns empty string when it success + if (result !== "") { + throw new Error("Send data failed"); + } + + await job.update({ + data_sent_status: "COMPLETED", + data_sent_date: new Date(), + }); + } catch (e) { + console.log("PUSH RESULT EXCEPTION", e.toString()); + + await job.update({ + data_sent_status: "ERROR", + data_sent_date: new Date(), + }); + } +}; + +export { + retrieveToken, + callQuery, + callPatch, + pushResult, + pushResultUrlNotExists, +}; diff --git a/utils/utils.ts b/utils/utils.ts index cdfef97..ea2c557 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -3,6 +3,7 @@ import { readFileSync } from "fs"; import { ValidationError } from "jsonschema"; import { Job } from "../types/models"; import { auditDictionary } from "pa-website-validator/dist/storage/auditDictionary"; +import axios from "axios"; const packageJSON = JSON.parse(readFileSync("../package.json").toString()) ?? {}; @@ -157,6 +158,13 @@ const mapPA2026Body = async ( } }; +const mapPA2026BodyUrlNotExists = async () => { + const body = []; + body[`Data_scansione_fallita__c`] = Date.now().toString(); + + return Object.assign({}, body); +}; + const calculatePassedAuditPercentage = async ( job: Job, cleanJsonResult @@ -204,9 +212,29 @@ const calculatePassedAuditPercentage = async ( return "0"; }; +const urlExists = async (url) => { + try { + let statusCode = undefined; + const response = await axios.get(url); + statusCode = response.status; + + if (statusCode === undefined || statusCode < 200 || statusCode >= 400) { + return false; + } + + return true; + } catch (ex) { + console.log("Url Exists Exception: ", ex); + + return false; + } +}; + export { arrayChunkify, mapValidationErrors, mapPA2026Body, calculatePassedAuditPercentage, + urlExists, + mapPA2026BodyUrlNotExists, }; From fee17d5341e489e67f6cee7c5182cdff368e298a Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Mon, 19 Dec 2022 10:52:35 +0100 Subject: [PATCH 04/15] feat: initial integration --- command/queueManager.ts | 9 +++---- command/scanManager.ts | 7 ++---- controller/PA2026/integrationController.ts | 7 ++++++ utils/utils.ts | 28 ++++++++++++++++------ 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/command/queueManager.ts b/command/queueManager.ts index 0a0676b..446738e 100644 --- a/command/queueManager.ts +++ b/command/queueManager.ts @@ -56,17 +56,14 @@ dbQM console.log("[QUEUE MANAGER]: start"); const crawlerQueue: Queue = new Queue("crawler-queue", { - connection: new Redis.Cluster([ - { + connection: { host: process.env.REDIS_HOST, port: parseInt(process.env.REDIS_PORT), - }, - ]), + }, defaultJobOptions: { removeOnComplete: true, removeOnFail: true, - }, - prefix: "{1}", + } }); const inProgressJobInError = await new jobController( diff --git a/command/scanManager.ts b/command/scanManager.ts index c9c238c..230165d 100644 --- a/command/scanManager.ts +++ b/command/scanManager.ts @@ -34,13 +34,10 @@ dbSM const worker: Worker = new Worker("crawler-queue", null, { lockDuration: 10000000, - connection: new Redis.Cluster([ - { + connection: { host: process.env.REDIS_HOST, port: parseInt(process.env.REDIS_PORT), - }, - ]), - prefix: "{1}", + }, }); const token = v4(); let job: bullJob; diff --git a/controller/PA2026/integrationController.ts b/controller/PA2026/integrationController.ts index dbdc36d..03caf9d 100644 --- a/controller/PA2026/integrationController.ts +++ b/controller/PA2026/integrationController.ts @@ -157,6 +157,7 @@ const pushResult = async ( } } +/* const result = await callPatch( scanBody, process.env.PA2026_UPDATE_RECORDS_PATH.replace( @@ -169,6 +170,9 @@ const pushResult = async ( if (result !== "") { throw new Error("Send data failed"); } +*/ + + console.log('SCAN BODY: ', scanBody) await job.update({ data_sent_status: "COMPLETED", @@ -190,6 +194,7 @@ const pushResultUrlNotExists = async (job: Job) => { const scanBody = await mapPA2026BodyUrlNotExists(); +/* const result = await callPatch( scanBody, process.env.PA2026_UPDATE_RECORDS_PATH.replace( @@ -202,6 +207,8 @@ const pushResultUrlNotExists = async (job: Job) => { if (result !== "") { throw new Error("Send data failed"); } +*/ + console.log('PUSH RESULT URL NOT EXISTS SCAN BODY: ', scanBody) await job.update({ data_sent_status: "COMPLETED", diff --git a/utils/utils.ts b/utils/utils.ts index ea2c557..ac66c41 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -1,12 +1,17 @@ "use strict"; + +import { dirname } from "path"; import { readFileSync } from "fs"; import { ValidationError } from "jsonschema"; import { Job } from "../types/models"; import { auditDictionary } from "pa-website-validator/dist/storage/auditDictionary"; import axios from "axios"; +import path from "path"; +import {fileURLToPath} from "url"; -const packageJSON = - JSON.parse(readFileSync("../package.json").toString()) ?? {}; +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +const __dirname = dirname(fileURLToPath(import.meta.url)); const arrayChunkify = async ( inputArray: [], @@ -97,11 +102,20 @@ const mapPA2026Body = async ( const key = isFirstScan ? "1" : "n"; + let packageJSON + try { + packageJSON = + JSON.parse(await readFileSync(path.resolve(__dirname, '../package.json')).toString()) ?? {}; + } catch (e) { + packageJSON = null + console.log("MAP PA2026 BODY EXCEPTION 01: ", e) + } + const initialBody = []; initialBody[`Versione_Crawler_${key}__c`] = packageJSON?.dependencies["pa-website-validator"]?.split("#")[1] ?? ""; initialBody[`Criteri_Superati_Crawler_${key}__c`] = - passedAuditsPercentage ?? 0; + passedAuditsPercentage + "%" ?? "0%"; initialBody[`Status_Generale_${key}__c`] = generalStatus; initialBody[`Data_Job_Crawler_${key}__c`] = new Date(job.end_at).getTime(); (initialBody[`URL_Scansione_${key}__c`] = job.scan_url), @@ -154,13 +168,13 @@ const mapPA2026Body = async ( return Object.assign({}, initialBody); } catch (e) { - console.log("MAP PA2026 BODY EXCEPTION: ", e.toString()); + console.log("MAP PA2026 BODY EXCEPTION 02: ", e.toString()); } }; const mapPA2026BodyUrlNotExists = async () => { const body = []; - body[`Data_scansione_fallita__c`] = Date.now().toString(); + body[`Data_scansione_fallita__c`] = (new Date()).toISOString().split('T')[0]; return Object.assign({}, body); }; @@ -206,7 +220,7 @@ const calculatePassedAuditPercentage = async ( } if (total > 0) { - return (passed / total).toFixed(2); + return ((passed / total) * 100).toFixed(0); } return "0"; @@ -224,7 +238,7 @@ const urlExists = async (url) => { return true; } catch (ex) { - console.log("Url Exists Exception: ", ex); + console.log("Url Exists Exception: ", ex.toString()); return false; } From 3b2728f1052529aeb3a2846b5df60bef5ab20cdd Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Mon, 19 Dec 2022 11:22:02 +0100 Subject: [PATCH 05/15] fix: removed old comments --- command/queueManager.ts | 6 +++--- command/scanManager.ts | 4 ++-- controller/PA2026/integrationController.ts | 7 +------ controller/auditController.ts | 19 ------------------- database/models/job.ts | 7 +------ storage/municipalityAudits.ts | 4 ---- utils/utils.ts | 22 ++++++++++------------ 7 files changed, 17 insertions(+), 52 deletions(-) diff --git a/command/queueManager.ts b/command/queueManager.ts index 446738e..806fe3b 100644 --- a/command/queueManager.ts +++ b/command/queueManager.ts @@ -57,13 +57,13 @@ dbQM const crawlerQueue: Queue = new Queue("crawler-queue", { connection: { - host: process.env.REDIS_HOST, - port: parseInt(process.env.REDIS_PORT), + host: process.env.REDIS_HOST, + port: parseInt(process.env.REDIS_PORT), }, defaultJobOptions: { removeOnComplete: true, removeOnFail: true, - } + }, }); const inProgressJobInError = await new jobController( diff --git a/command/scanManager.ts b/command/scanManager.ts index 230165d..2852125 100644 --- a/command/scanManager.ts +++ b/command/scanManager.ts @@ -35,8 +35,8 @@ dbSM const worker: Worker = new Worker("crawler-queue", null, { lockDuration: 10000000, connection: { - host: process.env.REDIS_HOST, - port: parseInt(process.env.REDIS_PORT), + host: process.env.REDIS_HOST, + port: parseInt(process.env.REDIS_PORT), }, }); const token = v4(); diff --git a/controller/PA2026/integrationController.ts b/controller/PA2026/integrationController.ts index 03caf9d..e100789 100644 --- a/controller/PA2026/integrationController.ts +++ b/controller/PA2026/integrationController.ts @@ -157,7 +157,6 @@ const pushResult = async ( } } -/* const result = await callPatch( scanBody, process.env.PA2026_UPDATE_RECORDS_PATH.replace( @@ -170,9 +169,8 @@ const pushResult = async ( if (result !== "") { throw new Error("Send data failed"); } -*/ - console.log('SCAN BODY: ', scanBody) + console.log("SCAN BODY: ", scanBody); await job.update({ data_sent_status: "COMPLETED", @@ -194,7 +192,6 @@ const pushResultUrlNotExists = async (job: Job) => { const scanBody = await mapPA2026BodyUrlNotExists(); -/* const result = await callPatch( scanBody, process.env.PA2026_UPDATE_RECORDS_PATH.replace( @@ -207,8 +204,6 @@ const pushResultUrlNotExists = async (job: Job) => { if (result !== "") { throw new Error("Send data failed"); } -*/ - console.log('PUSH RESULT URL NOT EXISTS SCAN BODY: ', scanBody) await job.update({ data_sent_status: "COMPLETED", diff --git a/controller/auditController.ts b/controller/auditController.ts index eaf373b..eb328fb 100644 --- a/controller/auditController.ts +++ b/controller/auditController.ts @@ -82,19 +82,6 @@ const cleanMunicipalityJSONReport = async (jsonResult: string) => { informedCitizenStatus = true; } - //const activeCitizenAudits = await getAuditByClusterGroup( - // parsedResult, - // municipalityAudits, - // "active-citizen" - //); - //let activeCitizenStatus = false; - //if ( - // Object.keys(activeCitizenAudits.passed).length > 0 && - // Object.keys(activeCitizenAudits.failed).length === 0 - //) { - // activeCitizenStatus = true; - //} - const recommendationsAudits = await getAuditByClusterGroup( parsedResult, municipalityAudits, @@ -138,11 +125,6 @@ const cleanMunicipalityJSONReport = async (jsonResult: string) => { }, }, - //"cittadino-attivo": { - // status: activeCitizenStatus, - // audits: { ...activeCitizenAudits.passed, ...activeCitizenAudits.failed }, - //}, - raccomandazioni: { status: recommendationsStatus, audits: { @@ -349,7 +331,6 @@ const isPassedReport = async ( } else if (subtype === allowedMunicipalitySubTypes[1]) { // eslint-disable-next-line passed = jsonReport["cittadino-informato"].status; - //&& jsonReport["cittadino-attivo"].status; } else { passed = false; } diff --git a/database/models/job.ts b/database/models/job.ts index 0cd3c1b..436f26b 100644 --- a/database/models/job.ts +++ b/database/models/job.ts @@ -145,12 +145,7 @@ const options = { Sequelize.literal( "((json_result->'raccomandazioni'->'audits'->>'municipality-metatag')::float)" ), - //Sequelize.literal( - // "((json_result->'cittadino-attivo'->'audits'->>'municipality-subdomain')::float)" - //), - //Sequelize.literal( - // "((json_result->'cittadino-attivo'->'audits'->>'municipality-personal-area-security')::float)" - //), + Sequelize.literal( "((json_result->'cittadino-informato'->'groups'->'normativa'->'audits'->>'municipality-legislation-privacy-is-present')::float)" ), diff --git a/storage/municipalityAudits.ts b/storage/municipalityAudits.ts index 4b322bf..d7758fb 100644 --- a/storage/municipalityAudits.ts +++ b/storage/municipalityAudits.ts @@ -23,9 +23,5 @@ export const audits = { ], security: ["municipality-security", "municipality-domain"], }, - //"active-citizen": [ - // "municipality-personal-area-security", - // "municipality-subdomain", - //], recommendations: ["municipality-metatag"], }; diff --git a/utils/utils.ts b/utils/utils.ts index ac66c41..c9690da 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -7,7 +7,7 @@ import { Job } from "../types/models"; import { auditDictionary } from "pa-website-validator/dist/storage/auditDictionary"; import axios from "axios"; import path from "path"; -import {fileURLToPath} from "url"; +import { fileURLToPath } from "url"; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore @@ -102,13 +102,17 @@ const mapPA2026Body = async ( const key = isFirstScan ? "1" : "n"; - let packageJSON + let packageJSON; try { packageJSON = - JSON.parse(await readFileSync(path.resolve(__dirname, '../package.json')).toString()) ?? {}; + JSON.parse( + await readFileSync( + path.resolve(__dirname, "../package.json") + ).toString() + ) ?? {}; } catch (e) { - packageJSON = null - console.log("MAP PA2026 BODY EXCEPTION 01: ", e) + packageJSON = null; + console.log("MAP PA2026 BODY EXCEPTION 01: ", e); } const initialBody = []; @@ -148,13 +152,7 @@ const mapPA2026Body = async ( const functionObj = cleanJsonResult[mainObjKey].groups["funzionalita"]; (initialBody[`Cittadino_Informato_${key}__c`] = cleanJsonResult[mainObjKey].status), - //(initialBody[`Cittadino_Attivo_${key}__c`] = - // cleanJsonResult["cittadino-attivo"].status), (initialBody[`Funzionalita_${key}__c`] = functionObj.status), - //(initialBody[`Cittadino_Attivo_${key}_Descrizione__c`] = - // getFailAudits(cleanJsonResult["cittadino-attivo"].audits) - // .map((x) => mapAuditTitle(x)) - // .join(" | ") ?? ""), (initialBody[`Funzionalita_${key}_Descrizione__c`] = getFailAudits(functionObj.audits) .map((x) => mapAuditTitle(x)) @@ -174,7 +172,7 @@ const mapPA2026Body = async ( const mapPA2026BodyUrlNotExists = async () => { const body = []; - body[`Data_scansione_fallita__c`] = (new Date()).toISOString().split('T')[0]; + body[`Data_scansione_fallita__c`] = new Date().toISOString().split("T")[0]; return Object.assign({}, body); }; From 8eddcc1761c91660cb3c6af2bb9849c8e2427150 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Mon, 19 Dec 2022 12:22:25 +0100 Subject: [PATCH 06/15] fix: moved queue manager console log to correct position --- command/queueManager.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/command/queueManager.ts b/command/queueManager.ts index 806fe3b..d603939 100644 --- a/command/queueManager.ts +++ b/command/queueManager.ts @@ -102,16 +102,6 @@ dbQM ); } - console.log("FIRST TIME ENTITIES", firstTimeEntityToBeAnalyzed.length); - console.log( - "RESCAN ENTITIES TO BE ANALYZED", - rescanEntityToBeAnalyzed.length - ); - console.log( - "RESCAN ASSEVERATED ENTITIES", - rescanEntityAsseveratedToBeAnalyzed.length - ); - if (firstTimeEntityToBeAnalyzed.length > 0) { await generateJobs( firstTimeEntityToBeAnalyzed, @@ -141,6 +131,16 @@ dbQM ); } + console.log("FIRST TIME ENTITIES", firstTimeEntityToBeAnalyzed.length); + console.log( + "RESCAN ENTITIES TO BE ANALYZED", + rescanEntityToBeAnalyzed.length + ); + console.log( + "RESCAN ASSEVERATED ENTITIES", + rescanEntityAsseveratedToBeAnalyzed.length + ); + const counts = await crawlerQueue.getJobCounts( "wait", "completed", From c7eb8255109b774a310e3148af92b12b38fc8271 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Mon, 19 Dec 2022 15:35:16 +0100 Subject: [PATCH 07/15] fix: removed unused console logs --- controller/PA2026/integrationController.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/controller/PA2026/integrationController.ts b/controller/PA2026/integrationController.ts index e100789..dbdc36d 100644 --- a/controller/PA2026/integrationController.ts +++ b/controller/PA2026/integrationController.ts @@ -170,8 +170,6 @@ const pushResult = async ( throw new Error("Send data failed"); } - console.log("SCAN BODY: ", scanBody); - await job.update({ data_sent_status: "COMPLETED", data_sent_date: new Date(), From d6b2285a0fba6983795bbeccfaa01a6135b35b96 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Tue, 20 Dec 2022 17:29:27 +0100 Subject: [PATCH 08/15] fix: aggiornamento calcolo audit passati su totali --- command/queueManager.ts | 1 + command/scanManager.ts | 1 + utils/utils.ts | 19 ++++++++++++++----- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/command/queueManager.ts b/command/queueManager.ts index d603939..121189c 100644 --- a/command/queueManager.ts +++ b/command/queueManager.ts @@ -55,6 +55,7 @@ dbQM .then(async () => { console.log("[QUEUE MANAGER]: start"); + //TODO: re-integrare Redis-Cluster const crawlerQueue: Queue = new Queue("crawler-queue", { connection: { host: process.env.REDIS_HOST, diff --git a/command/scanManager.ts b/command/scanManager.ts index 2852125..7c1fb60 100644 --- a/command/scanManager.ts +++ b/command/scanManager.ts @@ -32,6 +32,7 @@ dbSM .then(async () => { console.log("[SCAN MANAGER]: start"); + //TODO: re-integrare Redis-Cluster const worker: Worker = new Worker("crawler-queue", null, { lockDuration: 10000000, connection: { diff --git a/utils/utils.ts b/utils/utils.ts index c9690da..96823e8 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -118,8 +118,7 @@ const mapPA2026Body = async ( const initialBody = []; initialBody[`Versione_Crawler_${key}__c`] = packageJSON?.dependencies["pa-website-validator"]?.split("#")[1] ?? ""; - initialBody[`Criteri_Superati_Crawler_${key}__c`] = - passedAuditsPercentage + "%" ?? "0%"; + initialBody[`Criteri_Superati_Crawler_${key}__c`] = passedAuditsPercentage; initialBody[`Status_Generale_${key}__c`] = generalStatus; initialBody[`Data_Job_Crawler_${key}__c`] = new Date(job.end_at).getTime(); (initialBody[`URL_Scansione_${key}__c`] = job.scan_url), @@ -206,7 +205,17 @@ const calculatePassedAuditPercentage = async ( if (job.type === "municipality") { const functionalityAudits = cleanJsonResult[mainObjKey]["groups"]["funzionalita"]["audits"] ?? {}; - totalAudits = { ...totalAudits, ...functionalityAudits }; + const performancesResult = + cleanJsonResult[mainObjKey]["groups"]["prestazioni"]["status"] ?? 0; + const performancesAudits = { + "municipality-status": performancesResult === true ? 1 : 0, + }; + + totalAudits = { + ...totalAudits, + ...functionalityAudits, + ...performancesAudits, + }; } for (const auditResult of Object.values(totalAudits)) { @@ -218,10 +227,10 @@ const calculatePassedAuditPercentage = async ( } if (total > 0) { - return ((passed / total) * 100).toFixed(0); + return passed + " su " + total; } - return "0"; + return "0" + " su " + total; }; const urlExists = async (url) => { From 720f6d693f9e9a5d747f4b1c45a2d84f7d09527d Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Tue, 20 Dec 2022 17:30:39 +0100 Subject: [PATCH 09/15] fix: ripristino redis-cluster su queueManager --- command/queueManager.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/command/queueManager.ts b/command/queueManager.ts index 121189c..4fc4c71 100644 --- a/command/queueManager.ts +++ b/command/queueManager.ts @@ -55,16 +55,18 @@ dbQM .then(async () => { console.log("[QUEUE MANAGER]: start"); - //TODO: re-integrare Redis-Cluster const crawlerQueue: Queue = new Queue("crawler-queue", { - connection: { - host: process.env.REDIS_HOST, - port: parseInt(process.env.REDIS_PORT), - }, + connection: new Redis.Cluster([ + { + host: process.env.REDIS_HOST, + port: parseInt(process.env.REDIS_PORT), + }, + ]), defaultJobOptions: { removeOnComplete: true, removeOnFail: true, }, + prefix: "{1}", }); const inProgressJobInError = await new jobController( From 010bf957d8d254be57193acba5d61005838c726d Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Tue, 20 Dec 2022 17:31:26 +0100 Subject: [PATCH 10/15] fix: ripristino redis-cluster su scanManager --- command/scanManager.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/command/scanManager.ts b/command/scanManager.ts index 7c1fb60..c9c238c 100644 --- a/command/scanManager.ts +++ b/command/scanManager.ts @@ -32,13 +32,15 @@ dbSM .then(async () => { console.log("[SCAN MANAGER]: start"); - //TODO: re-integrare Redis-Cluster const worker: Worker = new Worker("crawler-queue", null, { lockDuration: 10000000, - connection: { - host: process.env.REDIS_HOST, - port: parseInt(process.env.REDIS_PORT), - }, + connection: new Redis.Cluster([ + { + host: process.env.REDIS_HOST, + port: parseInt(process.env.REDIS_PORT), + }, + ]), + prefix: "{1}", }); const token = v4(); let job: bullJob; From 4d14c9662d66aed3ec8db54e7374dbfe3b73d4f2 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Wed, 21 Dec 2022 10:16:05 +0100 Subject: [PATCH 11/15] fix: rimozione audit sezione Prestazioni da calcolo audit passati --- utils/utils.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/utils.ts b/utils/utils.ts index 96823e8..035f99b 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -187,8 +187,6 @@ const calculatePassedAuditPercentage = async ( const mainObjKey = job.type === "municipality" ? "cittadino-informato" : "criteri-conformita"; - const raccomandationAuditsObj = - cleanJsonResult["raccomandazioni"]["audits"] ?? {}; const legislationAudits = cleanJsonResult[mainObjKey]["groups"]["normativa"]["audits"] ?? {}; const securityAudits = @@ -196,7 +194,6 @@ const calculatePassedAuditPercentage = async ( const userExperienceAudits = cleanJsonResult[mainObjKey]["groups"]["esperienza-utente"]["audits"] ?? {}; totalAudits = { - ...raccomandationAuditsObj, ...legislationAudits, ...securityAudits, ...userExperienceAudits, From 989e3b42490387288270f0337916dd1de58f6861 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Wed, 21 Dec 2022 15:17:08 +0100 Subject: [PATCH 12/15] =?UTF-8?q?fix:=20aggiornamento=20calcolo=20status?= =?UTF-8?q?=20generale=20criteri-conformit=C3=A0=20scuole?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- controller/auditController.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/controller/auditController.ts b/controller/auditController.ts index eb328fb..d7d74e9 100644 --- a/controller/auditController.ts +++ b/controller/auditController.ts @@ -187,12 +187,7 @@ const cleanSchoolJSONReport = async (jsonResult: string) => { } let complianceCriteriaStatus = false; - if ( - userExperienceStatus && - legislationStatus && - securityStatus && - performanceStatus - ) { + if (userExperienceStatus && legislationStatus && securityStatus) { complianceCriteriaStatus = true; } From 5b80d5a17d97e69ef7d3a637ee590cab76d798c2 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Wed, 21 Dec 2022 15:18:04 +0100 Subject: [PATCH 13/15] fix: aggiornamento versione Handler --- package.json | 2 +- routes/routes.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 7b646d1..d2a771d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "crawler-handler", - "version": "1.2.0", + "version": "1.2.1", "description": "Handler per il validatore di comuni e scuole", "main": "index.js", "type": "module", diff --git a/routes/routes.ts b/routes/routes.ts index 42c890f..922545b 100644 --- a/routes/routes.ts +++ b/routes/routes.ts @@ -615,7 +615,7 @@ router.post( router.get( "/api/info", (req: emptyBodyType, res: successResponseType | errorResponseType): void => { - succesResponse({ version: "1.1.8" }, res, 200); + succesResponse({ version: "1.2.1" }, res, 200); } ); From 074319f7dbe466a16fb95a981a215305d2700e28 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Wed, 21 Dec 2022 15:27:53 +0100 Subject: [PATCH 14/15] fix: alleggerimento codice --- controller/auditController.ts | 14 ++++---------- utils/utils.ts | 10 +++------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/controller/auditController.ts b/controller/auditController.ts index d7d74e9..b987b23 100644 --- a/controller/auditController.ts +++ b/controller/auditController.ts @@ -71,16 +71,12 @@ const cleanMunicipalityJSONReport = async (jsonResult: string) => { performanceStatus = false; } - let informedCitizenStatus = false; - if ( + let informedCitizenStatus = userExperienceStatus && functionStatus && legislationStatus && securityStatus && - performanceStatus - ) { - informedCitizenStatus = true; - } + performanceStatus; const recommendationsAudits = await getAuditByClusterGroup( parsedResult, @@ -186,10 +182,8 @@ const cleanSchoolJSONReport = async (jsonResult: string) => { performanceStatus = false; } - let complianceCriteriaStatus = false; - if (userExperienceStatus && legislationStatus && securityStatus) { - complianceCriteriaStatus = true; - } + let complianceCriteriaStatus = + userExperienceStatus && legislationStatus && securityStatus; const recommendationsAudits = await getAuditByClusterGroup( parsedResult, diff --git a/utils/utils.ts b/utils/utils.ts index 035f99b..a158c12 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -180,8 +180,6 @@ const calculatePassedAuditPercentage = async ( job: Job, cleanJsonResult ): Promise => { - let passed = 0; - let total = 0; let totalAudits = {}; const mainObjKey = @@ -215,6 +213,8 @@ const calculatePassedAuditPercentage = async ( }; } + let passed = 0; + let total = 0; for (const auditResult of Object.values(totalAudits)) { total++; @@ -223,11 +223,7 @@ const calculatePassedAuditPercentage = async ( } } - if (total > 0) { - return passed + " su " + total; - } - - return "0" + " su " + total; + return passed + " su " + total; }; const urlExists = async (url) => { From c178f1a570bc50afcb27da180bf10d139a5de695 Mon Sep 17 00:00:00 2001 From: Andrea Cattaneo Date: Wed, 21 Dec 2022 15:35:51 +0100 Subject: [PATCH 15/15] fix: static checks --- controller/auditController.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controller/auditController.ts b/controller/auditController.ts index b987b23..8485d82 100644 --- a/controller/auditController.ts +++ b/controller/auditController.ts @@ -71,7 +71,7 @@ const cleanMunicipalityJSONReport = async (jsonResult: string) => { performanceStatus = false; } - let informedCitizenStatus = + const informedCitizenStatus = userExperienceStatus && functionStatus && legislationStatus && @@ -182,7 +182,7 @@ const cleanSchoolJSONReport = async (jsonResult: string) => { performanceStatus = false; } - let complianceCriteriaStatus = + const complianceCriteriaStatus = userExperienceStatus && legislationStatus && securityStatus; const recommendationsAudits = await getAuditByClusterGroup(