diff --git a/command/queueManager.ts b/command/queueManager.ts index 0a0676b..4fc4c71 100644 --- a/command/queueManager.ts +++ b/command/queueManager.ts @@ -105,16 +105,6 @@ dbQM ); } - console.log("FIRST TIME ENTITIES", firstTimeEntityToBeAnalyzed.length); - console.log( - "RESCAN ENTITIES TO BE ANALYZED", - rescanEntityToBeAnalyzed.length - ); - console.log( - "RESCAN ASSEVERATED ENTITIES", - rescanEntityAsseveratedToBeAnalyzed.length - ); - if (firstTimeEntityToBeAnalyzed.length > 0) { await generateJobs( firstTimeEntityToBeAnalyzed, @@ -144,6 +134,16 @@ dbQM ); } + console.log("FIRST TIME ENTITIES", firstTimeEntityToBeAnalyzed.length); + console.log( + "RESCAN ENTITIES TO BE ANALYZED", + rescanEntityToBeAnalyzed.length + ); + console.log( + "RESCAN ASSEVERATED ENTITIES", + rescanEntityAsseveratedToBeAnalyzed.length + ); + const counts = await crawlerQueue.getJobCounts( "wait", "completed", diff --git a/command/scanManager.ts b/command/scanManager.ts index bd7f434..c9c238c 100644 --- a/command/scanManager.ts +++ b/command/scanManager.ts @@ -21,7 +21,11 @@ import { isPassedReport, } from "../controller/auditController"; import { jobController } from "../controller/jobController"; -import { pushResult } from "../controller/PA2026/integrationController"; +import { + pushResult, + pushResultUrlNotExists, +} from "../controller/PA2026/integrationController"; +import { urlExists } from "../utils/utils"; dbSM .authenticate() @@ -76,8 +80,16 @@ const scan = async (jobId) => { }); const jobObjParsed = jobObj.toJSON(); + const urlToBeScanned = jobObjParsed.scan_url; + + const urlToBeScannedExists = await urlExists(urlToBeScanned); + if (!urlToBeScannedExists) { + await pushResultUrlNotExists(jobObj); + throw new Error("Scan URL does not exists"); + } + const lighthouseResult = await run( - jobObjParsed.scan_url, + urlToBeScanned, jobObjParsed.type, "online", logLevels.display_none, diff --git a/controller/PA2026/integrationController.ts b/controller/PA2026/integrationController.ts index 2d9ea84..dbdc36d 100644 --- a/controller/PA2026/integrationController.ts +++ b/controller/PA2026/integrationController.ts @@ -14,7 +14,11 @@ import { define as jobDefine, preserveReasons, } from "../../database/models/job"; -import { mapPA2026Body } from "../../utils/utils"; +import { + calculatePassedAuditPercentage, + mapPA2026Body, + mapPA2026BodyUrlNotExists, +} from "../../utils/utils"; const retrieveToken = async () => { try { @@ -115,11 +119,17 @@ const pushResult = async ( const isFirstScan = job.preserve && job.preserve_reason === preserveReasons[0]; + const passedAuditsPercentage = await calculatePassedAuditPercentage( + job, + cleanJsonReport + ); + let scanBody = await mapPA2026Body( job, cleanJsonReport, generalStatus, - false + false, + passedAuditsPercentage ); if (isFirstScan) { @@ -127,7 +137,8 @@ const pushResult = async ( job, cleanJsonReport, generalStatus, - true + true, + passedAuditsPercentage ); scanBody = { @@ -173,4 +184,43 @@ const pushResult = async ( } }; -export { retrieveToken, callQuery, callPatch, pushResult }; +const pushResultUrlNotExists = async (job: Job) => { + try { + const entity = await new entityController(dbSM).retrieveByPk(job.entity_id); + + const scanBody = await mapPA2026BodyUrlNotExists(); + + const result = await callPatch( + scanBody, + process.env.PA2026_UPDATE_RECORDS_PATH.replace( + "{external_entity_id}", + entity.external_id + ) + ); + + //Warn: API returns empty string when it success + if (result !== "") { + throw new Error("Send data failed"); + } + + await job.update({ + data_sent_status: "COMPLETED", + data_sent_date: new Date(), + }); + } catch (e) { + console.log("PUSH RESULT EXCEPTION", e.toString()); + + await job.update({ + data_sent_status: "ERROR", + data_sent_date: new Date(), + }); + } +}; + +export { + retrieveToken, + callQuery, + callPatch, + pushResult, + pushResultUrlNotExists, +}; diff --git a/controller/auditController.ts b/controller/auditController.ts index eaf373b..8485d82 100644 --- a/controller/auditController.ts +++ b/controller/auditController.ts @@ -71,29 +71,12 @@ const cleanMunicipalityJSONReport = async (jsonResult: string) => { performanceStatus = false; } - let informedCitizenStatus = false; - if ( + const informedCitizenStatus = userExperienceStatus && functionStatus && legislationStatus && securityStatus && - performanceStatus - ) { - informedCitizenStatus = true; - } - - //const activeCitizenAudits = await getAuditByClusterGroup( - // parsedResult, - // municipalityAudits, - // "active-citizen" - //); - //let activeCitizenStatus = false; - //if ( - // Object.keys(activeCitizenAudits.passed).length > 0 && - // Object.keys(activeCitizenAudits.failed).length === 0 - //) { - // activeCitizenStatus = true; - //} + performanceStatus; const recommendationsAudits = await getAuditByClusterGroup( parsedResult, @@ -138,11 +121,6 @@ const cleanMunicipalityJSONReport = async (jsonResult: string) => { }, }, - //"cittadino-attivo": { - // status: activeCitizenStatus, - // audits: { ...activeCitizenAudits.passed, ...activeCitizenAudits.failed }, - //}, - raccomandazioni: { status: recommendationsStatus, audits: { @@ -204,15 +182,8 @@ const cleanSchoolJSONReport = async (jsonResult: string) => { performanceStatus = false; } - let complianceCriteriaStatus = false; - if ( - userExperienceStatus && - legislationStatus && - securityStatus && - performanceStatus - ) { - complianceCriteriaStatus = true; - } + const complianceCriteriaStatus = + userExperienceStatus && legislationStatus && securityStatus; const recommendationsAudits = await getAuditByClusterGroup( parsedResult, @@ -349,7 +320,6 @@ const isPassedReport = async ( } else if (subtype === allowedMunicipalitySubTypes[1]) { // eslint-disable-next-line passed = jsonReport["cittadino-informato"].status; - //&& jsonReport["cittadino-attivo"].status; } else { passed = false; } diff --git a/database/models/job.ts b/database/models/job.ts index 0cd3c1b..436f26b 100644 --- a/database/models/job.ts +++ b/database/models/job.ts @@ -145,12 +145,7 @@ const options = { Sequelize.literal( "((json_result->'raccomandazioni'->'audits'->>'municipality-metatag')::float)" ), - //Sequelize.literal( - // "((json_result->'cittadino-attivo'->'audits'->>'municipality-subdomain')::float)" - //), - //Sequelize.literal( - // "((json_result->'cittadino-attivo'->'audits'->>'municipality-personal-area-security')::float)" - //), + Sequelize.literal( "((json_result->'cittadino-informato'->'groups'->'normativa'->'audits'->>'municipality-legislation-privacy-is-present')::float)" ), diff --git a/package.json b/package.json index 7b646d1..d2a771d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "crawler-handler", - "version": "1.2.0", + "version": "1.2.1", "description": "Handler per il validatore di comuni e scuole", "main": "index.js", "type": "module", diff --git a/routes/routes.ts b/routes/routes.ts index 42c890f..922545b 100644 --- a/routes/routes.ts +++ b/routes/routes.ts @@ -615,7 +615,7 @@ router.post( router.get( "/api/info", (req: emptyBodyType, res: successResponseType | errorResponseType): void => { - succesResponse({ version: "1.1.8" }, res, 200); + succesResponse({ version: "1.2.1" }, res, 200); } ); diff --git a/storage/municipalityAudits.ts b/storage/municipalityAudits.ts index 4b322bf..d7758fb 100644 --- a/storage/municipalityAudits.ts +++ b/storage/municipalityAudits.ts @@ -23,9 +23,5 @@ export const audits = { ], security: ["municipality-security", "municipality-domain"], }, - //"active-citizen": [ - // "municipality-personal-area-security", - // "municipality-subdomain", - //], recommendations: ["municipality-metatag"], }; diff --git a/utils/utils.ts b/utils/utils.ts index 4b026fc..a158c12 100644 --- a/utils/utils.ts +++ b/utils/utils.ts @@ -1,7 +1,17 @@ "use strict"; + +import { dirname } from "path"; +import { readFileSync } from "fs"; import { ValidationError } from "jsonschema"; import { Job } from "../types/models"; import { auditDictionary } from "pa-website-validator/dist/storage/auditDictionary"; +import axios from "axios"; +import path from "path"; +import { fileURLToPath } from "url"; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +const __dirname = dirname(fileURLToPath(import.meta.url)); const arrayChunkify = async ( inputArray: [], @@ -74,7 +84,8 @@ const mapPA2026Body = async ( job: Job, cleanJsonResult, generalStatus: boolean, - isFirstScan: boolean + isFirstScan: boolean, + passedAuditsPercentage: string ) => { try { const mainObjKey = @@ -91,7 +102,23 @@ const mapPA2026Body = async ( const key = isFirstScan ? "1" : "n"; + let packageJSON; + try { + packageJSON = + JSON.parse( + await readFileSync( + path.resolve(__dirname, "../package.json") + ).toString() + ) ?? {}; + } catch (e) { + packageJSON = null; + console.log("MAP PA2026 BODY EXCEPTION 01: ", e); + } + const initialBody = []; + initialBody[`Versione_Crawler_${key}__c`] = + packageJSON?.dependencies["pa-website-validator"]?.split("#")[1] ?? ""; + initialBody[`Criteri_Superati_Crawler_${key}__c`] = passedAuditsPercentage; initialBody[`Status_Generale_${key}__c`] = generalStatus; initialBody[`Data_Job_Crawler_${key}__c`] = new Date(job.end_at).getTime(); (initialBody[`URL_Scansione_${key}__c`] = job.scan_url), @@ -124,13 +151,7 @@ const mapPA2026Body = async ( const functionObj = cleanJsonResult[mainObjKey].groups["funzionalita"]; (initialBody[`Cittadino_Informato_${key}__c`] = cleanJsonResult[mainObjKey].status), - //(initialBody[`Cittadino_Attivo_${key}__c`] = - // cleanJsonResult["cittadino-attivo"].status), (initialBody[`Funzionalita_${key}__c`] = functionObj.status), - //(initialBody[`Cittadino_Attivo_${key}_Descrizione__c`] = - // getFailAudits(cleanJsonResult["cittadino-attivo"].audits) - // .map((x) => mapAuditTitle(x)) - // .join(" | ") ?? ""), (initialBody[`Funzionalita_${key}_Descrizione__c`] = getFailAudits(functionObj.audits) .map((x) => mapAuditTitle(x)) @@ -144,8 +165,90 @@ const mapPA2026Body = async ( return Object.assign({}, initialBody); } catch (e) { - console.log("MAP PA2026 BODY EXCEPTION: ", e.toString()); + console.log("MAP PA2026 BODY EXCEPTION 02: ", e.toString()); + } +}; + +const mapPA2026BodyUrlNotExists = async () => { + const body = []; + body[`Data_scansione_fallita__c`] = new Date().toISOString().split("T")[0]; + + return Object.assign({}, body); +}; + +const calculatePassedAuditPercentage = async ( + job: Job, + cleanJsonResult +): Promise => { + let totalAudits = {}; + + const mainObjKey = + job.type === "municipality" ? "cittadino-informato" : "criteri-conformita"; + + const legislationAudits = + cleanJsonResult[mainObjKey]["groups"]["normativa"]["audits"] ?? {}; + const securityAudits = + cleanJsonResult[mainObjKey]["groups"]["sicurezza"]["audits"] ?? {}; + const userExperienceAudits = + cleanJsonResult[mainObjKey]["groups"]["esperienza-utente"]["audits"] ?? {}; + totalAudits = { + ...legislationAudits, + ...securityAudits, + ...userExperienceAudits, + }; + + if (job.type === "municipality") { + const functionalityAudits = + cleanJsonResult[mainObjKey]["groups"]["funzionalita"]["audits"] ?? {}; + const performancesResult = + cleanJsonResult[mainObjKey]["groups"]["prestazioni"]["status"] ?? 0; + const performancesAudits = { + "municipality-status": performancesResult === true ? 1 : 0, + }; + + totalAudits = { + ...totalAudits, + ...functionalityAudits, + ...performancesAudits, + }; + } + + let passed = 0; + let total = 0; + for (const auditResult of Object.values(totalAudits)) { + total++; + + if (auditResult > 0) { + passed++; + } } + + return passed + " su " + total; }; -export { arrayChunkify, mapValidationErrors, mapPA2026Body }; +const urlExists = async (url) => { + try { + let statusCode = undefined; + const response = await axios.get(url); + statusCode = response.status; + + if (statusCode === undefined || statusCode < 200 || statusCode >= 400) { + return false; + } + + return true; + } catch (ex) { + console.log("Url Exists Exception: ", ex.toString()); + + return false; + } +}; + +export { + arrayChunkify, + mapValidationErrors, + mapPA2026Body, + calculatePassedAuditPercentage, + urlExists, + mapPA2026BodyUrlNotExists, +};