Skip to content

Commit 454cec1

Browse files
simone-amadio-acnsimone-amadio-acn
authored
Merge pull request #71 from italia/feat/update_query_logics
Feat/update query logics
2 parents 9e22459 + 4bd93a5 commit 454cec1

9 files changed

+73
-382
lines changed

‎command/PA2026Manager.ts

+37-79
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,6 @@ dbRoot
2929
const updateResult = await update();
3030
console.log("[PA2026 MANAGER]: UPDATE RESULT - ", updateResult);
3131

32-
const forcedScanResult = await forcedScanEntities();
33-
console.log("[PA2026 MANAGER]: FORCED SCAN RESULT - ", forcedScanResult);
34-
3532
const asseverationResult = await asseveration();
3633
console.log(
3734
"[PA2026 MANAGER]: ASSEVERATION RESULT - ",
@@ -52,12 +49,13 @@ dbRoot
5249

5350
const create = async () => {
5451
const createQuery =
55-
"SELECT id,Url_Sito_Internet__c, Pacchetto_1_4_1__c, ID_Crawler__c " +
52+
"SELECT id, Url_Sito_Internet__c, Pacchetto_1_4_1__c, ID_Crawler__c " +
5653
"FROM outfunds__Funding_Request__c " +
57-
"WHERE outfunds__Status__c ='Finanziata' " +
58-
"AND outfunds__FundingProgram__r.RecordType.DeveloperName='Misura_141' " +
54+
"WHERE outfunds__Status__c = 'Finanziata' " +
55+
"AND outfunds__FundingProgram__r.RecordType.DeveloperName = 'Misura_141' " +
5956
"AND Url_Sito_Internet__c !=null " +
60-
"AND ID_Crawler__c=null";
57+
"AND Da_Scansionare__c = true " +
58+
"AND ID_Crawler__c = null";
6159
const returnIds = [];
6260

6361
try {
@@ -91,7 +89,7 @@ const create = async () => {
9189
enable: true,
9290
type: type,
9391
subtype: subtype,
94-
forcedScan: false,
92+
forcedScan: true,
9593
});
9694

9795
if (!entity) {
@@ -102,7 +100,11 @@ const create = async () => {
102100
}
103101

104102
await callPatch(
105-
{ ID_Crawler__c: entity.id },
103+
{
104+
ID_Crawler__c: entity.id,
105+
Da_Scansionare__c: false,
106+
Da_Scansionare_Data_Scansione__c: new Date().getTime(),
107+
},
106108
process.env.PA2026_UPDATE_RECORDS_PATH.replace(
107109
"{external_entity_id}",
108110
entity.external_id
@@ -123,13 +125,13 @@ const create = async () => {
123125

124126
const update = async () => {
125127
const updateQuery =
126-
"SELECT id,Url_Sito_Internet__c, Pacchetto_1_4_1__c, ID_Crawler__c " +
128+
"SELECT id, Url_Sito_Internet__c, Pacchetto_1_4_1__c, ID_Crawler__c, Da_Scansionare__c " +
127129
"FROM outfunds__Funding_Request__c " +
128130
"WHERE outfunds__Status__c ='Finanziata' " +
129-
"AND outfunds__FundingProgram__r.RecordType.DeveloperName='Misura_141' " +
130-
"AND Url_Sito_Internet__c !=null " +
131-
"AND ID_Crawler__c!=null " +
132-
"AND Controllo_URL__c=false ";
131+
"AND outfunds__FundingProgram__r.RecordType.DeveloperName = 'Misura_141' " +
132+
"AND Url_Sito_Internet__c != null " +
133+
"AND ID_Crawler__c != null " +
134+
"AND ( Controllo_URL__c = false OR Da_Scansionare__c = true )";
133135

134136
const returnIds = [];
135137

@@ -162,6 +164,25 @@ const update = async () => {
162164
url: url,
163165
});
164166

167+
const forcedScan = record.Da_Scansionare__c ?? false;
168+
169+
if (forcedScan) {
170+
await entity.update({
171+
forcedScan: true,
172+
});
173+
174+
await callPatch(
175+
{
176+
Da_Scansionare__c: false,
177+
Da_Scansionare_Data_Scansione__c: new Date().getTime(),
178+
},
179+
process.env.PA2026_UPDATE_RECORDS_PATH.replace(
180+
"{external_entity_id}",
181+
entity.external_id
182+
)
183+
);
184+
}
185+
165186
returnIds.push(updateEntity.id);
166187
} catch (e) {
167188
console.log("UPDATE QUERY FOR-STATEMENT EXCEPTION: ", e.toString());
@@ -179,7 +200,8 @@ const asseveration = async () => {
179200
"SELECT id, Codice_amministrativo__c, ID_Crawler__c, ID_Crawler_Job_definitiva__c, Stato_Progetto__c " +
180201
"FROM outfunds__Funding_Request__c " +
181202
"WHERE Stato_Progetto__c IN ('COMPLETATO', 'RESPINTO', 'IN LIQUIDAZIONE', 'LIQUIDATO', 'ANNULLATO', 'RINUNCIATO') " +
182-
"AND Progetto_Terminato__c=false AND ID_Crawler__c != null";
203+
"AND Progetto_Terminato__c = false AND ID_Crawler__c != null";
204+
183205
const returnIds = [];
184206
try {
185207
const asseverationResult = await callQuery(asseverationQuery);
@@ -358,67 +380,3 @@ const sendRetryJobInSendError = async () => {
358380
console.log("SEND RETRY JOB EXCEPTION: ", e.toString());
359381
}
360382
};
361-
362-
const forcedScanEntities = async () => {
363-
const forcedScanEntitiesQuery =
364-
"SELECT id,Url_Sito_Internet__c, Pacchetto_1_4_1__c, ID_Crawler__c " +
365-
"FROM outfunds__Funding_Request__c " +
366-
"WHERE outfunds__Status__c ='Finanziata' " +
367-
"AND outfunds__FundingProgram__r.RecordType.DeveloperName='Misura_141' " +
368-
"AND Url_Sito_Internet__c !=null " +
369-
"AND Da_Scansionare__c=true";
370-
371-
const returnIds = [];
372-
373-
try {
374-
const scanFromPA2026Result = await callQuery(forcedScanEntitiesQuery);
375-
376-
if (!scanFromPA2026Result) {
377-
throw new Error("Empty values from create query");
378-
}
379-
380-
const records = scanFromPA2026Result?.records;
381-
if (records.length <= 0) {
382-
return [];
383-
}
384-
385-
for (const record of records) {
386-
try {
387-
const externalId = record.Id ?? "";
388-
const entity: Entity = await new entityController(dbWS).retrieve(
389-
externalId
390-
);
391-
392-
if (!entity) {
393-
throw new Error("Entity not found: " + externalId);
394-
}
395-
396-
const updateEntity = await entity.update({
397-
forcedScan: true,
398-
});
399-
400-
returnIds.push(updateEntity.id);
401-
402-
await callPatch(
403-
{
404-
Da_Scansionare__c: false,
405-
Da_Scansionare_Data_Scansione__c: new Date().getTime(),
406-
},
407-
process.env.PA2026_UPDATE_RECORDS_PATH.replace(
408-
"{external_entity_id}",
409-
entity.external_id
410-
)
411-
);
412-
} catch (e) {
413-
console.log(
414-
"FORCED SCAN QUERY FOR-STATEMENT EXCEPTION: ",
415-
e.toString()
416-
);
417-
}
418-
}
419-
} catch (e) {
420-
console.log("FORCED SCAN QUERY EXCEPTION: ", e.toString());
421-
}
422-
423-
return returnIds;
424-
};

‎command/queueManager.ts

+16-118
Original file line numberDiff line numberDiff line change
@@ -12,57 +12,15 @@ import { Queue } from "bullmq";
1212
import { preserveReasons } from "../database/models/job.js";
1313
import { jobController } from "../controller/jobController.js";
1414
import {
15-
getFirstTimeEntityToBeAnalyzed,
16-
getRescanEntityToBeAnalyzed,
17-
getRescanEntityAsseveratedToBeAnalyzed,
1815
generateJobs,
1916
getForcedRescanEntitiesToBeAnalyzed,
2017
getFirstTimeForcedEntityToBeAnalyzed,
2118
manageEntitiesInErrorJobs,
2219
} from "../controller/queueManagerController.js";
2320

24-
const command = yargs(hideBin(process.argv))
25-
.usage(
26-
"Usage: " +
27-
"--maxItems <maxItems> " +
28-
"--passedOlderThanDays <passedOlderThanDays> " +
29-
"--failedOlderThanDays <failedOlderThanDays> " +
30-
"--manualScanLogic"
31-
)
32-
.option("maxItems", {
33-
describe: "Numero massimo di entity da analizzare",
34-
type: "integer",
35-
demandOption: true,
36-
default: 100,
37-
})
38-
.option("passedOlderThanDays", {
39-
describe:
40-
"Giorni dopo i quali le entity con Job che ha fornito risultato PASSED vengono riaccodate per essere scansionate",
41-
type: "integer",
42-
demandOption: true,
43-
default: 28,
44-
})
45-
.option("failedOlderThanDays", {
46-
describe:
47-
"Giorni dopo i quali le entity con Job che ha fornito risultato FAILED vengono riaccodate per essere scansionate",
48-
type: "integer",
49-
demandOption: true,
50-
default: 14,
51-
})
52-
.option("asservationOlderThanDays", {
53-
describe:
54-
"Giorni dopo i quali le entity asseverate vengono riaccodate per essere scansionate",
55-
type: "integer",
56-
demandOption: true,
57-
default: 28,
58-
})
59-
.option("manualScanLogic", {
60-
describe:
61-
"Flag per permettere solo alle entity flaggate come 'da scansionare' di entrare in coda di scansione",
62-
type: "bool",
63-
demandOption: true,
64-
default: false,
65-
}).argv;
21+
const command = yargs(hideBin(process.argv)).usage(
22+
"Usage: " + "--maxItems <maxItems> "
23+
).argv;
6624

6725
dbQM
6826
.authenticate()
@@ -96,108 +54,56 @@ dbQM
9654
);
9755
console.log('MANAGE JOB IN "PENDING": ', inPendingJob.length);
9856

99-
const manualScanLogic = command.manualScanLogic;
100-
10157
let entityInErrorJob = [];
102-
if (manualScanLogic) {
103-
entityInErrorJob = await manageEntitiesInErrorJobs();
104-
console.log(
105-
'MANAGE ENTITY WITH LAST JOB IN "ERROR": ',
106-
entityInErrorJob.length
107-
);
108-
}
58+
entityInErrorJob = await manageEntitiesInErrorJobs();
59+
console.log(
60+
'MANAGE ENTITY WITH LAST JOB IN "ERROR": ',
61+
entityInErrorJob.length
62+
);
10963

11064
let gapLimit: number = parseInt(command.maxItems);
11165

112-
let firstTimeEntityToBeAnalyzed = [];
113-
if (command.passedOlderThanDays > 0 || command.failedOlderThanDays) {
114-
firstTimeEntityToBeAnalyzed = await getFirstTimeEntityToBeAnalyzed(
115-
gapLimit
116-
);
117-
118-
gapLimit = gapLimit - firstTimeEntityToBeAnalyzed.length;
119-
}
120-
12166
let firstTimeForcedEntityToBeAnalyzed = [];
122-
if (gapLimit > 0 && manualScanLogic) {
67+
if (gapLimit > 0) {
12368
firstTimeForcedEntityToBeAnalyzed =
12469
await getFirstTimeForcedEntityToBeAnalyzed(gapLimit);
12570

12671
gapLimit = gapLimit - firstTimeForcedEntityToBeAnalyzed.length;
12772
}
12873

12974
let forcedRescanEntitiesToBeAnalyzed = [];
130-
if (gapLimit > 0 && manualScanLogic) {
75+
if (gapLimit > 0) {
13176
forcedRescanEntitiesToBeAnalyzed =
13277
await getForcedRescanEntitiesToBeAnalyzed(gapLimit);
13378

13479
gapLimit = gapLimit - forcedRescanEntitiesToBeAnalyzed.length;
13580
}
13681

137-
let rescanEntityToBeAnalyzed = [];
138-
if (
139-
gapLimit > 0 &&
140-
(command.passedOlderThanDays > 0 || command.failedOlderThanDays)
141-
) {
142-
rescanEntityToBeAnalyzed = await getRescanEntityToBeAnalyzed(
143-
command.passedOlderThanDays,
144-
command.failedOlderThanDays,
145-
gapLimit
146-
);
147-
148-
gapLimit = gapLimit - rescanEntityToBeAnalyzed.length;
149-
}
150-
151-
let rescanEntityAsseveratedToBeAnalyzed = [];
152-
if (gapLimit > 0 && command.asservationOlderThanDays > 0) {
153-
rescanEntityAsseveratedToBeAnalyzed =
154-
await getRescanEntityAsseveratedToBeAnalyzed(
155-
command.asservationOlderThanDays,
156-
gapLimit
157-
);
158-
}
159-
160-
if (
161-
firstTimeEntityToBeAnalyzed.length > 0 ||
162-
firstTimeForcedEntityToBeAnalyzed.length > 0
163-
) {
82+
if (firstTimeForcedEntityToBeAnalyzed.length > 0) {
16483
await generateJobs(
165-
[...firstTimeEntityToBeAnalyzed, ...firstTimeForcedEntityToBeAnalyzed],
84+
[...firstTimeForcedEntityToBeAnalyzed],
16685
crawlerQueue,
16786
true,
16887
preserveReasons[0]
16988
);
17089
}
17190

172-
rescanEntityToBeAnalyzed = rescanEntityToBeAnalyzed.filter(
173-
(elem) => !firstTimeEntityToBeAnalyzed.find(({ id }) => elem.id === id)
91+
forcedRescanEntitiesToBeAnalyzed = forcedRescanEntitiesToBeAnalyzed.filter(
92+
(elem) =>
93+
!firstTimeForcedEntityToBeAnalyzed.find(({ id }) => elem.id === id)
17494
);
17595

176-
rescanEntityAsseveratedToBeAnalyzed =
177-
rescanEntityAsseveratedToBeAnalyzed.filter(
178-
(elem) => !firstTimeEntityToBeAnalyzed.find(({ id }) => elem.id === id)
179-
);
180-
18196
if (
182-
rescanEntityToBeAnalyzed.length > 0 ||
183-
rescanEntityAsseveratedToBeAnalyzed.length > 0 ||
18497
forcedRescanEntitiesToBeAnalyzed.length > 0 ||
18598
entityInErrorJob.length > 0
18699
) {
187100
await generateJobs(
188-
[
189-
...rescanEntityToBeAnalyzed,
190-
...rescanEntityAsseveratedToBeAnalyzed,
191-
...forcedRescanEntitiesToBeAnalyzed,
192-
...entityInErrorJob,
193-
],
101+
[...forcedRescanEntitiesToBeAnalyzed, ...entityInErrorJob],
194102
crawlerQueue,
195103
false
196104
);
197105
}
198106

199-
console.log("FIRST TIME ENTITIES", firstTimeEntityToBeAnalyzed.length);
200-
201107
console.log(
202108
"FIRST TIME FORCED ENTITIES",
203109
firstTimeForcedEntityToBeAnalyzed.length
@@ -207,14 +113,6 @@ dbQM
207113
"FORCED RESCAN ENTITIES",
208114
forcedRescanEntitiesToBeAnalyzed.length
209115
);
210-
console.log(
211-
"RESCAN ENTITIES TO BE ANALYZED",
212-
rescanEntityToBeAnalyzed.length
213-
);
214-
console.log(
215-
"RESCAN ASSEVERATED ENTITIES",
216-
rescanEntityAsseveratedToBeAnalyzed.length
217-
);
218116

219117
const counts = await crawlerQueue.getJobCounts(
220118
"wait",

0 commit comments

Comments
 (0)