Skip to content

Commit

Permalink
Append errMsg from validateReadings onto msgTotal in processData
Browse files Browse the repository at this point in the history
  • Loading branch information
@carlsonrob
carlsonrob committed Aug 24, 2023
1 parent 752d941 commit 8f20f9c
Show file tree
Hide file tree
Showing 3 changed files with 68 additions and 41 deletions.
20 changes: 12 additions & 8 deletions src/server/services/pipeline-in-progress/processData.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -627,14 +627,18 @@ async function processData(rows, meterID, timeSort = TimeSortTypesJS.increasing,
prevEndTimestampTz = endTimestampTz;
}
// Validate data if conditions given
if (conditionSet !== undefined && !conditionSet['disableChecks'] && !validateReadings(result, conditionSet, meterName)) {
errMsg = `<h2>For meter ${meterName}: error when validating data so all reading are rejected</h2>`;
log.error(errMsg);
({ msgTotal, msgTotalWarning } = appendMsgTotal(msgTotal, errMsg, msgTotalWarning));
// This empties the result array. Should be fast and okay with const.
result.splice(0, result.length);
isAllReadingsOk = false;
return { result, isAllReadingsOk, msgTotal };
if (conditionSet !== undefined && !conditionSet['disableChecks']) {
const { validReadings, errMsg: newErrMsg } = validateReadings(result, conditionSet, meterName);
({ msgTotal, msgTotalWarning } = appendMsgTotal(msgTotal, newErrMsg, msgTotalWarning));
if (!validReadings) {
errMsg = `<h2>For meter ${meterName}: error when validating data so all reading are rejected</h2>`;
log.error(errMsg);
({ msgTotal, msgTotalWarning } = appendMsgTotal(msgTotal, errMsg, msgTotalWarning));
// This empties the result array. Should be fast and okay with const.
result.splice(0, result.length);
isAllReadingsOk = false;
return { result, isAllReadingsOk, msgTotal };
}
}
// Update the meter to contain information for the last reading in the data file.
// Note this means that even if the last value was rejected we still store it as
Expand Down
69 changes: 46 additions & 23 deletions src/server/services/pipeline-in-progress/validateReadings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,14 @@ const { log } = require('../../log');
*/
function validateReadings(arrayToValidate, conditionSet, meterName = undefined) {
/* tslint:disable:no-string-literal */
validDates = checkDate(arrayToValidate, conditionSet['minDate'], conditionSet['maxDate'], conditionSet['maxError'] / 2, meterName);
validValues = checkValue(arrayToValidate, conditionSet['minVal'], conditionSet['maxVal'], conditionSet['maxError'] / 2, meterName);
const { validDates, errMsg: errMsgDate } = checkDate(arrayToValidate, conditionSet['minDate'], conditionSet['maxDate'], conditionSet['maxError'] / 2, meterName);
const { validValues, errMsg: errMsgValue } = checkValue(arrayToValidate, conditionSet['minVal'], conditionSet['maxVal'], conditionSet['maxError'] / 2, meterName);
/* tslint:enable:no-string-literal */
return validDates && validValues;
const errMsg = errMsgDate + errMsgValue;
return {
validReadings: validDates && validValues,
errMsg,
};
}

/**
Expand All @@ -28,26 +32,31 @@ function validateReadings(arrayToValidate, conditionSet, meterName = undefined)
* @param {number} maxError maximum number of errors to be reported, ignore the rest
*/
function checkDate(arrayToValidate, minDate, maxDate, maxError, meterName) {
let validDates = true;
let errMsg = '';
if (minDate === null && maxDate === null) {
return true;
return { validDates, errMsg };
}
validDates = true;
for (reading of arrayToValidate) {
for (const reading of arrayToValidate) {
if (maxError <= 0) {
break;
}
if (reading.startTimestamp < minDate) {
log.error(`error when checking reading time from meter ${meterName}: time ${reading.startTimestamp} is earlier than lower bound ${minDate}`);
const newErrMsg = `error when checking reading time from meter ${meterName}: time ${reading.startTimestamp} is earlier than lower bound ${minDate}`;
log.error(newErrMsg);
errMsg += newErrMsg + '<br>';
--maxError;
validDates = false;
}
if (reading.endTimestamp > maxDate) {
log.error(`error when checking reading time from meter ${meterName}: time ${reading.endTimestamp} is later than upper bound ${maxDate}`);
const newErrMsg = `error when checking reading time from meter ${meterName}: time ${reading.endTimestamp} is later than upper bound ${maxDate}`;
log.error(newErrMsg);
errMsg += newErrMsg + '<br>';
--maxError;
validDates = false;
}
}
return validDates;
return { validDates, errMsg };
}

/**
Expand All @@ -58,22 +67,27 @@ function checkDate(arrayToValidate, minDate, maxDate, maxError, meterName) {
* @param {number} maxError maximum number of errors to be reported, ignore the rest
*/
function checkValue(arrayToValidate, minVal, maxVal, maxError, meterName) {
validValues = true;
for (reading of arrayToValidate) {
let validValues = true;
let errMsg = '';
for (const reading of arrayToValidate) {
if (maxError <= 0) {
break;
}
if (reading.reading < minVal) {
log.error(`error when checking reading value from meter ${meterName}: value ${reading.reading} is smaller than lower bound ${minVal}`);
const newErrMsg = `error when checking reading value from meter ${meterName}: value ${reading.reading} is smaller than lower bound ${minVal}`;
log.error(newErrMsg);
errMsg += newErrMsg + '<br>';
--maxError;
validValues = false;
} else if (reading.reading > maxVal) {
log.error(`error when checking reading value from meter ${meterName}: value ${reading.reading} is larger than upper bound ${maxVal}`);
const newErrMsg = `error when checking reading value from meter ${meterName}: value ${reading.reading} is larger than upper bound ${maxVal}`;
log.error(newErrMsg);
errMsg += newErrMsg + '<br>';
--maxError;
validValues = false;
}
}
return validValues;
return { validValues, errMsg };
}

/**
Expand All @@ -82,28 +96,37 @@ function checkValue(arrayToValidate, minVal, maxVal, maxError, meterName) {
* @param {number} threshold the maximum allowed difference between consecutive data points' intervals
*/
function checkIntervals(arrayToValidate, threshold, meterName) {
let validIntervals = true;
let errMsg = '';

if (threshold === null) {
return true;
return { validIntervals, errMsg };
}

// Set the expected interval to be the time gap between the first 2 data points
interval = arrayToValidate[1].startTimestamp.diff(arrayToValidate[0].endTimestamp, 'seconds');
lastTime = arrayToValidate[1].endTimestamp;
const interval = arrayToValidate[1].startTimestamp.diff(arrayToValidate[0].endTimestamp, 'seconds');
let lastTime = arrayToValidate[1].endTimestamp;

// Calculate the time gap between every pair of consecutive data points
for (reading of arrayToValidate) {
for (const reading of arrayToValidate) {
if (reading === arrayToValidate[0]) {
continue;
}
currGap = reading.startTimestamp.diff(lastTime, 'seconds');
// Compare the current time gap with the expected interval. Terminate if the difference is larger than accepted threshold
const currGap = reading.startTimestamp.diff(lastTime, 'seconds');
// Compare the current time gap with the expected interval. Terminate if the difference is larger than the accepted threshold
if (Math.abs(currGap - interval) > threshold) {
log.error(`unequal interval is detected from meter ${meterName}: time gap between ${reading.startTimestamp} and ${lastTime} is too big`);
return false;
const newErrMsg = `unequal interval is detected from meter ${meterName}: time gap between ${reading.startTimestamp} and ${lastTime} is too big`;
log.error(newErrMsg);
errMsg += newErrMsg + '<br>';
validIntervals = false;
break;
}
lastTime = reading.endTimestamp;
}
return true;
return { validIntervals, errMsg }; // Return both values in an object
}


module.exports = {
validateReadings,
checkDate,
Expand Down
20 changes: 10 additions & 10 deletions src/server/test/db/validateReadingsTests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ mocha.describe('PIPELINE: Validate Readings', () => {
)).map(reading => checkDate([reading], minDate, maxDate, Number.MAX_VALUE));
for (let i = 0; i < 4; ++i) {
if (i % 2 === 0) {
expect(results[i]).to.equal(true);
expect(results[i].validDates).to.equal(true);
} else {
expect(results[i]).to.equal(false);
expect(results[i].validDates).to.equal(false);
}
}
});
Expand All @@ -46,9 +46,9 @@ mocha.describe('PIPELINE: Validate Readings', () => {
)).map(reading => checkValue([reading], minVal, maxVal, Number.MAX_VALUE));
for (let i = 0; i < 7; ++i) {
if (i % 2 === 0) {
expect(results[i]).to.equal(false);
expect(results[i].validValues).to.equal(false);
} else {
expect(results[i]).to.equal(true);
expect(results[i].validValues).to.equal(true);
}
}
});
Expand All @@ -57,14 +57,14 @@ mocha.describe('PIPELINE: Validate Readings', () => {
new Reading(undefined, 0, moment('1970-01-01 00:01:00'), moment('1970-01-01 00:01:01')),
new Reading(undefined, 0, moment('1970-01-01 00:04:00'), moment('1970-01-01 00:04:01'))];
let result = checkIntervals(testing, 119);
expect(result).to.equal(false);
expect(result.validIntervals).to.equal(false);
});
mocha.it('detects equal intervals', async () => {
let testing = [ new Reading(undefined, 0, moment('1970-01-01 00:00:00'), moment('1970-01-01 00:01:00')),
new Reading(undefined, 0, moment('1970-01-01 00:01:00'), moment('1970-01-01 00:01:01')),
new Reading(undefined, 0, moment('1970-01-01 00:01:30'), moment('1970-01-01 00:02:01'))];
let result = checkIntervals(testing, 29);
expect(result).to.equal(true);
expect(result.validIntervals).to.equal(true);
});
mocha.it('reject data with any type of error', async () => {
let conditionSet = {
Expand Down Expand Up @@ -92,10 +92,10 @@ mocha.describe('PIPELINE: Validate Readings', () => {
new Reading(undefined, 20, moment('1970-01-01 00:01:00'), moment('1970-01-01 00:01:01')),
new Reading(undefined, 0, moment('1970-01-01 00:01:30'), moment('1970-01-01 00:02:01'))];

expect(checkIntervals(badIntervals, conditionSet['threshold'])).to.equal(false);
expect(validateReadings(badDate, conditionSet)).to.equal(false);
expect(validateReadings(badValue, conditionSet)).to.equal(false);
expect(validateReadings(goodData, conditionSet)).to.equal(true);
expect(checkIntervals(badIntervals, conditionSet['threshold']).validIntervals).to.equal(false);
expect(validateReadings(badDate, conditionSet).validReadings).to.equal(false);
expect(validateReadings(badValue, conditionSet).validReadings).to.equal(false);
expect(validateReadings(goodData, conditionSet).validReadings).to.equal(true);

});
});

0 comments on commit 8f20f9c

Please sign in to comment.