From 1bbd0ccefb33126a78b1293f98f09c008cf44246 Mon Sep 17 00:00:00 2001 From: pandutibil Date: Fri, 2 Feb 2024 11:33:19 +0530 Subject: [PATCH] Data debugger API changes are done --- .../controller/ingestion.controller.ts | 62 +++++++-------- .../dimension.grammar.validator.ts | 51 ++++++++---- src/ingestion/dto/request.ts | 11 ++- .../validators/event-grammar.validator.ts | 78 ++++++++++++++----- src/ingestion/validators/utilities.ts | 7 ++ 5 files changed, 139 insertions(+), 70 deletions(-) create mode 100644 src/ingestion/validators/utilities.ts diff --git a/src/ingestion/controller/ingestion.controller.ts b/src/ingestion/controller/ingestion.controller.ts index 225cc99..719944e 100644 --- a/src/ingestion/controller/ingestion.controller.ts +++ b/src/ingestion/controller/ingestion.controller.ts @@ -401,51 +401,49 @@ export class IngestionController { uploadFileN( @UploadedFiles() files: { - grammar?: Express.Multer.File[]; - data?: Express.Multer.File[]; + grammar?: Express.Multer.File[]; + data?: Express.Multer.File[]; }, @Body() body: FileValidateRequest, ) { + const { type } = body; this.logger.debug(files.grammar); const grammarFilePath = files.grammar[0].path; + if (!type) + throw new BadRequestException('Schema Type is required'); + if (!grammarFilePath || !fs.existsSync(grammarFilePath)) - throw new BadRequestException('Grammar file is required'); + throw new BadRequestException('Grammar file is required'); const grammarContent = fs.readFileSync(grammarFilePath, 'utf8'); const dataFilePath = files?.data ? files?.data[0]?.path : undefined; let resp; - switch (body.type.trim()) { - case FileType.DimensionGrammar: - resp = - this.validatorService.checkDimensionGrammarForValidationErrors( - grammarContent, - ); - break; - case FileType.DimensionData: - if (!dataFilePath || !fs.existsSync(dataFilePath)) - throw new BadRequestException('Data file is required'); - - resp = this.validatorService.checkDimensionDataForValidationErrors( - grammarContent, - fs.readFileSync(dataFilePath, 'utf8'), - ); - break; - case FileType.EventGrammar: - resp = - this.validatorService.checkEventGrammarForValidationErrors( - grammarContent, - ); + switch (type.trim()) { + case FileType.Dimension: + if (!dataFilePath) { + resp = this.validatorService.checkDimensionGrammarForValidationErrors( + grammarContent, + ); + } else { + resp = this.validatorService.checkDimensionDataForValidationErrors( + grammarContent, + fs.readFileSync(dataFilePath, 'utf8'), + ); + } break; - case FileType.EventData: - if (!dataFilePath || !fs.existsSync(dataFilePath)) - throw new BadRequestException('Data file is required'); - - resp = this.validatorService.checkEventDataForValidationErrors( - grammarContent, - fs.readFileSync(dataFilePath, 'utf8'), - ); + case FileType.Event: + if (!dataFilePath) { + resp = this.validatorService.checkEventGrammarForValidationErrors( + grammarContent, + ); + } else { + resp = this.validatorService.checkEventDataForValidationErrors( + grammarContent, + fs.readFileSync(dataFilePath, 'utf8'), + ); + } break; default: throw new BadRequestException('Invalid file type'); diff --git a/src/ingestion/cqube-spec-checker/dimension.grammar.validator.ts b/src/ingestion/cqube-spec-checker/dimension.grammar.validator.ts index fd3d46f..cfde8ff 100644 --- a/src/ingestion/cqube-spec-checker/dimension.grammar.validator.ts +++ b/src/ingestion/cqube-spec-checker/dimension.grammar.validator.ts @@ -4,6 +4,7 @@ export class DimensionValidator { pkIndexLine: any; dataTypesLine: any; headerLine: any; + constructor(content) { this.content = content; this.lines = this.content.trim().split('\n'); @@ -45,32 +46,54 @@ export class DimensionValidator { } verifyPkIndexLine() { - const errors = []; - if ( - this.pkIndexLine.indexOf('PK') === -1 || - this.pkIndexLine.indexOf('Index') === -1 - ) { - errors.push({ - row: 0, - col: 0, - errorCode: 1003, - error: `Invalid PK/Index: First row must include 'PK' and 'Index' but found "${this.pkIndexLine}"`, - data: this.pkIndexLine, + if (this.pkIndexLine && this.pkIndexLine.length > 0) { + const errors = []; + let isAllEmpty = true; + this.pkIndexLine.forEach((index, ind) => { + console.log(index); + if (typeof index === 'string'){ + index.trim(); + } + + if (index && index !== '' && index !== "PK" && index !== "Index") { + errors.push({ + row: 0, + col: ind, + errorCode: 1003, + error: `Invalid PK/Index: First row must include 'PK' and 'Index' but found "${index}"`, + data: this.pkIndexLine, + }); + } else if (index === "PK" || index === "Index") { + isAllEmpty = false; + } }); + + if (isAllEmpty) { + errors.push({ + row: -1, + col: "common", + errorCode: 1003, + error: `Dimension should contain atleast one PK or Index Field`, + data: this.pkIndexLine, + }); + } + + return errors; } - return errors; + + return []; } verifyDataTypes() { const errors = []; this.dataTypesLine.forEach((dataType, columnIndex) => { - if (dataType !== 'string' && dataType !== 'integer') { + if (dataType !== 'string' && dataType !== 'number' && dataType !== 'integer') { errors.push({ row: 1, col: columnIndex, errorCode: 1002, error: `Invalid data type at column ${columnIndex + 1 - }: Only 'string' and 'integer' are allowed but found '${dataType}'`, + }: Only 'string', 'number', and 'integer' are allowed but found '${dataType}'`, data: this.dataTypesLine, }); } diff --git a/src/ingestion/dto/request.ts b/src/ingestion/dto/request.ts index 0dfa822..f4a3c2b 100644 --- a/src/ingestion/dto/request.ts +++ b/src/ingestion/dto/request.ts @@ -1,8 +1,11 @@ export enum FileType { - DimensionGrammar = 'dimension-grammar', - DimensionData = 'dimension-data', - EventGrammar = 'event-grammar', - EventData = 'event-data', + Dimension = 'dimension', + Event = 'event', +} + +export enum ValidationType { + Grammar = 'grammar', + Data = 'data', } export class FileValidateRequest { diff --git a/src/ingestion/validators/event-grammar.validator.ts b/src/ingestion/validators/event-grammar.validator.ts index 15546c4..b123f17 100644 --- a/src/ingestion/validators/event-grammar.validator.ts +++ b/src/ingestion/validators/event-grammar.validator.ts @@ -9,7 +9,7 @@ export class EventGrammarValidator { } verify() { - const errors: any[] = []; + let errors: any[] = []; // check for length const len = this.content.length; if (len !== 5) { @@ -22,12 +22,33 @@ export class EventGrammarValidator { } // check for equal number of columns in each row - const colCount = []; - this.content.forEach((row) => { - colCount.push(row.split(',').length); + let prevColCount; + let colCountMatchFailed = false; + this.content.forEach((row, lineNo) => { + if (lineNo === 0) { + prevColCount = row.split(',').length; + } + + if (lineNo > 0 && !colCountMatchFailed && prevColCount !== row.split(',').length) { + colCountMatchFailed = true; + } + + if (lineNo === 2) { + row.split(',').forEach((dataType: string, index: number) => { + if (!['string', 'integer', 'date', 'number'].includes(dataType.trim())) { + errors.push({ + row: lineNo, + col: index, + errorCode: 1002, + error: `Invalid data type: ${dataType}. Supported data types are: string, integer, date, number`, + data: dataType, + }); + } + }); + } }); - if (colCount.every((val) => val === colCount[0]) === false) { + if (colCountMatchFailed) { errors.push({ row: 0, col: 0, @@ -35,25 +56,42 @@ export class EventGrammarValidator { error: 'Invalid CSV file: all rows should have equal number of columns', }); } + // TODO: add check for dimension names + const dimensionNameRow = this.content[0].split(','); + const dimensionKeyNameRow = this.content[1].split(','); + const dimensionErrors = dimensionNameRow.map((dimensionName: string, index: number) => { + if (dimensionName === '' && dimensionKeyNameRow[index] === '') { + return null; + } else if (dimensionName === '') { + return { + row: 0, + col: index, + errorCode: 1004, + error: `Dimension name is missing for dimension key:${dimensionKeyNameRow[index]}`, + data: this.content[0], + } + } else if (dimensionKeyNameRow[index] === '') { + return { + row: 0, + col: index, + errorCode: 1004, + error: `Dimension key name is missing for dimension:${dimensionName}`, + data: this.content[1], + } + } + }).filter(error => error !== null && error !== undefined); + + if (dimensionErrors.length > 0) { + errors = [...errors, ...dimensionErrors]; + } // TODO: add check for dimension key names // Check for supported data types - const dataTypeRow = this.content[2]; // const dataTypeErrors = []; - dataTypeRow.split(',').forEach((dataType: string, index: number) => { - if (!['string', 'integer', 'date'].includes(dataType.trim())) { - errors.push({ - row: 1, - col: index, - errorCode: 1002, - error: `Invalid data type: ${dataType}. Supported data types are: string, integer, date`, - data: dataType, - }); - } - }); + // errors.push({ // row: 3, @@ -72,7 +110,7 @@ export class EventGrammarValidator { row: 4, col: idx, errorCode: 1004, - error: `Dimension Grammar Specification Error: Wrong values in fieldType row, allowed values are 1. dimension 2.timeDimension 3. metric, but received ${item}`, + error: `Dimension Grammar Specification Error: Wrong values in fieldType row, allowed values are 'dimension', 'timeDimension', 'metric', but received ${item}`, data: lastRow, }); } else if (item.trim() === 'dimension') { @@ -88,7 +126,7 @@ export class EventGrammarValidator { .split(',') .map((item: string) => item.trim()); - dimensionIdxs.forEach((idx: number) => { + /* dimensionIdxs.forEach((idx: number) => { if (fkKeysRow[idx] !== headerRow[idx]) { errors.push( { @@ -107,7 +145,7 @@ export class EventGrammarValidator { }, ); } - }); + });*/ return errors; } diff --git a/src/ingestion/validators/utilities.ts b/src/ingestion/validators/utilities.ts new file mode 100644 index 0000000..969192f --- /dev/null +++ b/src/ingestion/validators/utilities.ts @@ -0,0 +1,7 @@ +export function containsNonEmptyValues(arr) { + return arr.some(item => item !== null || item !== undefined || item !== ''); +} + +export function containsEmptyValues(arr) { + return arr.some(item => item === null || item === undefined || item === ''); +}