Data debugger API changes are done

Sunbird-cQube · Feb 2, 2024 · 1bbd0cc · 1bbd0cc
1 parent d8608c1
commit 1bbd0cc
Show file tree

Hide file tree

Showing 5 changed files with 139 additions and 70 deletions.
diff --git a/src/ingestion/controller/ingestion.controller.ts b/src/ingestion/controller/ingestion.controller.ts
@@ -401,51 +401,49 @@ export class IngestionController {
     uploadFileN(
         @UploadedFiles()
         files: {
-        grammar?: Express.Multer.File[];
-        data?: Express.Multer.File[];
+            grammar?: Express.Multer.File[];
+            data?: Express.Multer.File[];
         },
         @Body() body: FileValidateRequest,
     ) {
+        const { type } = body;
         this.logger.debug(files.grammar);
         const grammarFilePath = files.grammar[0].path;
 
+        if (!type)
+            throw new BadRequestException('Schema Type is required'); 
+
         if (!grammarFilePath || !fs.existsSync(grammarFilePath))
-        throw new BadRequestException('Grammar file is required');
+            throw new BadRequestException('Grammar file is required');
 
         const grammarContent = fs.readFileSync(grammarFilePath, 'utf8');
         const dataFilePath = files?.data ? files?.data[0]?.path : undefined;
 
         let resp;
-        switch (body.type.trim()) {
-        case FileType.DimensionGrammar:
-            resp =
-            this.validatorService.checkDimensionGrammarForValidationErrors(
-                grammarContent,
-            );
-            break;
-        case FileType.DimensionData:
-            if (!dataFilePath || !fs.existsSync(dataFilePath))
-            throw new BadRequestException('Data file is required');
-
-            resp = this.validatorService.checkDimensionDataForValidationErrors(
-            grammarContent,
-            fs.readFileSync(dataFilePath, 'utf8'),
-            );
-            break;
-        case FileType.EventGrammar:
-            resp =
-            this.validatorService.checkEventGrammarForValidationErrors(
-                grammarContent,
-            );
+        switch (type.trim()) {
+        case FileType.Dimension:
+            if (!dataFilePath) {
+                resp = this.validatorService.checkDimensionGrammarForValidationErrors(
+                    grammarContent,
+                );
+            } else {
+                resp = this.validatorService.checkDimensionDataForValidationErrors(
+                    grammarContent,
+                    fs.readFileSync(dataFilePath, 'utf8'),
+                );
+            }
             break;
-        case FileType.EventData:
-            if (!dataFilePath || !fs.existsSync(dataFilePath))
-            throw new BadRequestException('Data file is required');
-
-            resp = this.validatorService.checkEventDataForValidationErrors(
-            grammarContent,
-            fs.readFileSync(dataFilePath, 'utf8'),
-            );
+        case FileType.Event:
+            if (!dataFilePath) {
+                resp = this.validatorService.checkEventGrammarForValidationErrors(
+                    grammarContent,
+                );
+            } else {
+                resp = this.validatorService.checkEventDataForValidationErrors(
+                    grammarContent,
+                    fs.readFileSync(dataFilePath, 'utf8'),
+                );
+            }
             break;
         default:
             throw new BadRequestException('Invalid file type');

diff --git a/src/ingestion/cqube-spec-checker/dimension.grammar.validator.ts b/src/ingestion/cqube-spec-checker/dimension.grammar.validator.ts
@@ -4,6 +4,7 @@ export class DimensionValidator {
   pkIndexLine: any;
   dataTypesLine: any;
   headerLine: any;
+
   constructor(content) {
     this.content = content;
     this.lines = this.content.trim().split('\n');
@@ -45,32 +46,54 @@ export class DimensionValidator {
   }
 
   verifyPkIndexLine() {
-    const errors = [];
-    if (
-      this.pkIndexLine.indexOf('PK') === -1 ||
-      this.pkIndexLine.indexOf('Index') === -1
-    ) {
-      errors.push({
-        row: 0,
-        col: 0,
-        errorCode: 1003,
-        error: `Invalid PK/Index: First row must include 'PK' and 'Index' but found "${this.pkIndexLine}"`,
-        data: this.pkIndexLine,
+    if (this.pkIndexLine && this.pkIndexLine.length > 0) {
+      const errors = [];
+      let isAllEmpty = true;
+      this.pkIndexLine.forEach((index, ind) => {
+        console.log(index);
+        if (typeof index === 'string'){
+          index.trim();
+        }
+
+        if (index && index !== '' && index !== "PK" && index !== "Index") {
+          errors.push({
+            row: 0,
+            col: ind,
+            errorCode: 1003,
+            error: `Invalid PK/Index: First row must include 'PK' and 'Index' but found "${index}"`,
+            data: this.pkIndexLine,
+          });
+        } else if (index === "PK" || index === "Index") {
+          isAllEmpty = false;
+        }
       });
+
+      if (isAllEmpty) {
+        errors.push({
+          row: -1,
+          col: "common",
+          errorCode: 1003,
+          error: `Dimension should contain atleast one PK or Index Field`,
+          data: this.pkIndexLine,
+        });
+      }
+
+      return errors;
     }
-    return errors;
+
+    return [];
   }
 
   verifyDataTypes() {
     const errors = [];
     this.dataTypesLine.forEach((dataType, columnIndex) => {
-      if (dataType !== 'string' && dataType !== 'integer') {
+      if (dataType !== 'string' && dataType !== 'number' && dataType !== 'integer') {
         errors.push({
           row: 1,
           col: columnIndex,
           errorCode: 1002,
           error: `Invalid data type at column ${columnIndex + 1
-            }: Only 'string' and 'integer' are allowed but found '${dataType}'`,
+            }: Only 'string', 'number', and 'integer' are allowed but found '${dataType}'`,
           data: this.dataTypesLine,
         });
       }

diff --git a/src/ingestion/dto/request.ts b/src/ingestion/dto/request.ts
@@ -1,8 +1,11 @@
 export enum FileType {
-  DimensionGrammar = 'dimension-grammar',
-  DimensionData = 'dimension-data',
-  EventGrammar = 'event-grammar',
-  EventData = 'event-data',
+  Dimension = 'dimension',
+  Event = 'event',
+}
+
+export enum ValidationType {
+  Grammar = 'grammar',
+  Data = 'data',
 }
 
 export class FileValidateRequest {

diff --git a/src/ingestion/validators/event-grammar.validator.ts b/src/ingestion/validators/event-grammar.validator.ts
@@ -9,7 +9,7 @@ export class EventGrammarValidator {
   }
 
   verify() {
-    const errors: any[] = [];
+    let errors: any[] = [];
     // check for length
     const len = this.content.length;
     if (len !== 5) {
@@ -22,38 +22,76 @@ export class EventGrammarValidator {
     }
 
     // check for equal number of columns in each row
-    const colCount = [];
-    this.content.forEach((row) => {
-      colCount.push(row.split(',').length);
+    let prevColCount;
+    let colCountMatchFailed = false;
+    this.content.forEach((row, lineNo) => {
+      if (lineNo === 0) {
+        prevColCount = row.split(',').length;
+      }
+
+      if (lineNo > 0 && !colCountMatchFailed && prevColCount !== row.split(',').length) {
+        colCountMatchFailed = true;
+      }
+
+      if (lineNo === 2) {
+        row.split(',').forEach((dataType: string, index: number) => {
+          if (!['string', 'integer', 'date', 'number'].includes(dataType.trim())) {
+            errors.push({
+              row: lineNo,
+              col: index,
+              errorCode: 1002,
+              error: `Invalid data type: ${dataType}. Supported data types are: string, integer, date, number`,
+              data: dataType,
+            });
+          }
+        });
+      }
     });
 
-    if (colCount.every((val) => val === colCount[0]) === false) {
+    if (colCountMatchFailed) {
       errors.push({
         row: 0,
         col: 0,
         errorCode: 2003,
         error: 'Invalid CSV file: all rows should have equal number of columns',
       });
     }
+
     // TODO: add check for dimension names
+    const dimensionNameRow = this.content[0].split(',');
+    const dimensionKeyNameRow = this.content[1].split(',');
+    const dimensionErrors = dimensionNameRow.map((dimensionName: string, index: number) => {
+      if (dimensionName === '' && dimensionKeyNameRow[index] === '') {
+        return null;
+      } else if (dimensionName === '') {
+        return {
+          row: 0,
+          col: index,
+          errorCode: 1004,
+          error: `Dimension name is missing for dimension key:${dimensionKeyNameRow[index]}`,
+          data: this.content[0],
+        }
+      } else if (dimensionKeyNameRow[index] === '') {
+        return {
+          row: 0,
+          col: index,
+          errorCode: 1004,
+          error: `Dimension key name is missing for dimension:${dimensionName}`,
+          data: this.content[1],
+        }
+      }
+    }).filter(error => error !== null && error !== undefined);
+
+    if (dimensionErrors.length > 0) {
+      errors = [...errors, ...dimensionErrors];
+    }
 
     // TODO: add check for dimension key names
 
     // Check for supported data types
-    const dataTypeRow = this.content[2];
     // const dataTypeErrors = [];
 
-    dataTypeRow.split(',').forEach((dataType: string, index: number) => {
-      if (!['string', 'integer', 'date'].includes(dataType.trim())) {
-        errors.push({
-          row: 1,
-          col: index,
-          errorCode: 1002,
-          error: `Invalid data type: ${dataType}. Supported data types are: string, integer, date`,
-          data: dataType,
-        });
-      }
-    });
+
 
     // errors.push({
     //   row: 3,
@@ -72,7 +110,7 @@ export class EventGrammarValidator {
           row: 4,
           col: idx,
           errorCode: 1004,
-          error: `Dimension Grammar Specification Error: Wrong values in fieldType row, allowed values are 1. dimension 2.timeDimension 3. metric, but received ${item}`,
+          error: `Dimension Grammar Specification Error: Wrong values in fieldType row, allowed values are 'dimension', 'timeDimension', 'metric', but received ${item}`,
           data: lastRow,
         });
       } else if (item.trim() === 'dimension') {
@@ -88,7 +126,7 @@ export class EventGrammarValidator {
       .split(',')
       .map((item: string) => item.trim());
 
-    dimensionIdxs.forEach((idx: number) => {
+    /* dimensionIdxs.forEach((idx: number) => {
       if (fkKeysRow[idx] !== headerRow[idx]) {
         errors.push(
           {
@@ -107,7 +145,7 @@ export class EventGrammarValidator {
           },
         );
       }
-    });
+    });*/
 
     return errors;
   }

diff --git a/src/ingestion/validators/utilities.ts b/src/ingestion/validators/utilities.ts
@@ -0,0 +1,7 @@
+export function containsNonEmptyValues(arr) {
+    return arr.some(item => item !== null || item !== undefined || item !== '');
+}
+
+export function containsEmptyValues(arr) {
+    return arr.some(item => item === null || item === undefined || item === '');
+}