From b56e2242922259a34af475b37a393b4bc089d45d Mon Sep 17 00:00:00 2001 From: amonsour Date: Wed, 17 Jul 2024 19:35:30 +0000 Subject: [PATCH 1/4] [CustomReport] Chunked the URL queries by 50 for averting crashes and gateway timeouts --- libs/api/custom-reports/src/lib/custom-reports.strategies.ts | 5 ++++- libs/utils-common/src/lib/utils-common.ts | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/api/custom-reports/src/lib/custom-reports.strategies.ts b/libs/api/custom-reports/src/lib/custom-reports.strategies.ts index 44e73a5c2..a46cef6a4 100644 --- a/libs/api/custom-reports/src/lib/custom-reports.strategies.ts +++ b/libs/api/custom-reports/src/lib/custom-reports.strategies.ts @@ -12,6 +12,7 @@ import type { } from '@dua-upd/types-common'; import { arrayToDictionary, + chunkMap, dateRangeToGranularity, } from '@dua-upd/utils-common'; import type { ReportDataPoint } from './custom-reports.service'; @@ -101,10 +102,12 @@ export function decomposeConfig(config: ReportConfig) { urls, }) as AAQueryConfig; + const chunkedUrls = chunkMap(urls, (url) => url, 50); + const queries = dateRanges.flatMap((dateRange) => !grouped && breakdownDimension ? urls.map((url) => toQueryConfig(dateRange, [url])) - : [toQueryConfig(dateRange, urls)], + : chunkedUrls.map((url) => toQueryConfig(dateRange, url)), ); return queries.map((query) => ({ diff --git a/libs/utils-common/src/lib/utils-common.ts b/libs/utils-common/src/lib/utils-common.ts index a053f2123..58fbc33f3 100644 --- a/libs/utils-common/src/lib/utils-common.ts +++ b/libs/utils-common/src/lib/utils-common.ts @@ -358,6 +358,7 @@ export function chunkMap( mapFunc: (val: T[]) => ReturnT, chunkSize: number, ): ReturnT[] { + array = array.slice(); const chunks = []; while (array.length) { From 93e4b043d56b6064107f2a60ce28ddd351277b91 Mon Sep 17 00:00:00 2001 From: amonsour Date: Thu, 18 Jul 2024 14:12:46 +0000 Subject: [PATCH 2/4] [Data Table] Implemented a customSort function; Nulls are sorted last in all cases --- .../lib/data-table/data-table.component.html | 2 + .../lib/data-table/data-table.component.ts | 41 +++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/libs/upd/components/src/lib/data-table/data-table.component.html b/libs/upd/components/src/lib/data-table/data-table.component.html index 8940e2d88..3ccce9b50 100644 --- a/libs/upd/components/src/lib/data-table/data-table.component.html +++ b/libs/upd/components/src/lib/data-table/data-table.component.html @@ -5,6 +5,7 @@ [rows]="displayRows" [rowHover]="true" [showCurrentPageReport]="true" + [customSort]="true" [rowsPerPageOptions]="[5, 10, 25, 50, 100]" [loading]="loading" styleClass="p-datatable-striped p-datatable-sm border" @@ -19,6 +20,7 @@ [sortOrder]="sortOrder === 'desc' ? -1 : 1" dataKey="_id" (selectionChange)="onSelectionChange($event)" + (sortFunction)="customSort($event)" [(selection)]="selectedRows" > diff --git a/libs/upd/components/src/lib/data-table/data-table.component.ts b/libs/upd/components/src/lib/data-table/data-table.component.ts index 979b0e5d9..99bd639c9 100644 --- a/libs/upd/components/src/lib/data-table/data-table.component.ts +++ b/libs/upd/components/src/lib/data-table/data-table.component.ts @@ -16,6 +16,7 @@ import { Table } from 'primeng/table'; import type { ColumnConfig } from '@dua-upd/types-common'; import type { SelectedNode } from '../filter-table/filter-table.component'; import { toGroupedColumnSelect } from '@dua-upd/upd/utils'; +import { SortEvent } from 'primeng/api'; @Component({ selector: 'upd-data-table', @@ -40,7 +41,7 @@ export class DataTableComponent { @Input() placeholderText = 'dt_search_keyword'; @Input() selectedNodes: SelectedNode[] = []; @Input() allowHeaderWrap = false; - + node: SelectedNode | null = null; @Output() rowSelectionChanged = new EventEmitter(); @@ -64,9 +65,13 @@ export class DataTableComponent { ); selectedColumnsSynced = computed(() => { - const selectedColumnFields = this.selectedColumns().map(({ field }) => field); + const selectedColumnFields = this.selectedColumns().map( + ({ field }) => field, + ); - return this.cols().filter((col) => selectedColumnFields.includes(col.field)); + return this.cols().filter((col) => + selectedColumnFields.includes(col.field), + ); }); searchFields = computed(() => @@ -91,10 +96,14 @@ export class DataTableComponent { lang = this.i18n.currentLang; + isSorted: boolean | null = null; + selectableCols = computed(() => { const cols = this.cols() .filter((col: ColumnConfig) => !col.frozen) - .sort((a: ColumnConfig, b: ColumnConfig) => a.header.localeCompare(b.header)); + .sort((a: ColumnConfig, b: ColumnConfig) => + a.header.localeCompare(b.header), + ); if (this.groupedColumnSelection()) { return toGroupedColumnSelect(cols); @@ -162,4 +171,28 @@ export class DataTableComponent { this.selectedColumns.set(selectedColumns); } + + customSort(event: SortEvent) { + event.data?.sort((a, b) => { + a = a[event.field as keyof typeof a]; + b = b[event.field as keyof typeof b]; + const order = event.order === 1 ? 1 : -1; + + let result = 0; + + if ((a == null && b != null) || (a === '' && b !== '')) { + result = order === 1 ? 1 : -1; + } else if ((a != null && b == null) || (a !== '' && b === '')) { + result = order === 1 ? -1 : 1; + } else if ((a == null && b == null) || (a === '' && b === '')) { + result = 0; + } else if (typeof a === 'string' && typeof b === 'string') { + result = a.localeCompare(b); + } else { + result = a < b ? -1 : a > b ? 1 : 0; + } + + return order * result; + }); + } } From 6ea7c72b1944391890469cc9c1b80c3fec991835 Mon Sep 17 00:00:00 2001 From: amonsour Date: Thu, 18 Jul 2024 14:33:38 +0000 Subject: [PATCH 3/4] [Custom Report : Query service] Updated the pages to grab and sort the visits --- libs/api/query/src/lib/query.service.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/libs/api/query/src/lib/query.service.ts b/libs/api/query/src/lib/query.service.ts index ef08abec7..80758dc4e 100644 --- a/libs/api/query/src/lib/query.service.ts +++ b/libs/api/query/src/lib/query.service.ts @@ -1,7 +1,8 @@ -import { DbService } from '@dua-upd/db'; +import { DbService, Page } from '@dua-upd/db'; import { Injectable } from '@nestjs/common'; import { Model, Types, isValidObjectId } from 'mongoose'; import type { DbQuery } from '@dua-upd/types-common'; +import dayjs from 'dayjs'; @Injectable() export class QueryService { @@ -10,6 +11,11 @@ export class QueryService { async getData(serializedQueries: { [key: string]: string }) { const results: { [key: string]: unknown } = {}; + const queryDateRange = { + start: dayjs().subtract(1, 'week').startOf('week').toDate(), + end: dayjs().subtract(1, 'week').endOf('week').toDate(), + }; + for (const key in serializedQueries) { const query: DbQuery[keyof DbQuery] = JSON.parse( atob(serializedQueries[key]), @@ -26,11 +32,17 @@ export class QueryService { // large queries on this collection can cripple the db, so we'll limit the results like this for now const limit = collection === 'pageMetrics' ? 1000 : undefined; - results[key] = await collectionModel - .find(filter, project, { limit }) - .sort(sort) - .lean() - .exec(); + results[key] = + collection === 'pages' + ? await this.db.views.pageVisits.getVisitsWithPageData( + queryDateRange, + collectionModel as Model, + ) + : await collectionModel + .find(filter, project, { limit }) + .sort(sort) + .lean() + .exec(); } return results; From 5eee186a3cedf23c92436ac625cb146d38eedffc Mon Sep 17 00:00:00 2001 From: amonsour Date: Thu, 18 Jul 2024 16:02:26 +0000 Subject: [PATCH 4/4] Update data-table.component.ts --- .../lib/data-table/data-table.component.ts | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/libs/upd/components/src/lib/data-table/data-table.component.ts b/libs/upd/components/src/lib/data-table/data-table.component.ts index 99bd639c9..6a0e5ecb9 100644 --- a/libs/upd/components/src/lib/data-table/data-table.component.ts +++ b/libs/upd/components/src/lib/data-table/data-table.component.ts @@ -17,6 +17,7 @@ import type { ColumnConfig } from '@dua-upd/types-common'; import type { SelectedNode } from '../filter-table/filter-table.component'; import { toGroupedColumnSelect } from '@dua-upd/upd/utils'; import { SortEvent } from 'primeng/api'; +import { isNullish } from '@dua-upd/utils-common'; @Component({ selector: 'upd-data-table', @@ -176,23 +177,24 @@ export class DataTableComponent { event.data?.sort((a, b) => { a = a[event.field as keyof typeof a]; b = b[event.field as keyof typeof b]; + const order = event.order === 1 ? 1 : -1; - let result = 0; - - if ((a == null && b != null) || (a === '' && b !== '')) { - result = order === 1 ? 1 : -1; - } else if ((a != null && b == null) || (a !== '' && b === '')) { - result = order === 1 ? -1 : 1; - } else if ((a == null && b == null) || (a === '' && b === '')) { - result = 0; - } else if (typeof a === 'string' && typeof b === 'string') { - result = a.localeCompare(b); - } else { - result = a < b ? -1 : a > b ? 1 : 0; + // if a is nullish, it goes to the end + if (isNullish(a) || a === '') { + return 1; + } + + // if b is nullish, it goes to the end + if (isNullish(b) || b === '') { + return -1; + } + + if (typeof a === 'string') { + return a.localeCompare(b) * order; } - return order * result; + return (a - b) * order; }); } }