Skip to content

Commit

Permalink
Fetch runs CSV from server on export runs (#40)
Browse files Browse the repository at this point in the history 
Fetch csv file of exported runs from server instead of creating it on the client.
  • Loading branch information
@fabiovincenzi
fabiovincenzi authored Feb 2, 2024
1 parent cbd531c commit 7537110
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 114 deletions.
8 changes: 8 additions & 0 deletions src/src/services/api/runs/runsService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,13 @@ function getRunsData(query?: string, limit?: number, offset?: string) {
});
}

function getCsvData(query?: string) {
return API.getStream<ReadableStream>(endpoints.GET_RUNS, {
q: query || '',
action: 'export',
});
}

function getRunLogs(id: string, record_range?: string) {
return API.getStream<ReadableStream>(endpoints.GET_RUN_LOGS(id), {
record_range: record_range ?? '',
Expand Down Expand Up @@ -111,6 +118,7 @@ const runsService = {
getBatch,
getBatchByStep,
getRunsData,
getCsvData,
getRunInfo,
getRunLogs,
getRunMetricsBatch,
Expand Down
131 changes: 17 additions & 114 deletions src/src/services/models/explorer/createAppModel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2272,46 +2272,6 @@ function createAppModel(appConfig: IAppInitialConfig) {
onRunsTagsChange({ runHash, tags, model, updateModelData });
}

function getAllRunsData(queryString?: string): {
call: (exceptionHandler: (detail: any) => void) => Promise<any>;
abort: () => void;
} {
if (runsRequestRef) {
runsRequestRef.abort();
}
runsRequestRef = runsService.getRunsData(queryString);
return {
call: async () => {
try {
const stream = await runsRequestRef.call((detail) => {
exceptionHandler({ detail, model });
});
let bufferPairs = decodeBufferPairs(
stream as ReadableStream<any>,
);
let decodedPairs = decodePathsVals(bufferPairs);
let objects = iterFoldTree(decodedPairs, 1);
const runsData: IRun<IMetricTrace | IParamTrace>[] = [];

for await (let [keys, val] of objects) {
const data = { ...(val as any), hash: keys[0] };
if (!data.hash.startsWith('progress')) {
const runData: any = val;
runsData.push({ ...runData, hash: keys[0] } as any);
}
}
return runsData;
} catch (ex: Error | any) {
if (ex.name === 'AbortError') {
// eslint-disable-next-line no-console
console.error(`${ex.name}, ${ex.message}`);
}
}
},
abort: runsRequestRef.abort,
};
}

function getRunsData(
shouldUrlUpdate?: boolean,
shouldResetSelectedRows?: boolean,
Expand Down Expand Up @@ -2938,81 +2898,24 @@ function createAppModel(appConfig: IAppInitialConfig) {
}
}

function onExportTableData(): Promise<void> {
// @TODO need to get data and params from state not from processData
const runsDataToExport = getAllRunsData(
model.getState()?.config?.select?.query,
);
const exceptionHandler = (detail: any) => {
// eslint-disable-next-line no-console
console.error('An error occurred:', detail);
};

return runsDataToExport.call(exceptionHandler).then((rawData) => {
const { data, params, metricsColumns } = processData(rawData);
const tableData = getDataAsTableRows(
data,
metricsColumns,
params,
true,
);
const configData = model.getState()?.config;
const tableColumns: ITableColumn[] = getRunsTableColumns(
metricsColumns,
params,
configData?.table.columnsOrder!,
configData?.table.hiddenColumns!,
);
const excludedFields: string[] = ['#', 'actions'];
const filteredHeader: string[] = tableColumns.reduce(
(acc: string[], column: ITableColumn) =>
acc.concat(
excludedFields.indexOf(column.key) === -1 && !column.isHidden
? column.key
: [],
),
[],
);

let emptyRow: { [key: string]: string } = {};
filteredHeader.forEach((column: string) => {
emptyRow[column] = '--';
});

const groupedRows: IMetricTableRowData[][] =
data.length > 1
? Object.keys(tableData.rows).map(
(groupedRowKey: string) =>
tableData.rows[groupedRowKey].items,
)
: [
Array.isArray(tableData.rows)
? tableData.rows
: tableData.rows[Object.keys(tableData.rows)[0]].items,
];

const dataToExport: { [key: string]: string }[] = [];

groupedRows?.forEach(
(groupedRow: IMetricTableRowData[], groupedRowIndex: number) => {
groupedRow?.forEach((row: IMetricTableRowData) => {
const filteredRow = getFilteredRow({
columnKeys: filteredHeader,
row,
});
dataToExport.push(filteredRow);
});
if (groupedRows?.length - 1 !== groupedRowIndex) {
dataToExport.push(emptyRow);
}
},
);
const blob = new Blob([JsonToCSV(dataToExport)], {
type: 'text/csv;charset=utf-8;',
});
saveAs(blob, `runs-${moment().format(DATE_EXPORTING_FORMAT)}.csv`);
analytics.trackEvent(ANALYTICS_EVENT_KEYS[appName].table.exports.csv);
async function onExportTableData(): Promise<void> {
const query = model.getState()?.config?.select?.query;
const request = runsService.getCsvData(query);
const readableStream = await request.call((detail) => {
exceptionHandler({ detail, model });
});
const reader = readableStream.getReader();
const data = [];
let readResult = await reader.read();
while (!readResult.done) {
data.push(readResult.value);
readResult = await reader.read();
}
const blob = new Blob(data, {
type: 'text/csv;charset=utf-8;',
});
saveAs(blob, `runs-${moment().format(DATE_EXPORTING_FORMAT)}.csv`);
analytics.trackEvent(ANALYTICS_EVENT_KEYS[appName].table.exports.csv);
}

function onModelNotificationDelete(id: number): void {
Expand Down

0 comments on commit 7537110

Please sign in to comment.