From db8f2e6b9ca5547c912c28bec17a0778e19200c6 Mon Sep 17 00:00:00 2001 From: "Thomas.G" Date: Sun, 30 Jun 2024 23:45:04 +0200 Subject: [PATCH] feat: add queryRangeMatrix & add missing Unix timestamp (#167) --- docs/Loki.md | 53 ++++++++++++++++++++++++++++++------ src/class/Loki.class.ts | 32 +++++++++++++++++++--- src/types.ts | 16 ++++++++--- src/utils.ts | 16 ++++++----- test/Loki.spec.ts | 60 ++++++++++++++++++++++++++++++++++++----- 5 files changed, 150 insertions(+), 27 deletions(-) diff --git a/docs/Loki.md b/docs/Loki.md index 189e625..d7566bd 100644 --- a/docs/Loki.md +++ b/docs/Loki.md @@ -14,7 +14,8 @@ const api = new GrafanaApi({ await api.Loki.series(`{env="production"}`); ``` -Note that a TimeRange is defined as following: +Note that a TimeRange is defined as follows: + ```ts export type TimeRange = [first: number, last: number]; ``` @@ -23,7 +24,9 @@ export type TimeRange = [first: number, last: number]; ### queryRange< T = string >(logQL: LogQL | string, options?: LokiQueryOptions< T >): Promise< QueryRangeResponse< T > > -Note that you can provide a custom parser to queryRange (by default it inject a NoopParser doing nothing). +The `queryRange` method returns raw logs (without timestamp or metric/stream labels). + +You can provide a custom parser to queryRange (by default it injects a **NoopParser** doing nothing). ```ts const logs = await api.Loki.queryRange( @@ -32,7 +35,9 @@ const logs = await api.Loki.queryRange( console.log(logs); ``` -queryRange options is described by the following TypeScript interface +#### options + +The queryRange options are described by the following TypeScript interface: ```ts export interface LokiQueryOptions { @@ -47,7 +52,9 @@ export interface LokiQueryOptions { } ``` -start and end arguments can be either a unix timestamp or a duration like `6h`. +start and end arguments can be either a Unix timestamp or a duration like `6h`. + +#### response The response is described by the following interface: ```ts @@ -57,6 +64,11 @@ export interface QueryRangeResponse { } ``` +`timerange` is **null** when there are no logs available with the given LogQL. + +> [!CAUTION] +> When you use an incorrect pattern, any logs that are not correctly parsed will be removed from the result. + ### queryRangeStream< T = string >(logQL: LogQL | string, options?: LokiQueryOptions< T >): Promise< QueryRangeStreamResponse< T > > Same as `queryRange` but returns the labels key-value pairs stream @@ -68,21 +80,46 @@ const logs = await api.Loki.queryRangeStream( for (const { stream, values } of logs) { // Record console.log(stream); - // string[] + // [unixEpoch: number, value: T][] console.log(values); } ``` The response is described by the following interface: ```ts -export interface QueryRangeStreamResponse { +interface QueryRangeStreamResponse { logs: LokiStreamResult>[]; timerange: TimeRange | null; } -interface LokiStreamResult { +interface LokiStreamResult { stream: Record; - values: T[]; + values: [unixEpoch: number, value: T][]; +} +``` + +### queryRangeMatrix< T = string >(logQL: LogQL | string, options?: LokiQueryOptions< T >): Promise< QueryRangeMatrixResponse< T > > + +Same as `queryRange` but returns the labels key-value pairs metric + +Matrix are returned for [Metric queries](https://grafana.com/docs/loki/latest/query/metric_queries/) +``` +count_over_time({ label="value" }[5m]) +``` + +--- + +The response is described by the following interface: + +```ts +interface QueryRangeMatrixResponse { + logs: LokiMatrixResult>[]; + timerange: TimeRange | null; +} + +interface LokiMatrixResult { + metric: Record; + values: [unixEpoch: number, value: T][]; } ``` diff --git a/src/class/Loki.class.ts b/src/class/Loki.class.ts index 1997df0..b79b347 100644 --- a/src/class/Loki.class.ts +++ b/src/class/Loki.class.ts @@ -20,7 +20,8 @@ import { LokiStream, LokiMatrix, QueryRangeResponse, - QueryRangeStreamResponse + QueryRangeStreamResponse, + QueryRangeMatrixResponse } from "../types.js"; import { ApiCredential } from "./ApiCredential.class.js"; @@ -112,6 +113,29 @@ export class Loki { ); } + async queryRangeMatrix( + logQL: LogQL | string, + options: LokiQueryOptions = {} + ): Promise> { + const { pattern = new NoopPattern() } = options; + const parser: PatternShape = pattern instanceof NoopPattern ? + pattern : new Pattern(pattern); + + const { data } = await this.#fetchQueryRange(logQL, options); + + return { + logs: data.data.result.map((result) => { + return { + metric: result.metric, + values: result.values + .map(([unixEpoch, log]) => [unixEpoch, ...parser.executeOnLogs([log])]) + .filter((log) => log.length > 1) as any[] + }; + }), + timerange: utils.streamOrMatrixTimeRange(data.data.result) + }; + } + async queryRangeStream( logQL: LogQL | string, options: LokiQueryOptions = {} @@ -126,10 +150,12 @@ export class Loki { logs: data.data.result.map((result) => { return { stream: result.stream, - values: result.values.flatMap(([, log]) => parser.executeOnLogs([log])) as any[] + values: result.values + .map(([unixEpoch, log]) => [unixEpoch, ...parser.executeOnLogs([log])]) + .filter((log) => log.length > 1) as any[] }; }), - timerange: utils.queryRangeStreamTimeRange(data.data.result) + timerange: utils.streamOrMatrixTimeRange(data.data.result) }; } diff --git a/src/types.ts b/src/types.ts index 439fb05..d4fb9b2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -9,17 +9,22 @@ import { TimeRange } from "./utils.js"; export interface LokiStream { stream: Record; - values: [unixEpoch: string, log: string][]; + values: [unixEpoch: number, log: string][]; } export interface LokiMatrix { metric: Record; - values: [unixEpoch: string, value: string][]; + values: [unixEpoch: number, value: string][]; +} + +export interface LokiMatrixResult { + metric: Record; + values: [unixEpoch: number, value: T][]; } export interface LokiStreamResult { stream: Record; - values: T[]; + values: [unixEpoch: number, value: T][]; } export interface RawQueryRangeResponse { @@ -79,4 +84,9 @@ export interface QueryRangeStreamResponse { timerange: TimeRange | null; } +export interface QueryRangeMatrixResponse { + logs: LokiMatrixResult>[]; + timerange: TimeRange | null; +} + export { TimeRange }; diff --git a/src/utils.ts b/src/utils.ts index 85bdc4f..8b16fe3 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,5 @@ // Import Third-party Dependencies -import dayjs from "dayjs"; +import dayjs, { Dayjs } from "dayjs"; import ms from "ms"; // Import Internal Dependencies @@ -23,9 +23,9 @@ export function escapeStringRegExp(str: string): string { return str.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&"); } -export function transformStreamValue( - value: [unixEpoch: string, log: string] -) { +export function transformStreamOrMatrixValue( + value: [unixEpoch: number, log: string] +): { date: Dayjs; log: string } { const [unixEpoch, log] = value; return { @@ -45,7 +45,7 @@ export function inlineLogs( const flatLogs = result.data.result .flatMap( - (host) => host.values.map(transformStreamValue) + (host) => host.values.map(transformStreamOrMatrixValue) ) .sort((left, right) => (left.date.isBefore(right.date) ? 1 : -1)); if (flatLogs.length === 0) { @@ -58,14 +58,16 @@ export function inlineLogs( }; } -export function queryRangeStreamTimeRange(result: LokiStream[]): [number, number] | null { +export function streamOrMatrixTimeRange( + result: (LokiStream | LokiMatrix)[] +): [number, number] | null { if (result.length === 0) { return null; } const flatLogs = result .flatMap( - (host) => host.values.map(transformStreamValue) + (host) => host.values.map(transformStreamOrMatrixValue) ) .sort((left, right) => (left.date.isBefore(right.date) ? 1 : -1)); diff --git a/test/Loki.spec.ts b/test/Loki.spec.ts index a3bdcef..45877b7 100644 --- a/test/Loki.spec.ts +++ b/test/Loki.spec.ts @@ -9,7 +9,8 @@ import { MockAgent, setGlobalDispatcher, getGlobalDispatcher } from "@myunisoft/ import { GrafanaApi, LokiStandardBaseResponse, - RawQueryRangeResponse + RawQueryRangeResponse, + LokiMatrix } from "../src/index.js"; // CONSTANTS @@ -84,12 +85,42 @@ describe("GrafanaApi.Loki", () => { const sdk = new GrafanaApi({ remoteApiURL: kDummyURL }); const result = await sdk.Loki.queryRangeStream("{app='foo'}"); + const resultLogs = result.logs[0]!; + assert.ok( + resultLogs.values.every((arr) => typeof arr[0] === "number") + ); + assert.deepEqual( + resultLogs.values.map((arr) => arr[1]), + expectedLogs.slice(0) + ); + assert.deepEqual(resultLogs.stream, { foo: "bar" }); + }); + + it("should return expectedLogs with no modification (using NoopParser, queryRangeMatrix)", async() => { + const expectedLogs = ["hello world", "foobar"]; + + agentPoolInterceptor + .intercept({ + path: (path) => path.includes("loki/api/v1/query_range") + }) + .reply(200, mockMatrixResponse(expectedLogs), { + headers: { "Content-Type": "application/json" } + }); + + const sdk = new GrafanaApi({ remoteApiURL: kDummyURL }); + + const result = await sdk.Loki.queryRangeMatrix("{app='foo'}"); + const resultLogs = result.logs[0]!; + + assert.ok( + resultLogs.values.every((arr) => typeof arr[0] === "number") + ); assert.deepEqual( - result.logs[0].values, + resultLogs.values.map((arr) => arr[1]), expectedLogs.slice(0) ); - assert.deepEqual(result.logs[0].stream, { foo: "bar" }); + assert.deepEqual(resultLogs.metric, { foo: "bar" }); }); it("should return empty list of logs (using NoopParser, queryRangeStream)", async() => { @@ -152,13 +183,14 @@ describe("GrafanaApi.Loki", () => { const result = await sdk.Loki.queryRangeStream("{app='foo'}", { pattern: "hello ''" }); + const resultLogs = result.logs[0]!; assert.strictEqual(result.logs.length, 1); assert.deepEqual( - result.logs[0].values[0], + resultLogs.values[0][1], { name: "Thomas" } ); - assert.deepEqual(result.logs[0].stream, { foo: "bar" }); + assert.deepEqual(resultLogs.stream, { foo: "bar" }); }); it("should return empty list of logs (using LogParser, queryRangeStream)", async() => { @@ -324,6 +356,22 @@ function mockStreamResponse(logs: string[]): DeepPartial }; } +function mockMatrixResponse(logs: string[]): DeepPartial> { + return { + status: "success", + data: { + resultType: "matrix", + result: logs.length > 0 ? [ + { + metric: { foo: "bar" }, + values: logs.map((log) => [getNanoSecTime(), log]) + } + ] : [], + stats: {} + } + }; +} + function mockLabelResponse( status: "success" | "failed", response: T[] @@ -337,6 +385,6 @@ function mockLabelResponse( function getNanoSecTime() { const hrTime = process.hrtime(); - return String((hrTime[0] * 1000000000) + hrTime[1]); + return (hrTime[0] * 1000000000) + hrTime[1]; }