From c5189b814f881304e22a5694567f1dc27f182541 Mon Sep 17 00:00:00 2001 From: fraxken Date: Thu, 23 Nov 2023 22:57:18 +0100 Subject: [PATCH] feat: implement @sigyn/pattern to replace LogParser --- README.md | 16 +++- docs/LogParser.md | 67 -------------- docs/Loki.md | 33 ++++++- package-lock.json | 9 ++ package.json | 1 + src/class/LogParser.class.ts | 111 ---------------------- src/class/Loki.class.ts | 32 +++++-- src/index.ts | 1 - src/types.ts | 16 +++- test/LogParser.spec.ts | 172 ----------------------------------- test/Loki.spec.ts | 13 ++- 11 files changed, 91 insertions(+), 380 deletions(-) delete mode 100644 docs/LogParser.md delete mode 100644 src/class/LogParser.class.ts delete mode 100644 test/LogParser.spec.ts diff --git a/README.md b/README.md index e176a30..5b072cd 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,20 @@ const logs = await api.Loki.queryRange( console.log(logs); ``` +You can also provide a Loki pattern to automatically parse logs (and infer the right type with TypeScript) + +```ts +const logs = await api.Loki.queryRange( + `{app="serviceName", env="production"}` + { + pattern: " <_> " + } +); +for (const { verb, endpoint } of logs) { + console.log({verb, endpoint }); +} +``` + ## API ### GrafanaAPI @@ -85,8 +99,6 @@ export interface GrafanaApiOptions { - [Loki](./docs/Loki.md) - [Datasources](./docs/Datasources.md) -You can also parse logs using our internal [LogParser](./docs/LogParser.md) implementation. - ## Contributors ✨ diff --git a/docs/LogParser.md b/docs/LogParser.md deleted file mode 100644 index df51e4e..0000000 --- a/docs/LogParser.md +++ /dev/null @@ -1,67 +0,0 @@ -# LogParser - -> [!CAUTION] -> We are working on a new @sigyn/pattern package to replace this - -The LogParser class make easy to parse/type logs - -```ts -import { GrafanaApi, LogParser } from "@myunisoft/loki"; - -interface CustomParser { - date: string; - requestId: string; - endpoint: string; - method: string; - statusCode: number; -} - -const parser = new LogParser( - ": [req-] " -); - -const api = new GrafanaApi({ }); -const logs = await api.Loki.queryRange( - `{app="serviceName", env="production"}`, - { - parser - } -); -for (const data of logs) { - console.log(`requestId: ${data.requestId}`); -} -``` - -## Fields - -```ts -const kAvailableRegExField: Record = { - all: { - pattern: ".*", id: kNoopField - }, - num: { - pattern: "0-9", id: kOneOrManyField - }, - numStar: { - pattern: "0-9*", id: kOneOrManyField - }, - ip: { - pattern: "0-9.", id: kOneOrManyField - }, - word: { - pattern: "\\w", id: kOneOrManyField - }, - httpMethod: { - pattern: "GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS", id: kNoopField - }, - httpStatusCode: { - pattern: "0-9", id: (str) => `[${str}]{3}` - }, - alphanum: { - pattern: "a-zA-Z0-9", id: kOneOrManyField - }, - string: { - pattern: "_\\u00C0-\\u017F*\\w\\s-", id: kOneOrManyField - } -}; -``` diff --git a/docs/Loki.md b/docs/Loki.md index 9cba2ad..189e625 100644 --- a/docs/Loki.md +++ b/docs/Loki.md @@ -43,7 +43,7 @@ export interface LokiQueryOptions { start?: number | string; end?: number | string; since?: string; - parser?: LogParserLike; + pattern?: T | Array | ReadonlyArray; } ``` @@ -51,8 +51,8 @@ export interface LokiQueryOptions { The response is described by the following interface: ```ts -export interface QueryRangeResponse { - values: T[]; +export interface QueryRangeResponse { + values: LokiLiteralPattern[]; timerange: TimeRange | null; } ``` @@ -75,8 +75,8 @@ for (const { stream, values } of logs) { The response is described by the following interface: ```ts -interface QueryRangeStreamResponse { - logs: LokiStreamResult[]; +export interface QueryRangeStreamResponse { + logs: LokiStreamResult>[]; timerange: TimeRange | null; } @@ -178,3 +178,26 @@ async series>( ...match: [StreamSelector | string, ...(StreamSelector | string)[]] ): Promise ``` + +## Pattern usage +**queryRange** and **queryRangeStream** API allow the usage of pattern. + +```ts +import { GrafanaApi } from "@myunisoft/loki"; + +const api = new GrafanaApi({ + remoteApiURL: "https://name.loki.com" +}); + +await api.Loki.queryRange("...", { + pattern: " " +}); + +// or use an Array (tuple) +await api.Loki.queryRange("...", { + pattern: [ + " ", + " + ] as const +}); +``` diff --git a/package-lock.json b/package-lock.json index 0649528..0a52272 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@myunisoft/httpie": "^2.0.1", "@openally/auto-url": "^1.0.1", "@sigyn/logql": "^2.0.0", + "@sigyn/pattern": "^1.0.0", "dayjs": "^1.11.10", "ms": "^2.1.3" }, @@ -1285,6 +1286,14 @@ "node": ">=18" } }, + "node_modules/@sigyn/pattern": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@sigyn/pattern/-/pattern-1.0.0.tgz", + "integrity": "sha512-87GOtkYiEcN3qWtHpL8rBLJHErC0xkU7Ym/BOZB/6LWh3CkRVxcgSNfpUSaR1rVxSGLJbIUK4Ns4ZbUajFnFsg==", + "engines": { + "node": ">=18" + } + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", diff --git a/package.json b/package.json index cf811b2..c62791e 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "@myunisoft/httpie": "^2.0.1", "@openally/auto-url": "^1.0.1", "@sigyn/logql": "^2.0.0", + "@sigyn/pattern": "^1.0.0", "dayjs": "^1.11.10", "ms": "^2.1.3" } diff --git a/src/class/LogParser.class.ts b/src/class/LogParser.class.ts deleted file mode 100644 index 211ce1a..0000000 --- a/src/class/LogParser.class.ts +++ /dev/null @@ -1,111 +0,0 @@ -/* eslint-disable func-style */ - -// Import Internal Dependencies -import { escapeStringRegExp } from "../utils.js"; - -// CONSTANTS -const kNoopField = (str: string) => str; -const kOneOrManyField = (str: string) => `[${str}]+`; - -type RegExField = { - pattern: string; - id: (str: string) => string; -} - -const kAvailableRegExField: Record = { - all: { - pattern: ".*", id: kNoopField - }, - num: { - pattern: "0-9", id: kOneOrManyField - }, - numStar: { - pattern: "0-9*", id: kOneOrManyField - }, - ip: { - pattern: "0-9.", id: kOneOrManyField - }, - word: { - pattern: "\\w", id: kOneOrManyField - }, - httpMethod: { - pattern: "GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS", id: kNoopField - }, - httpStatusCode: { - pattern: "0-9", id: (str) => `[${str}]{3}` - }, - alphanum: { - pattern: "a-zA-Z0-9", id: kOneOrManyField - }, - string: { - pattern: "_\\u00C0-\\u017F*\\w\\s-", id: kOneOrManyField - } -}; - -export interface LogParserLike { - executeOnLogs(logs: string[]): T[]; -} - -export class NoopLogParser implements LogParserLike { - executeOnLogs(logs: string[]): T[] { - return logs as T[]; - } -} - -export class LogParser implements LogParserLike { - static RegExp() { - return /<([a-zA-Z0-9: ]+)>/g; - } - - private fields = new Map(); - private pattern: string; - - constructor(pattern: string | string[]) { - this.pattern = escapeStringRegExp( - Array.isArray(pattern) ? pattern.join("") : pattern - ); - } - - define( - fieldName: keyof T & string, - fieldKind: keyof typeof kAvailableRegExField, - additionalPattern = "" - ) { - const regexField = kAvailableRegExField[fieldKind]; - this.fields.set(fieldName, regexField.id(regexField.pattern + additionalPattern)); - - return this; - } - - compile(): (log: string) => [] | [log: T] { - const exprStr = this.pattern.replaceAll( - LogParser.RegExp(), - this.replacer.bind(this) - ); - - return (log) => { - const match = new RegExp(exprStr).exec(log); - - return match === null ? [] : [match.groups as T]; - }; - } - - executeOnLogs(logs: string[]): T[] { - return logs.flatMap(this.compile()); - } - - private replacer(_: string, matchingFieldOne: string) { - const [fieldName, fieldType = null] = matchingFieldOne.split(":"); - const trimmedFieldName = fieldName.trim(); - - if (fieldType !== null && fieldType in kAvailableRegExField) { - this.define(trimmedFieldName as any, fieldType.trim()); - } - - const pattern = this.fields.has(trimmedFieldName) ? - this.fields.get(trimmedFieldName)! : - kAvailableRegExField.all.pattern; - - return `(?<${trimmedFieldName}>${pattern})`; - } -} diff --git a/src/class/Loki.class.ts b/src/class/Loki.class.ts index 07469a4..1997df0 100644 --- a/src/class/Loki.class.ts +++ b/src/class/Loki.class.ts @@ -1,7 +1,16 @@ // Import Third-party Dependencies import * as httpie from "@myunisoft/httpie"; import autoURL from "@openally/auto-url"; -import { LogQL, StreamSelector } from "@sigyn/logql"; +import { + LogQL, + StreamSelector +} from "@sigyn/logql"; +import { + Pattern, + NoopPattern, + LokiPatternType, + PatternShape +} from "@sigyn/pattern"; // Import Internal Dependencies import * as utils from "../utils.js"; @@ -14,7 +23,6 @@ import { QueryRangeStreamResponse } from "../types.js"; import { ApiCredential } from "./ApiCredential.class.js"; -import { NoopLogParser, LogParserLike } from "./LogParser.class.js"; // CONSTANTS const kDurationTransformer = (value: string | number) => utils.durationToUnixTimestamp(value); @@ -33,8 +41,8 @@ interface LokiQueryBaseOptions { since?: string; } -export interface LokiQueryOptions extends LokiQueryBaseOptions { - parser?: LogParserLike; +export interface LokiQueryOptions extends LokiQueryBaseOptions { + pattern?: T; } export interface LokiLabelsOptions { @@ -104,11 +112,13 @@ export class Loki { ); } - async queryRangeStream( + async queryRangeStream( logQL: LogQL | string, options: LokiQueryOptions = {} ): Promise> { - const { parser = new NoopLogParser() } = options; + const { pattern = new NoopPattern() } = options; + const parser: PatternShape = pattern instanceof NoopPattern ? + pattern : new Pattern(pattern); const { data } = await this.#fetchQueryRange(logQL, options); @@ -116,18 +126,20 @@ export class Loki { logs: data.data.result.map((result) => { return { stream: result.stream, - values: result.values.flatMap(([, log]) => parser.executeOnLogs([log])) + values: result.values.flatMap(([, log]) => parser.executeOnLogs([log])) as any[] }; }), timerange: utils.queryRangeStreamTimeRange(data.data.result) }; } - async queryRange( + async queryRange( logQL: LogQL | string, options: LokiQueryOptions = {} ): Promise> { - const { parser = new NoopLogParser() } = options; + const { pattern = new NoopPattern() } = options; + const parser: PatternShape = pattern instanceof NoopPattern ? + pattern : new Pattern(pattern); const { data } = await this.#fetchQueryRange(logQL, options); @@ -139,7 +151,7 @@ export class Loki { } return { - values: parser.executeOnLogs(inlinedLogs.values), + values: parser.executeOnLogs(inlinedLogs.values) as any[], timerange: inlinedLogs.timerange }; } diff --git a/src/index.ts b/src/index.ts index 61328c2..d119591 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,3 +1,2 @@ export * from "./class/GrafanaApi.class.js"; -export * from "./class/LogParser.class.js"; export * from "./types"; diff --git a/src/types.ts b/src/types.ts index ba44afd..439fb05 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,9 @@ +// Import Third-party Dependencies +import { + LokiLiteralPattern, + LokiPatternType +} from "@sigyn/pattern"; + // Import Internal Dependencies import { TimeRange } from "./utils.js"; @@ -11,7 +17,7 @@ export interface LokiMatrix { values: [unixEpoch: string, value: string][]; } -export interface LokiStreamResult { +export interface LokiStreamResult { stream: Record; values: T[]; } @@ -63,13 +69,13 @@ export type LokiStandardBaseResponse = { data: S; } -export interface QueryRangeResponse { - values: T[]; +export interface QueryRangeResponse { + values: LokiLiteralPattern[]; timerange: TimeRange | null; } -export interface QueryRangeStreamResponse { - logs: LokiStreamResult[]; +export interface QueryRangeStreamResponse { + logs: LokiStreamResult>[]; timerange: TimeRange | null; } diff --git a/test/LogParser.spec.ts b/test/LogParser.spec.ts deleted file mode 100644 index 0e09d96..0000000 --- a/test/LogParser.spec.ts +++ /dev/null @@ -1,172 +0,0 @@ -// Import Node.js Dependencies -import { describe, it, test } from "node:test"; -import assert from "node:assert"; - -// Import Internal Dependencies -import { NoopLogParser, LogParser } from "../src/class/LogParser.class.js"; - -describe("NoopLogParser", () => { - it("should return Array of logs without any modification (same ref)", () => { - const input = ["A", "B"]; - - const noop = new NoopLogParser(); - const result = noop.executeOnLogs(input); - assert.strictEqual(input, result); - }); -}); - -describe("LogParser", () => { - test("constructor must accept array of patterns", () => { - const parser = new LogParser<{ A: number, B: number, name: string }>([ - "[scope: |]", - "[name: " - ]); - - const logs = parser.executeOnLogs([ - "[scope: 10|20][name: Thomas]" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual(parsedLog, { A: 10, B: 20, name: "Thomas" }); - }); - - describe("define", () => { - it("should assign the right field type and parse it successfully", () => { - const parser = new LogParser<{ A: number, B: number }>("[scope: |]") - .define("A", "num") - .define("B", "num"); - - const logs = parser.executeOnLogs([ - "[scope: 10|20]" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual(parsedLog, { A: 10, B: 20 }); - }); - - it("should add an additional pattern to the selected type", () => { - const parser = new LogParser<{ A: number, B: number | "*" }>("[scope: |]") - .define("A", "num") - .define("B", "num", "*"); - - const logs = parser.executeOnLogs([ - "[scope: 10|*]" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual(parsedLog, { A: 10, B: "*" }); - }); - - it("should trim pattern name and type", () => { - const parser = new LogParser<{ A: number, B: number }>( - "[scope: < A : num >|< B : num >]" - ); - - const logs = parser.executeOnLogs([ - "[scope: 10|20]" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual(parsedLog, { A: 10, B: 20 }); - }); - }); - - describe("compile", () => { - it("should return a function that we can execute on one log at a time", () => { - const parser = new LogParser<{ A: number, B: number }>("[scope: |]"); - - const parseLog = parser.compile(); - assert.deepEqual( - parseLog("[scope: 10|20]"), - [{ A: 10, B: 20 }] - ); - }); - - it("should return an empty Array if the log doesn't match the pattern", () => { - const parser = new LogParser<{ A: number, B: number }>("[scope: |]"); - - const parseLog = parser.compile(); - assert.deepEqual( - parseLog("hello world"), - [] - ); - }); - }); - - describe("RegExp fields", () => { - it("should be able to parse 'all', 'httpMethod' and `httpStatusCode` fields", () => { - const parser = new LogParser<{ - method: string, endpoint: string, statusCode: number - }>(" "); - - const logs = parser.executeOnLogs([ - "GET https://github.com/OpenAlly 204" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual( - parsedLog, - { - method: "GET", - endpoint: "https://github.com/OpenAlly", - statusCode: 204 - } - ); - }); - - it("should be able to parse a 'num' field", () => { - const parser = new LogParser<{ A: number, B: number }>("[scope: |]"); - - const logs = parser.executeOnLogs([ - "[scope: 10|20]" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual(parsedLog, { A: 10, B: 20 }); - }); - - it("should be able to parse a 'numStar' field", () => { - const parser = new LogParser<{ - A: number | "*", B: number | "*" - }>("[scope: |]"); - - const logs = parser.executeOnLogs([ - "[scope: *|*]" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual(parsedLog, { A: "*", B: "*" }); - }); - - it("should be able to parse an 'ip' field", () => { - const parser = new LogParser<{ ip: string }>(""); - - const logs = parser.executeOnLogs([ - "127.0.0.1" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual(parsedLog, { ip: "127.0.0.1" }); - }); - - it("should be able to parse a 'string' field", () => { - const parser = new LogParser<{ ip: string }>("Introduction... "); - - const logs = parser.executeOnLogs([ - "Introduction... Bienvenue-à_toi" - ]); - assert.strictEqual(logs.length, 1); - - const [parsedLog] = logs; - assert.deepEqual(parsedLog, { phrase: "Bienvenue-à_toi" }); - }); - }); -}); diff --git a/test/Loki.spec.ts b/test/Loki.spec.ts index 325e031..a3bdcef 100644 --- a/test/Loki.spec.ts +++ b/test/Loki.spec.ts @@ -8,7 +8,6 @@ import { MockAgent, setGlobalDispatcher, getGlobalDispatcher } from "@myunisoft/ // Import Internal Dependencies import { GrafanaApi, - LogParser, LokiStandardBaseResponse, RawQueryRangeResponse } from "../src/index.js"; @@ -127,8 +126,8 @@ describe("GrafanaApi.Loki", () => { const sdk = new GrafanaApi({ remoteApiURL: kDummyURL }); - const result = await sdk.Loki.queryRange<{ name: string }>("{app='foo'}", { - parser: new LogParser("hello ''") + const result = await sdk.Loki.queryRange("{app='foo'}", { + pattern: "hello ''" }); assert.strictEqual(result.values.length, 1); assert.deepEqual( @@ -150,8 +149,8 @@ describe("GrafanaApi.Loki", () => { const sdk = new GrafanaApi({ remoteApiURL: kDummyURL }); - const result = await sdk.Loki.queryRangeStream<{ name: string }>("{app='foo'}", { - parser: new LogParser("hello ''") + const result = await sdk.Loki.queryRangeStream("{app='foo'}", { + pattern: "hello ''" }); assert.strictEqual(result.logs.length, 1); @@ -175,8 +174,8 @@ describe("GrafanaApi.Loki", () => { const sdk = new GrafanaApi({ remoteApiURL: kDummyURL }); - const result = await sdk.Loki.queryRangeStream<{ name: string }>("{app='foo'}", { - parser: new LogParser("hello ''") + const result = await sdk.Loki.queryRangeStream("{app='foo'}", { + pattern: "hello ''" }); assert.deepEqual(