diff --git a/defaultFilters/apprisk/azure-repos.json b/defaultFilters/apprisk/azure-repos.json index 7505d6765..88de38725 100644 --- a/defaultFilters/apprisk/azure-repos.json +++ b/defaultFilters/apprisk/azure-repos.json @@ -219,25 +219,6 @@ "token": "${BROKER_CLIENT_VALIDATION_BASIC_AUTH}" } }, - { - "//": "get organization context from file", - "method": "GET", - "path": "/:org/_apis/git/repositories/:repo/items", - "origin": "https://${AZURE_REPOS_HOST}", - "auth": { - "scheme": "basic", - "token": "${BROKER_CLIENT_VALIDATION_BASIC_AUTH}" - }, - "valid": [ - { - "queryParam": "path", - "values": [ - "**/*.json", - "**%2F*.json" - ] - } - ] - }, { "//": "get organization context from file without org", "method": "GET", diff --git a/defaultFilters/azure-repos.json b/defaultFilters/azure-repos.json index 9c6a80adb..7a3da5bb6 100644 --- a/defaultFilters/azure-repos.json +++ b/defaultFilters/azure-repos.json @@ -149,7 +149,11 @@ "**/go.sum", "**%2Fgo.sum", "**/*Dockerfile*", - "**%2F*Dockerfile*" + "**%2F*Dockerfile*", + "**/*.json", + "**%2F*.json", + "**/.azuredevops/snyk_pull_request_template.yaml", + "**%2F.azuredevops%2Fsnyk_pull_request_template.yaml" ] }, { @@ -242,7 +246,11 @@ "**/go.sum", "**%2Fgo.sum", "**/*Dockerfile*", - "**%2F*Dockerfile*" + "**%2F*Dockerfile*", + "**/*.json", + "**%2F*.json", + "**/.azuredevops/snyk_pull_request_template.yaml", + "**%2F.azuredevops%2Fsnyk_pull_request_template.yaml" ] }, { diff --git a/defaultFilters/customPrTemplates/azure-repos.json b/defaultFilters/customPrTemplates/azure-repos.json index 3126817ba..fe51488c7 100644 --- a/defaultFilters/customPrTemplates/azure-repos.json +++ b/defaultFilters/customPrTemplates/azure-repos.json @@ -1,17 +1 @@ -[ - { - "//": "get file existence. restrict by file types", - "method": "GET", - "path": "/:owner/_apis/git/repositories/:repo/items", - "origin": "https://${AZURE_REPOS_HOST}/${AZURE_REPOS_ORG}", - "valid": [ - { - "queryParam": "path", - "values": [ - "**/.azuredevops/snyk_pull_request_template.yaml", - "**%2F.azuredevops%2Fsnyk_pull_request_template.yaml" - ] - } - ] - } -] +[] diff --git a/lib/hybrid-sdk/common/filter/filter-rules-loading.ts b/lib/hybrid-sdk/common/filter/filter-rules-loading.ts index d1db3c8d1..dd78fd963 100644 --- a/lib/hybrid-sdk/common/filter/filter-rules-loading.ts +++ b/lib/hybrid-sdk/common/filter/filter-rules-loading.ts @@ -6,7 +6,7 @@ import { log as logger } from '../../../logs/logger'; import { findProjectRoot } from '../config/config'; import camelcase from 'camelcase'; import { FiltersType, Rule } from '../types/filter'; -import { retrieveFilters, isValidURI, deepMergeRules } from './utils'; +import { retrieveFilters, isValidURI } from './utils'; import { CONFIGURATION } from '../types/options'; const SUPPORTED_IAC_EXTENSIONS = ['tf', 'yaml', 'yml', 'tpl', 'json']; @@ -208,6 +208,7 @@ function injectRulesAtRuntime( templateGET.path = '*/info/refs*'; templateGETForSnippets.path = '/:owner/_apis/git/repositories/:repo/items'; + delete templateGETForSnippets.valid; templatePOST.path = '*/git-upload-pack'; break; case 'GITHUB': @@ -280,7 +281,20 @@ function injectRulesAtRuntime( `defaultFilters/apprisk/${type}.json`, )) as Rule[]; - filters.private = deepMergeRules(filters.private, appRiskRules); + const appRiskRulesMethodPattern = appRiskRules.map( + (x) => `${x.method}|${x.path}`, + ); + + const possibleOverlappingRules = filters.private.filter((x) => { + return !appRiskRulesMethodPattern.includes(`${x.method}|${x.path}`); + }); + if (possibleOverlappingRules.length > 0) { + logger.debug( + { possibleOverlappingRules }, + `Caution, possible overlapping rules with apprisk rules extension.`, + ); + } + filters.private.push(...appRiskRules); } } @@ -312,7 +326,21 @@ function injectRulesAtRuntime( `defaultFilters/customPrTemplates/${type}.json`, )) as Rule[]; - filters.private = deepMergeRules(filters.private, customPRTemplatesRules); + const customPRTemplatesRulesMethodPattern = customPRTemplatesRules.map( + (x) => `${x.method}|${x.path}`, + ); + const possibleOverlappingRules = filters.private.filter((x) => { + return !customPRTemplatesRulesMethodPattern.includes( + `${x.method}|${x.path}`, + ); + }); + if (possibleOverlappingRules.length > 0) { + logger.debug( + { possibleOverlappingRules }, + `Caution, possible overlapping rules with custom PR templates.`, + ); + } + filters.private.push(...customPRTemplatesRules); } } diff --git a/lib/hybrid-sdk/common/filter/utils.ts b/lib/hybrid-sdk/common/filter/utils.ts index b7346ec80..a8557402b 100644 --- a/lib/hybrid-sdk/common/filter/utils.ts +++ b/lib/hybrid-sdk/common/filter/utils.ts @@ -2,7 +2,6 @@ import path from 'node:path'; import fs from 'fs'; import { findProjectRoot } from '../config/config'; import { log as logger } from '../../../logs/logger'; -import { Rule } from '../types/filter'; import { PostFilterPreparedRequest } from '../../../broker-workload/prepareRequest'; import { makeSingleRawRequestToDownstream } from '../../http/request'; @@ -70,65 +69,3 @@ export const retrieveFilters = async (locations: Map) => { return retrievedFiltersMap; }; - -export function deepMergeRules(arr1: Rule[], arr2: Rule[]): Rule[] { - const isObject = (item: any): item is Record => - item && typeof item === 'object' && !Array.isArray(item); - - const mergeArrays = (target: any[], source: any[]): any[] => { - const result = [...target]; - - for (const item of source) { - if (isObject(item) && item.queryParam) { - const existing = result.find( - (i) => isObject(i) && i.queryParam === item.queryParam, - ); - if (existing) { - existing.values = [ - ...new Set([...(existing.values || []), ...(item.values || [])]), - ]; - } else { - result.push(item); - } - } else if (!result.includes(item)) { - result.push(item); - } - } - - return result; - }; - - const mergeObjects = ( - target: Record, - source: Record, - ): Record => { - for (const key of Object.keys(source)) { - if (isObject(target[key]) && isObject(source[key])) { - target[key] = mergeObjects(target[key], source[key]); - } else if (Array.isArray(target[key]) && Array.isArray(source[key])) { - target[key] = mergeArrays(target[key], source[key]); - } else { - target[key] = source[key]; - } - } - return target; - }; - - const grouped: Record = {}; - - for (const rule of [...arr1, ...arr2]) { - if (!rule || !isObject(rule)) continue; - - if ('method' in rule && 'path' in rule && 'origin' in rule) { - const key = `${rule.method}:${rule.path}:${rule.origin}`; - - if (grouped[key]) { - grouped[key] = mergeObjects(grouped[key], rule) as Rule; - } else { - grouped[key] = rule; - } - } - } - - return Object.values(grouped); -} diff --git a/test/unit/__snapshots__/runtime-rules-hotloading.test.ts.snap b/test/unit/__snapshots__/runtime-rules-hotloading.test.ts.snap index 443536fcf..47b6f39ac 100644 --- a/test/unit/__snapshots__/runtime-rules-hotloading.test.ts.snap +++ b/test/unit/__snapshots__/runtime-rules-hotloading.test.ts.snap @@ -39944,7 +39944,7 @@ Object { "path": "/_apis/hooks/subscriptions/:subscriptionId", }, Object { - "//": "get file existence. restrict by file types", + "//": "get file content. restrict by file types", "auth": Object { "scheme": "basic", "token": "\${BROKER_CLIENT_VALIDATION_BASIC_AUTH}", @@ -40020,29 +40020,105 @@ Object { "**%2Fgo.sum", "**/*Dockerfile*", "**%2F*Dockerfile*", - "**/.azuredevops/snyk_pull_request_template.yaml", - "**%2F.azuredevops%2Fsnyk_pull_request_template.yaml", ], }, Object { "queryParam": "recursionLevel", "values": Array [ "none", - "full", ], }, Object { "queryParam": "download", "values": Array [ "true", - "false", ], }, Object { "queryParam": "includeContent", "values": Array [ "true", - "false", + ], + }, + ], + }, + Object { + "//": "check file existence. restrict by file types", + "auth": Object { + "scheme": "basic", + "token": "\${BROKER_CLIENT_VALIDATION_BASIC_AUTH}", + }, + "method": "GET", + "origin": "https://\${AZURE_REPOS_HOST}/\${AZURE_REPOS_ORG}", + "path": "/:owner/_apis/git/repositories/:repo/items", + "valid": Array [ + Object { + "queryParam": "path", + "values": Array [ + "**/package.json", + "**%2Fpackage.json", + "**/yarn.lock", + "**%2Fyarn.lock", + "**/package-lock.json", + "**%2Fpackage-lock.json", + "**/Gemfile", + "**%2FGemfile", + "**/Gemfile.lock", + "**%2FGemfile.lock", + "**/pom.xml", + "**%2Fpom.xml", + "**/*req*.txt", + "**%2F*req*.txt", + "**/requirements/*.txt", + "**%2Frequirements%2F*.txt", + "**/pyproject.toml", + "**%2Fpyproject.toml", + "**/poetry.lock", + "**%2Fpoetry.lock", + "**/Pipfile", + "**%2FPipfile", + "**/Pipfile.lock", + "**%2FPipfile.lock", + "**/build.gradle", + "**%2Fbuild.gradle", + "**/gradle.lockfile", + "**%2Fgradle.lockfile", + "**/build.sbt", + "**%2Fbuild.sbt", + "**/.snyk", + "**%2F.snyk", + "**/packages.config", + "**%2Fpackages.config", + "**/*.csproj", + "**%2F*.csproj", + "**/*.vbproj", + "**%2F*.vbproj", + "**/*.fsproj", + "**%2F*.fsproj", + "**/project.json", + "**%2Fproject.json", + "**/Gopkg.toml", + "**%2FGopkg.toml", + "**/Gopkg.lock", + "**%2FGopkg.lock", + "**/vendor.json", + "**%2Fvendor.json", + "**/composer.lock", + "**%2Fcomposer.lock", + "**/composer.json", + "**%2Fcomposer.json", + "**/project.assets.json", + "**%2Fproject.assets.json", + "**/Podfile", + "**%2FPodfile", + "**/Podfile.lock", + "**%2FPodfile.lock", + "**/go.mod", + "**%2Fgo.mod", + "**/go.sum", + "**%2Fgo.sum", + "**/*Dockerfile*", + "**%2F*Dockerfile*", ], }, Object { @@ -40059,6 +40135,36 @@ Object { }, ], }, + Object { + "//": "get list of files for given repository", + "auth": Object { + "scheme": "basic", + "token": "\${BROKER_CLIENT_VALIDATION_BASIC_AUTH}", + }, + "method": "GET", + "origin": "https://\${AZURE_REPOS_HOST}/\${AZURE_REPOS_ORG}", + "path": "/:owner/_apis/git/repositories/:repo/items", + "valid": Array [ + Object { + "queryParam": "recursionLevel", + "values": Array [ + "full", + ], + }, + Object { + "queryParam": "download", + "values": Array [ + "false", + ], + }, + Object { + "queryParam": "includeContent", + "values": Array [ + "false", + ], + }, + ], + }, Object { "//": "get list of commits for given repository", "auth": Object { @@ -46205,7 +46311,7 @@ exports[`filter Rules Loading Injection of valid ACCEPT_CUSTOM_PR_TEMPLATES rule Object { "private": Array [ Object { - "//": "get current user. so we'll be able to fetch it's projects", + "//": "identify user", "method": "GET", "origin": "https://\${GITLAB}", "path": "/api/v4/user", @@ -46745,8 +46851,6 @@ Object { "**%2Fgo.mod", "**/go.sum", "**%2Fgo.sum", - "**/.config/snyk_pull_request_template.yaml", - "**%2F.config%2Fsnyk_pull_request_template.yaml", ], }, ], @@ -47741,6 +47845,12 @@ Object { "origin": "https://\${GITLAB}", "path": "/api/v4/groups/:id/projects", }, + Object { + "//": "get current user. so we'll be able to fetch it's projects", + "method": "GET", + "origin": "https://\${GITLAB}", + "path": "/api/v4/user", + }, Object { "//": "get user's projects", "method": "GET", @@ -47801,6 +47911,21 @@ Object { "origin": "https://\${GITLAB}", "path": "/api/v4/projects/:project/repository/files*%2Fsnyk_pull_request_template.yaml", }, + Object { + "//": "used to determine the full dependency tree for v3 protocol", + "method": "GET", + "origin": "https://\${GITLAB}", + "path": "/api/v3/projects/:project/repository/files", + "valid": Array [ + Object { + "queryParam": "file_path", + "values": Array [ + "**/.config/snyk_pull_request_template.yaml", + "**%2F.config%2Fsnyk_pull_request_template.yaml", + ], + }, + ], + }, ], "public": Array [ Object { @@ -47886,7 +48011,7 @@ Object { "path": "/_apis/hooks/subscriptions/:subscriptionId", }, Object { - "//": "get list of files for given repository", + "//": "get file content. restrict by file types", "auth": Object { "scheme": "basic", "token": "\${BROKER_CLIENT_VALIDATION_BASIC_AUTH}", @@ -47968,21 +48093,99 @@ Object { "queryParam": "recursionLevel", "values": Array [ "none", - "full", ], }, Object { "queryParam": "download", "values": Array [ "true", - "false", ], }, Object { "queryParam": "includeContent", "values": Array [ "true", - "false", + ], + }, + ], + }, + Object { + "//": "check file existence. restrict by file types", + "auth": Object { + "scheme": "basic", + "token": "\${BROKER_CLIENT_VALIDATION_BASIC_AUTH}", + }, + "method": "GET", + "origin": "https://\${AZURE_REPOS_HOST}/\${AZURE_REPOS_ORG}", + "path": "/:owner/_apis/git/repositories/:repo/items", + "valid": Array [ + Object { + "queryParam": "path", + "values": Array [ + "**/package.json", + "**%2Fpackage.json", + "**/yarn.lock", + "**%2Fyarn.lock", + "**/package-lock.json", + "**%2Fpackage-lock.json", + "**/Gemfile", + "**%2FGemfile", + "**/Gemfile.lock", + "**%2FGemfile.lock", + "**/pom.xml", + "**%2Fpom.xml", + "**/*req*.txt", + "**%2F*req*.txt", + "**/requirements/*.txt", + "**%2Frequirements%2F*.txt", + "**/pyproject.toml", + "**%2Fpyproject.toml", + "**/poetry.lock", + "**%2Fpoetry.lock", + "**/Pipfile", + "**%2FPipfile", + "**/Pipfile.lock", + "**%2FPipfile.lock", + "**/build.gradle", + "**%2Fbuild.gradle", + "**/gradle.lockfile", + "**%2Fgradle.lockfile", + "**/build.sbt", + "**%2Fbuild.sbt", + "**/.snyk", + "**%2F.snyk", + "**/packages.config", + "**%2Fpackages.config", + "**/*.csproj", + "**%2F*.csproj", + "**/*.vbproj", + "**%2F*.vbproj", + "**/*.fsproj", + "**%2F*.fsproj", + "**/project.json", + "**%2Fproject.json", + "**/Gopkg.toml", + "**%2FGopkg.toml", + "**/Gopkg.lock", + "**%2FGopkg.lock", + "**/vendor.json", + "**%2Fvendor.json", + "**/composer.lock", + "**%2Fcomposer.lock", + "**/composer.json", + "**%2Fcomposer.json", + "**/project.assets.json", + "**%2Fproject.assets.json", + "**/Podfile", + "**%2FPodfile", + "**/Podfile.lock", + "**%2FPodfile.lock", + "**/go.mod", + "**%2Fgo.mod", + "**/go.sum", + "**%2Fgo.sum", + "**/*Dockerfile*", + "**%2F*Dockerfile*", ], }, Object { @@ -47999,6 +48202,36 @@ Object { }, ], }, + Object { + "//": "get list of files for given repository", + "auth": Object { + "scheme": "basic", + "token": "\${BROKER_CLIENT_VALIDATION_BASIC_AUTH}", + }, + "method": "GET", + "origin": "https://\${AZURE_REPOS_HOST}/\${AZURE_REPOS_ORG}", + "path": "/:owner/_apis/git/repositories/:repo/items", + "valid": Array [ + Object { + "queryParam": "recursionLevel", + "values": Array [ + "full", + ], + }, + Object { + "queryParam": "download", + "values": Array [ + "false", + ], + }, + Object { + "queryParam": "includeContent", + "values": Array [ + "false", + ], + }, + ], + }, Object { "//": "get list of commits for given repository", "auth": Object { @@ -48354,25 +48587,6 @@ Object { "origin": "https://\${AZURE_REPOS_HOST}/\${AZURE_REPOS_ORG}", "path": "/:project/_apis/git/repositories/:repo/commits", }, - Object { - "//": "get organization context from file", - "auth": Object { - "scheme": "basic", - "token": "\${BROKER_CLIENT_VALIDATION_BASIC_AUTH}", - }, - "method": "GET", - "origin": "https://\${AZURE_REPOS_HOST}", - "path": "/:org/_apis/git/repositories/:repo/items", - "valid": Array [ - Object { - "queryParam": "path", - "values": Array [ - "**/*.json", - "**%2F*.json", - ], - }, - ], - }, Object { "//": "get organization context from file without org", "auth": Object { @@ -49768,6 +49982,17 @@ Object { "origin": "https://\${BITBUCKET_API}", "path": "/workspaces", }, + Object { + "//": "list the user's projects", + "auth": Object { + "password": "\${BITBUCKET_PASSWORD}", + "scheme": "basic", + "username": "\${BITBUCKET_USERNAME}", + }, + "method": "GET", + "origin": "https://\${BITBUCKET_API}", + "path": "/projects", + }, Object { "//": "used to get groups", "auth": Object { @@ -51025,6 +51250,16 @@ Object { "origin": "https://\${BITBUCKET_API}", "path": "/workspaces", }, + Object { + "//": "list the user's projects", + "auth": Object { + "scheme": "bearer", + "token": "\${BITBUCKET_PAT}", + }, + "method": "GET", + "origin": "https://\${BITBUCKET_API}", + "path": "/projects", + }, Object { "//": "used to get groups", "auth": Object { @@ -54816,7 +55051,7 @@ exports[`filter Rules Loading Injection of valid AppRisk rules - Testing gitlab Object { "private": Array [ Object { - "//": "get current user. so we'll be able to fetch it's projects", + "//": "identify user", "method": "GET", "origin": "https://\${GITLAB}", "path": "/api/v4/user", @@ -56350,6 +56585,12 @@ Object { "origin": "https://\${GITLAB}", "path": "/api/v4/groups/:id/projects", }, + Object { + "//": "get current user. so we'll be able to fetch it's projects", + "method": "GET", + "origin": "https://\${GITLAB}", + "path": "/api/v4/user", + }, Object { "//": "get user's projects", "method": "GET", @@ -121524,6 +121765,10 @@ Object { "**%2Fgo.sum", "**/*Dockerfile*", "**%2F*Dockerfile*", + "**/*.json", + "**%2F*.json", + "**/.azuredevops/snyk_pull_request_template.yaml", + "**%2F.azuredevops%2Fsnyk_pull_request_template.yaml", ], }, Object { @@ -121623,6 +121868,10 @@ Object { "**%2Fgo.sum", "**/*Dockerfile*", "**%2F*Dockerfile*", + "**/*.json", + "**%2F*.json", + "**/.azuredevops/snyk_pull_request_template.yaml", + "**%2F.azuredevops%2Fsnyk_pull_request_template.yaml", ], }, Object {