diff --git a/packages/cubejs-server-core/package.json b/packages/cubejs-server-core/package.json index cb681ec30a71b..dca346a1d428e 100644 --- a/packages/cubejs-server-core/package.json +++ b/packages/cubejs-server-core/package.json @@ -40,6 +40,8 @@ "codesandbox-import-utils": "^2.1.12", "cross-spawn": "^7.0.1", "fs-extra": "^8.1.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", "is-docker": "^2.1.1", "joi": "^17.8.3", "jsonwebtoken": "^9.0.2", diff --git a/packages/cubejs-server-core/src/core/agentCollect.ts b/packages/cubejs-server-core/src/core/agentCollect.ts index 4235257e8ccc6..be7b226b5ca18 100644 --- a/packages/cubejs-server-core/src/core/agentCollect.ts +++ b/packages/cubejs-server-core/src/core/agentCollect.ts @@ -1,6 +1,8 @@ import { getEnv } from '@cubejs-backend/shared'; import http from 'http'; import https from 'https'; +import { HttpsProxyAgent } from 'https-proxy-agent'; +import { HttpProxyAgent } from 'http-proxy-agent'; import fetch from 'node-fetch'; import crypto from 'crypto'; import WebSocket from 'ws'; @@ -50,12 +52,12 @@ class WebSocketTransport implements AgentTransport { clearTimeout(this.pingTimeout); this.onClose(); }); - + this.wsClient.on('error', e => { connectionPromiseReject(e); this.logger('Agent Error', { error: (e.stack || e).toString() }); }); - + this.wsClient.on('message', (data: WebSocket.Data) => { try { const { method, params } = JSON.parse(data.toString()); @@ -103,17 +105,45 @@ class WebSocketTransport implements AgentTransport { } } +function isOnNoProxyList(url: string): boolean { + const noProxy = process.env.NO_PROXY || process.env.no_proxy; + if (!noProxy) { + return false; + } + + const parsedUrl = new URL(url); + const { hostname } = parsedUrl; + const noProxyList = noProxy.split(',').map((entry) => entry.trim()); + + return noProxyList.some((entry) => { + if (entry === '*') { + return true; + } + if (entry.startsWith('.')) { + return hostname.endsWith(entry); + } + + return hostname === entry; + }); +} + class HttpTransport implements AgentTransport { - private agent: http.Agent | https.Agent; + private agent: http.Agent | https.Agent | HttpProxyAgent | HttpsProxyAgent; public constructor( private readonly endpointUrl: string ) { - const AgentClass = endpointUrl.startsWith('https') ? https.Agent : http.Agent; - this.agent = new AgentClass({ + const agentParams = { keepAlive: true, maxSockets: getEnv('agentMaxSockets') - }); + }; + if (!isOnNoProxyList(endpointUrl) && (process.env.http_proxy || process.env.https_proxy)) { + this.agent = endpointUrl.startsWith('https') ? + new HttpsProxyAgent(process.env.https_proxy, agentParams) : + new HttpProxyAgent(process.env.http_proxy, agentParams); + } else { + this.agent = endpointUrl.startsWith('https') ? new https.Agent(agentParams) : new http.Agent(agentParams); + } } public ready() { @@ -167,7 +197,7 @@ export default async (event: Record, endpointUrl: string, logger: a const sentAt = new Date().toJSON(); const result = await transport.send(toFlush.map(r => ({ ...r, sentAt }))); if (!result && retries > 0) return flush(toFlush, retries - 1); - + return true; } catch (e: any) { if (retries > 0) return flush(toFlush, retries - 1); diff --git a/yarn.lock b/yarn.lock index a9fd2f6e0bc4f..0f65f7623d125 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11335,6 +11335,11 @@ agent-base@^7.0.2, agent-base@^7.1.0: dependencies: debug "^4.3.4" +agent-base@^7.1.2: + version "7.1.3" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.3.tgz#29435eb821bc4194633a5b89e5bc4703bafc25a1" + integrity sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw== + agentkeepalive@^4.1.3: version "4.1.4" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.1.4.tgz#d928028a4862cb11718e55227872e842a44c945b" @@ -18204,6 +18209,14 @@ http-proxy-agent@^7.0.0: agent-base "^7.1.0" debug "^4.3.4" +http-proxy-agent@^7.0.2: + version "7.0.2" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e" + integrity sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig== + dependencies: + agent-base "^7.1.0" + debug "^4.3.4" + http-proxy-middleware@^2.0.0, http-proxy-middleware@^2.0.3: version "2.0.7" resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6" @@ -18278,6 +18291,14 @@ https-proxy-agent@^7.0.1: agent-base "^7.0.2" debug "4" +https-proxy-agent@^7.0.6: + version "7.0.6" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9" + integrity sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw== + dependencies: + agent-base "^7.1.2" + debug "4" + human-signals@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3"