From 6718de08c261c037e4c711e4edaefc9da6479aec Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Mon, 1 Jul 2024 15:14:41 -0700 Subject: [PATCH 01/56] add log debug headers --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index c82e8a4e..b8a61829 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -66,6 +66,7 @@ protected void postConnectionHook( log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), var9); + debugLogHeaders(request); callback.failed(var9); } } From 28bde1d22322954caa99ea024e40385c1ae3f073 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Mon, 1 Jul 2024 15:21:35 -0700 Subject: [PATCH 02/56] use error log level --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index b8a61829..b2b0ee62 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -66,11 +66,20 @@ protected void postConnectionHook( log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), var9); - debugLogHeaders(request); + errorLogHeaders(request); callback.failed(var9); } } + protected void errorLogHeaders(HttpServletRequest request) { + log.error("-------HttpServletRequest headers---------"); + Enumeration headers = request.getHeaderNames(); + while (headers.hasMoreElements()) { + String header = headers.nextElement(); + log.error(header + "->" + request.getHeader(header)); + } + } + protected void debugLogHeaders(HttpServletRequest request) { if (log.isDebugEnabled()) { log.debug("-------HttpServletRequest headers---------"); From 2773cf3efee01ce6ededb5de4c5e274ebe3c8be4 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Mon, 1 Jul 2024 15:36:24 -0700 Subject: [PATCH 03/56] add logging for error response header --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index b2b0ee62..ec20f0d8 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -67,12 +67,13 @@ protected void postConnectionHook( + " servlet path {} , toString {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), var9); errorLogHeaders(request); + errorLogHeaders(response); callback.failed(var9); } } protected void errorLogHeaders(HttpServletRequest request) { - log.error("-------HttpServletRequest headers---------"); + log.error("------- error HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); @@ -80,6 +81,14 @@ protected void errorLogHeaders(HttpServletRequest request) { } } + protected void errorLogHeaders(HttpServletResponse response) { + log.error("------- error HttpServletResponse headers---------"); + Collection headers = response.getHeaderNames(); + for (String header : headers) { + log.error(header + "->" + response.getHeader(header)); + } + } + protected void debugLogHeaders(HttpServletRequest request) { if (log.isDebugEnabled()) { log.debug("-------HttpServletRequest headers---------"); From 8bc1af92e60ca5e2ffefd7ca0d4a433c79ad9d52 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Mon, 1 Jul 2024 18:32:55 -0700 Subject: [PATCH 04/56] other config changes --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 89e8bd1b..4087e0b8 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -86,10 +86,13 @@ private void setupContext(ProxyServerConfiguration config) { connector.setPort(config.getLocalPort()); connector.setName(config.getName()); connector.setAccepting(true); + connector.setIdleTimeout(150000L); + connector.setAcceptQueueSize(1024); this.server.addConnector(connector); // Setup proxy handler to handle CONNECT methods ConnectHandler proxyConnectHandler = new ConnectHandler(); + proxyConnectHandler.getIdleTimeout() this.server.setHandler(proxyConnectHandler); if (proxyHandler != null) { @@ -102,6 +105,7 @@ private void setupContext(ProxyServerConfiguration config) { proxyServlet.setInitParameter("prefix", config.getPrefix()); proxyServlet.setInitParameter("trustAll", config.getTrustAll()); proxyServlet.setInitParameter("preserveHost", config.getPreserveHost()); + proxyServlet.setInitParameter("timeout", "120000"); // Setup proxy servlet this.context = From c4e320064dd6d97758b8850227671cca9cd83e4b Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Mon, 1 Jul 2024 18:34:08 -0700 Subject: [PATCH 05/56] config testing --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 4087e0b8..cbddcdca 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -92,7 +92,6 @@ private void setupContext(ProxyServerConfiguration config) { // Setup proxy handler to handle CONNECT methods ConnectHandler proxyConnectHandler = new ConnectHandler(); - proxyConnectHandler.getIdleTimeout() this.server.setHandler(proxyConnectHandler); if (proxyHandler != null) { From d1ad10aa1ca6a768dcae1fea16058ffa3cda406a Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Mon, 1 Jul 2024 18:56:05 -0700 Subject: [PATCH 06/56] test --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index cbddcdca..0d22efb7 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -74,6 +74,7 @@ private void setupContext(ProxyServerConfiguration config) { src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); httpsConfig.addCustomizer(src); + httpsConfig.addCustomizer( new org.eclipse.jetty.server.ForwardedRequestCustomizer() ); connector = new ServerConnector( server, From 365e481445513b778e2425095bf32a8b153eeed2 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:01:43 -0700 Subject: [PATCH 07/56] add more changes --- gateway-ha/gateway-ha-config.yml | 4 ++++ .../data/gateway/ha/module/HaGatewayProviderModule.java | 1 + .../main/java/com/lyft/data/proxyserver/ProxyServer.java | 8 +++++++- .../java/com/lyft/data/proxyserver/ProxyServletImpl.java | 4 +++- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/gateway-ha/gateway-ha-config.yml b/gateway-ha/gateway-ha-config.yml index 70f1ee92..e8cd62e0 100644 --- a/gateway-ha/gateway-ha-config.yml +++ b/gateway-ha/gateway-ha-config.yml @@ -6,6 +6,10 @@ requestRouter: port: 8080 name: prestoRouter historySize: 1000 + requestHeaderSize: 2048000 + responseHeaderSize: 2048000 + requestBufferSize: 2048000 + responseBufferSize: 2048000 dataStore: jdbcUrl: jdbc:mysql://127.0.0.1:3306/prestogateway diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java index edadc249..d0184393 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java @@ -81,6 +81,7 @@ public ProxyServer provideGateway() { routerProxyConfig.setKeystorePass(routerConfiguration.getKeystorePass()); routerProxyConfig.setForwardKeystore(routerConfiguration.isForwardKeystore()); routerProxyConfig.setPreserveHost("false"); + ProxyHandler proxyHandler = getProxyHandler(); gateway = new ProxyServer(routerProxyConfig, proxyHandler); } diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 0d22efb7..bcc690bf 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -59,6 +59,7 @@ private void setupContext(ProxyServerConfiguration config) { sslContextFactory.setStopTimeout(TimeUnit.SECONDS.toMillis(15)); sslContextFactory.setSslSessionTimeout((int) TimeUnit.SECONDS.toMillis(15)); + if (!TextUtils.isBlank(keystorePath)) { sslContextFactory.setKeyStorePath(keystoreFile.getAbsolutePath()); sslContextFactory.setKeyStorePassword(keystorePass); @@ -68,13 +69,18 @@ private void setupContext(ProxyServerConfiguration config) { HttpConfiguration httpsConfig = new HttpConfiguration(); httpsConfig.setSecureScheme(HttpScheme.HTTPS.asString()); httpsConfig.setSecurePort(config.getLocalPort()); + httpsConfig.setIdleTimeout(150000); httpsConfig.setOutputBufferSize(32768); + httpsConfig.setRequestHeaderSize(2048000); + httpsConfig.setResponseHeaderSize(2048000); + + SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); httpsConfig.addCustomizer(src); - httpsConfig.addCustomizer( new org.eclipse.jetty.server.ForwardedRequestCustomizer() ); + httpsConfig.addCustomizer(new org.eclipse.jetty.server.ForwardedRequestCustomizer()); connector = new ServerConnector( server, diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java index 385dd56e..b541f589 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java @@ -45,7 +45,9 @@ protected HttpClient newHttpClient() { HttpClient httpClient = new HttpClient(sslFactory); httpClient.setMaxConnectionsPerDestination(10000); - httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(60)); + httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(65)); + httpClient.setIdleTimeout(TimeUnit.SECONDS.toMillis(65)); + return httpClient; } From 57e0995935596fd95a3da72389fef5cd5e41ac7c Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:02:57 -0700 Subject: [PATCH 08/56] more config changes --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index bcc690bf..b3ac5a42 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -74,8 +74,6 @@ private void setupContext(ProxyServerConfiguration config) { httpsConfig.setRequestHeaderSize(2048000); httpsConfig.setResponseHeaderSize(2048000); - - SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); From 3eb9da8b9f80cc2afd5848f691399c556cb570ae Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:06:57 -0700 Subject: [PATCH 09/56] add logging for error request/response header size --- .../lyft/data/proxyserver/ProxyHandler.java | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index ec20f0d8..44acf8e6 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -64,29 +64,38 @@ protected void postConnectionHook( callback.succeeded(); } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," - + " servlet path {} , toString {}", request.getRequestURL(), - request.getRequestURI(), request.getServletPath(), request.toString(), var9); - errorLogHeaders(request); - errorLogHeaders(response); + + " servlet path {} , toString {}, size {}", request.getRequestURL(), + request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), var9); + + requestHeaderLog = getErrorLogHeaders(request); + log.error(requestHeaderLog); + + responseHeaderLog = getErrorLogHeaders(response); + log.error(responseHeaderLog); callback.failed(var9); } } - protected void errorLogHeaders(HttpServletRequest request) { - log.error("------- error HttpServletRequest headers---------"); + protected String getErrorLogHeaders(HttpServletRequest request) { + StringBuilder sb = new StringBuilder(); + sb.append("------- error HttpServletRequest headers---------\n"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); - log.error(header + "->" + request.getHeader(header)); + sb.append(header).append("->").append(request.getHeader(header)).append("\n"); } + return sb.toString(); } - protected void errorLogHeaders(HttpServletResponse response) { - log.error("------- error HttpServletResponse headers---------"); - Collection headers = response.getHeaderNames(); - for (String header : headers) { - log.error(header + "->" + response.getHeader(header)); + protected void getErrorLogHeaders(HttpServletResponse response) { + StringBuilder sb = new StringBuilder(); + sb.append("------- error HttpServletResponse headers---------\n"); + Enumeration headers = response.getHeaderNames(); + while (headers.hasMoreElements()) { + String header = headers.nextElement(); + sb.append(header).append("->").append(response.getHeader(header)).append("\n"); } + return sb.toString(); } protected void debugLogHeaders(HttpServletRequest request) { From 7bd21d9139c34941ca85bb982f34daf58d5515cc Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:08:01 -0700 Subject: [PATCH 10/56] add logging for success at 1% --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 44acf8e6..3bc91147 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -61,6 +61,15 @@ protected void postConnectionHook( } response.getOutputStream().write(buffer, offset, length); + // [sev-16337] with a 1% probably, log the request and response headers and size for debugging + if (Math.random() < 0.01) { + debugLogHeaders(request); + debugLogHeaders(response); + log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength()); + } + callback.succeeded(); } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," From e781b3fb5848798a5afbd9647b5f80c04368e02b Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:10:00 -0700 Subject: [PATCH 11/56] remove full header debug from success --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 3bc91147..bf09b71b 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -61,10 +61,8 @@ protected void postConnectionHook( } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 1% probably, log the request and response headers and size for debugging - if (Math.random() < 0.01) { - debugLogHeaders(request); - debugLogHeaders(response); + // [sev-16337] with a 10% probably, log the request and response headers and size for debugging + if (Math.random() < 0.10) { log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength()); @@ -76,10 +74,10 @@ protected void postConnectionHook( + " servlet path {} , toString {}, size {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), var9); - requestHeaderLog = getErrorLogHeaders(request); + String requestHeaderLog = getErrorLogHeaders(request); log.error(requestHeaderLog); - responseHeaderLog = getErrorLogHeaders(response); + String responseHeaderLog = getErrorLogHeaders(response); log.error(responseHeaderLog); callback.failed(var9); } @@ -96,7 +94,7 @@ protected String getErrorLogHeaders(HttpServletRequest request) { return sb.toString(); } - protected void getErrorLogHeaders(HttpServletResponse response) { + protected String getErrorLogHeaders(HttpServletResponse response) { StringBuilder sb = new StringBuilder(); sb.append("------- error HttpServletResponse headers---------\n"); Enumeration headers = response.getHeaderNames(); From fba8a403c036473d51fde75280f61874769aad51 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:16:28 -0700 Subject: [PATCH 12/56] more updates --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 +- .../main/java/com/lyft/data/proxyserver/ProxyServletImpl.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index b3ac5a42..3fd2424f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -73,7 +73,7 @@ private void setupContext(ProxyServerConfiguration config) { httpsConfig.setOutputBufferSize(32768); httpsConfig.setRequestHeaderSize(2048000); httpsConfig.setResponseHeaderSize(2048000); - + SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java index b541f589..8a1c36c2 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java @@ -47,6 +47,8 @@ protected HttpClient newHttpClient() { httpClient.setMaxConnectionsPerDestination(10000); httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(65)); httpClient.setIdleTimeout(TimeUnit.SECONDS.toMillis(65)); + httpClient.setRequestBufferSize(2048000); + httpClient.setResponseBufferSize(2048000); return httpClient; } From 603ff4d0ce324985bb39965d5b4262b2a010e203 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:15:58 -0700 Subject: [PATCH 13/56] fix --- .../lyft/data/proxyserver/ProxyHandler.java | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index bf09b71b..d5ff024d 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -60,7 +60,6 @@ protected void postConnectionHook( request.getRequestURL()); } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 10% probably, log the request and response headers and size for debugging if (Math.random() < 0.10) { log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", @@ -73,36 +72,27 @@ protected void postConnectionHook( log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}, size {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), var9); - - String requestHeaderLog = getErrorLogHeaders(request); - log.error(requestHeaderLog); - - String responseHeaderLog = getErrorLogHeaders(response); - log.error(responseHeaderLog); + errorLogHeaders(request); + errorLogHeaders(response); callback.failed(var9); } } - protected String getErrorLogHeaders(HttpServletRequest request) { - StringBuilder sb = new StringBuilder(); - sb.append("------- error HttpServletRequest headers---------\n"); + protected void errorLogHeaders(HttpServletRequest request) { + log.error("------- error HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); - sb.append(header).append("->").append(request.getHeader(header)).append("\n"); + log.error(header + "->" + request.getHeader(header)); } - return sb.toString(); } - protected String getErrorLogHeaders(HttpServletResponse response) { - StringBuilder sb = new StringBuilder(); - sb.append("------- error HttpServletResponse headers---------\n"); - Enumeration headers = response.getHeaderNames(); - while (headers.hasMoreElements()) { - String header = headers.nextElement(); - sb.append(header).append("->").append(response.getHeader(header)).append("\n"); + protected void errorLogHeaders(HttpServletResponse response) { + log.error("------- error HttpServletResponse headers---------"); + Collection headers = response.getHeaderNames(); + for (String header : headers) { + log.error(header + "->" + response.getHeader(header)); } - return sb.toString(); } protected void debugLogHeaders(HttpServletRequest request) { @@ -172,4 +162,4 @@ protected boolean isCompressed(final byte[] compressed) { return (compressed[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (compressed[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8)); } -} +} \ No newline at end of file From c17fbf7cab167cb8068bd3e16799ae4ad35549f1 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:29:17 -0700 Subject: [PATCH 14/56] fix style --- .../main/java/com/lyft/data/proxyserver/ProxyHandler.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index d5ff024d..e52e9e8f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -60,8 +60,9 @@ protected void postConnectionHook( request.getRequestURL()); } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 10% probably, log the request and response headers and size for debugging - if (Math.random() < 0.10) { + // [sev-16337] with a 1% probably, log the request and response headers + // and size for debugging + if (Math.random() < 0.01) { log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength()); @@ -71,7 +72,8 @@ protected void postConnectionHook( } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}, size {}", request.getRequestURL(), - request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), var9); + request.getRequestURI(), request.getServletPath(), request.toString(), + request.getContentLength(), var9); errorLogHeaders(request); errorLogHeaders(response); callback.failed(var9); From ccde25ea6376c29bfb442e09863046fe34ad1601 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 09:40:23 -0700 Subject: [PATCH 15/56] add check for header not just content size --- .../lyft/data/proxyserver/ProxyHandler.java | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index e52e9e8f..67ddb3da 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -63,23 +63,46 @@ protected void postConnectionHook( // [sev-16337] with a 1% probably, log the request and response headers // and size for debugging if (Math.random() < 0.01) { - log.debug("Request URL: {} , request URI {} , servlet path {} , toString {}, size {}", + log.debug("Request URL: {} , request URI {} , servlet path {} ," + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), - request.toString(), request.getContentLength()); + request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response)); } callback.succeeded(); } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," - + " servlet path {} , toString {}, size {}", request.getRequestURL(), - request.getRequestURI(), request.getServletPath(), request.toString(), - request.getContentLength(), var9); + + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," + + "getResponseHeaderSize {}", request.getRequestURL(), request.getRequestURI(), + request.getServletPath(), request.toString(), + request.getContentLength(), getRequestHeaderSize(request), getRequestHeaderSize(response) var9); errorLogHeaders(request); errorLogHeaders(response); callback.failed(var9); } } + protected int getRequestHeaderSize(HttpServletRequest request) { + int headerSize = 0; + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + String headerValue = request.getHeader(headerName); + headerSize += headerName.length() + headerValue.length(); + } + return headerSize; +} + +private int getResponseHeaderSize(HttpServletResponse response) { + int headerSize = 0; + for (String headerName : response.getHeaderNames()) { + String headerValue = response.getHeader(headerName); + headerSize += headerName.length() + headerValue.length(); + } + return headerSize; +} + + protected void errorLogHeaders(HttpServletRequest request) { log.error("------- error HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); From a3eb31957107668a9b25fffdb56adb802a92b7c7 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 22:13:33 +0530 Subject: [PATCH 16/56] Update ProxyHandler.java --- .../lyft/data/proxyserver/ProxyHandler.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 67ddb3da..da9ac050 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -73,11 +73,11 @@ protected void postConnectionHook( } catch (Throwable var9) { log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," - + "getResponseHeaderSize {}", request.getRequestURL(), request.getRequestURI(), - request.getServletPath(), request.toString(), - request.getContentLength(), getRequestHeaderSize(request), getRequestHeaderSize(response) var9); - errorLogHeaders(request); - errorLogHeaders(response); + + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), + request.getContentLength(), getRequestHeaderSize(request), getRequestHeaderSize(response), + errorLogHeaders(request), errorLogHeaders(response), var9); + callback.failed(var9); } } @@ -102,22 +102,24 @@ private int getResponseHeaderSize(HttpServletResponse response) { return headerSize; } - - protected void errorLogHeaders(HttpServletRequest request) { - log.error("------- error HttpServletRequest headers---------"); + protected String errorLogHeaders(HttpServletRequest request) { + StringBuilder sb = "------- error HttpServletRequest headers---------"; Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); - log.error(header + "->" + request.getHeader(header)); + sb.append(header); } + return sb.toString(); } - protected void errorLogHeaders(HttpServletResponse response) { - log.error("------- error HttpServletResponse headers---------"); - Collection headers = response.getHeaderNames(); - for (String header : headers) { - log.error(header + "->" + response.getHeader(header)); + protected String errorLogHeaders(HttpServletResponse response) { + StringBuilder sb = "------- error HttpServletResponse headers---------"; + Enumeration headers = request.getHeaderNames(); + while (headers.hasMoreElements()) { + String header = headers.nextElement(); + sb.append(header); } + return sb.toString(); } protected void debugLogHeaders(HttpServletRequest request) { @@ -187,4 +189,4 @@ protected boolean isCompressed(final byte[] compressed) { return (compressed[0] == (byte) (GZIPInputStream.GZIP_MAGIC)) && (compressed[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8)); } -} \ No newline at end of file +} From 8f72c712e6c9601964cabf546c6be74bf3dab268 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:45:09 -0700 Subject: [PATCH 17/56] dropwizard update --- baseapp/pom.xml | 2 +- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/baseapp/pom.xml b/baseapp/pom.xml index c6be2d49..3662cb1f 100644 --- a/baseapp/pom.xml +++ b/baseapp/pom.xml @@ -17,7 +17,7 @@ 1.8 UTF-8 UTF-8 - 1.3.29 + 1.3.7 4.1.0 0.9.10 diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 3fd2424f..b3ac5a42 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -73,7 +73,7 @@ private void setupContext(ProxyServerConfiguration config) { httpsConfig.setOutputBufferSize(32768); httpsConfig.setRequestHeaderSize(2048000); httpsConfig.setResponseHeaderSize(2048000); - + SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); From f488a042f00849c71483a85679653cbdd89bf8e8 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:51:10 -0700 Subject: [PATCH 18/56] compile fix --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index da9ac050..9b52daa9 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -103,22 +103,23 @@ private int getResponseHeaderSize(HttpServletResponse response) { } protected String errorLogHeaders(HttpServletRequest request) { - StringBuilder sb = "------- error HttpServletRequest headers---------"; + StringBuilder sb = new StringBuilder("------- error HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); sb.append(header); } + return sb.toString(); } protected String errorLogHeaders(HttpServletResponse response) { - StringBuilder sb = "------- error HttpServletResponse headers---------"; - Enumeration headers = request.getHeaderNames(); - while (headers.hasMoreElements()) { - String header = headers.nextElement(); + StringBuilder sb = new StringBuilder("------- error HttpServletResponse headers---------"); + Collection headers = response.getHeaderNames(); + for (String header : headers) { sb.append(header); } + return sb.toString(); } From 60db65e652676e22c520d399f058baaf47cb34fe Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 09:53:13 -0700 Subject: [PATCH 19/56] more fixes --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 9b52daa9..1c582a27 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -75,7 +75,7 @@ protected void postConnectionHook( + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), - request.getContentLength(), getRequestHeaderSize(request), getRequestHeaderSize(response), + request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), errorLogHeaders(response), var9); callback.failed(var9); @@ -109,7 +109,7 @@ protected String errorLogHeaders(HttpServletRequest request) { String header = headers.nextElement(); sb.append(header); } - + return sb.toString(); } From 7092cdb998a6b9289c08322c8d1bfea46a3bd352 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 10:05:07 -0700 Subject: [PATCH 20/56] lint fixes --- .../lyft/data/proxyserver/ProxyHandler.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 1c582a27..08d3f628 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -64,9 +64,10 @@ protected void postConnectionHook( // and size for debugging if (Math.random() < 0.01) { log.debug("Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}", - request.getRequestURL(), request.getRequestURI(), request.getServletPath(), - request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response)); + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + getResponseHeaderSize(response)); } callback.succeeded(); @@ -74,9 +75,10 @@ protected void postConnectionHook( log.error("Exception occurred while processing request URL: {} , request URI {} ," + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", - request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), - request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), - errorLogHeaders(request), errorLogHeaders(response), var9); + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + getResponseHeaderSize(response), errorLogHeaders(request), + errorLogHeaders(response), var9); callback.failed(var9); } @@ -86,21 +88,21 @@ protected int getRequestHeaderSize(HttpServletRequest request) { int headerSize = 0; Enumeration headerNames = request.getHeaderNames(); while (headerNames.hasMoreElements()) { - String headerName = headerNames.nextElement(); - String headerValue = request.getHeader(headerName); - headerSize += headerName.length() + headerValue.length(); + String headerName = headerNames.nextElement(); + String headerValue = request.getHeader(headerName); + headerSize += headerName.length() + headerValue.length(); } return headerSize; -} + } -private int getResponseHeaderSize(HttpServletResponse response) { - int headerSize = 0; - for (String headerName : response.getHeaderNames()) { + private int getResponseHeaderSize(HttpServletResponse response) { + int headerSize = 0; + for (String headerName : response.getHeaderNames()) { String headerValue = response.getHeader(headerName); headerSize += headerName.length() + headerValue.length(); + } + return headerSize; } - return headerSize; -} protected String errorLogHeaders(HttpServletRequest request) { StringBuilder sb = new StringBuilder("------- error HttpServletRequest headers---------"); From fd6b9f63af374b6c009102c14d7f1e0c13fb1e5a Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 10:45:52 -0700 Subject: [PATCH 21/56] fix header logging --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 08d3f628..4b04c29f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -109,7 +109,7 @@ protected String errorLogHeaders(HttpServletRequest request) { Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); - sb.append(header); + sb.append(header + "->" + request.getHeader(header) + "\n"); } return sb.toString(); @@ -119,7 +119,7 @@ protected String errorLogHeaders(HttpServletResponse response) { StringBuilder sb = new StringBuilder("------- error HttpServletResponse headers---------"); Collection headers = response.getHeaderNames(); for (String header : headers) { - sb.append(header); + sb.append(header + "->" + response.getHeader(header) + "\n"); } return sb.toString(); From 765f61d627e0e4b959028a9608a6bfefa811cc8c Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 23:40:33 +0530 Subject: [PATCH 22/56] Update ProxyHandler.java --- .../main/java/com/lyft/data/proxyserver/ProxyHandler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 4b04c29f..1406d904 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -64,10 +64,11 @@ protected void postConnectionHook( // and size for debugging if (Math.random() < 0.01) { log.debug("Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}", + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), - getResponseHeaderSize(response)); + getResponseHeaderSize(response), errorLogHeaders(request), + errorLogHeaders(response)); } callback.succeeded(); From b145513798489803b0f8d01dc19a641115bed9ee Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 23:42:41 +0530 Subject: [PATCH 23/56] Update ProxyHandler.java --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 1406d904..0afd1c91 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -64,7 +64,8 @@ protected void postConnectionHook( // and size for debugging if (Math.random() < 0.01) { log.debug("Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, + requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), From 1ce21041649fb9e1a5055e30517bc1a6c53616be Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 14:29:56 -0700 Subject: [PATCH 24/56] add logging in preconnection hook --- .../lyft/data/proxyserver/ProxyHandler.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 0afd1c91..cb790383 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -35,6 +35,15 @@ protected String rewriteTarget(HttpServletRequest request) { */ public void preConnectionHook(HttpServletRequest request, Request proxyRequest) { // you may override it. + + // [sev-16337] with a 10% probably, log the request headers for debugging + if (Math.random() < 0.10) { + log.debug("(preConnectionHook) Request URL: {} , request URI {} , servlet path {} ," + + "toString {}, getContentLength {}, getRequestHeaderSize {}, requestHeaders {}", + request.getRequestURL(), request.getRequestURI(), request.getServletPath(), + request.toString(), request.getContentLength(), getRequestHeaderSize(request), + errorLogHeaders(request)); + } } /** @@ -60,12 +69,12 @@ protected void postConnectionHook( request.getRequestURL()); } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 1% probably, log the request and response headers + // [sev-16337] with a 10% probably, log the request and response headers // and size for debugging - if (Math.random() < 0.01) { - log.debug("Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, - requestHeaders {}, responseHeaders {}", + if (Math.random() < 0.10) { + log.debug("(postConnectionHook - success) Request URL: {} , request URI {} , servlet path {} ," + + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, " + + "requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), @@ -74,9 +83,9 @@ protected void postConnectionHook( callback.succeeded(); } catch (Throwable var9) { - log.error("Exception occurred while processing request URL: {} , request URI {} ," - + " servlet path {} , toString {}, getContentLength {}, getRequestHeaderSize {}," - + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + log.error("(postConnectionHook - failed) Exception occurred while processing request URL: {} , " + + "request URI {} , servlet path {} , toString {}, getContentLength {}, " + + "getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), From 80ac96c5b220e3b102708ed886272cc5ae72c313 Mon Sep 17 00:00:00 2001 From: Anuved Verma Date: Tue, 2 Jul 2024 14:43:06 -0700 Subject: [PATCH 25/56] checkstyle fixes --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index cb790383..b34153e3 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -72,9 +72,9 @@ protected void postConnectionHook( // [sev-16337] with a 10% probably, log the request and response headers // and size for debugging if (Math.random() < 0.10) { - log.debug("(postConnectionHook - success) Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, getResponseHeaderSize {}, " - + "requestHeaders {}, responseHeaders {}", + log.debug("(postConnectionHook) Request URL: {} , request URI {} , servlet path {} , " + + "toString {}, getContentLength {}, getRequestHeaderSize {}, " + + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), @@ -83,9 +83,10 @@ protected void postConnectionHook( callback.succeeded(); } catch (Throwable var9) { - log.error("(postConnectionHook - failed) Exception occurred while processing request URL: {} , " + log.error("(postConnectionHook) Exception occurred while processing request URL: {} , " + "request URI {} , servlet path {} , toString {}, getContentLength {}, " - + "getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", + + "getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, " + + "responseHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), getResponseHeaderSize(response), errorLogHeaders(request), From 5911d0b229894b2fc7a17884c3a3e7cd39ab185e Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 17:47:39 -0400 Subject: [PATCH 26/56] adding error counters --- .../handler/QueryIdCachingProxyHandler.java | 19 ++++++++++++++++++- .../ha/module/HaGatewayProviderModule.java | 11 ++++++++--- proxyserver/pom.xml | 7 +++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index c16dabd0..e272811b 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -1,6 +1,8 @@ package com.lyft.data.gateway.ha.handler; +import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; import com.google.common.io.CharStreams; @@ -28,6 +30,8 @@ import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.util.Callback; +import static com.codahale.metrics.MetricRegistry.name; + @Slf4j public class QueryIdCachingProxyHandler extends ProxyHandler { public static final String PROXY_TARGET_HEADER = "proxytarget"; @@ -57,13 +61,20 @@ public class QueryIdCachingProxyHandler extends ProxyHandler { private final Meter requestMeter; private final int serverApplicationPort; + private final Counter errorCounter4xx; + private final Counter errorCounter5xx; + private final MetricRegistry metrics = new MetricRegistry(); + public QueryIdCachingProxyHandler( QueryHistoryManager queryHistoryManager, RoutingManager routingManager, RoutingGroupSelector routingGroupSelector, int serverApplicationPort, - Meter requestMeter) { + Meter requestMeter, + MetricRegistry metrics) { this.requestMeter = requestMeter; + this.errorCounter4xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "4xx")); + this.errorCounter5xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "5xx"));; this.routingManager = routingManager; this.routingGroupSelector = routingGroupSelector; this.queryHistoryManager = queryHistoryManager; @@ -270,6 +281,12 @@ protected void postConnectionHook( } catch (Exception e) { log.error("Error in proxying falling back to super call", e); } +// if (response.getStatus() >= 500){ +// errorCounter5xx.inc(); +// } else if (response.getStatus() >= 400){ +// errorCounter4xx.inc(); +// } + super.postConnectionHook(request, response, buffer, offset, length, callback); } diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java index d0184393..6d4fbf99 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java @@ -1,6 +1,8 @@ package com.lyft.data.gateway.ha.module; +import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; +import com.codahale.metrics.MetricRegistry; import com.google.inject.Provides; import com.google.inject.Singleton; import com.lyft.data.baseapp.AppModule; @@ -23,6 +25,8 @@ import com.lyft.data.proxyserver.ProxyServerConfiguration; import io.dropwizard.setup.Environment; +import static com.codahale.metrics.MetricRegistry.name; + public class HaGatewayProviderModule extends AppModule { private final ResourceGroupsManager resourceGroupsManager; @@ -42,9 +46,9 @@ public HaGatewayProviderModule(HaGatewayConfiguration configuration, Environment } protected ProxyHandler getProxyHandler() { + MetricRegistry metrics = getEnvironment().metrics(); Meter requestMeter = - getEnvironment() - .metrics() + metrics .meter(getConfiguration().getRequestRouter().getName() + ".requests"); // By default, use routing group header to route @@ -61,7 +65,8 @@ protected ProxyHandler getProxyHandler() { getRoutingManager(), routingGroupSelector, getApplicationPort(), - requestMeter); + requestMeter, + metrics); } @Provides diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index d14b6999..9ad19af2 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -18,6 +18,7 @@ 1.8 UTF-8 UTF-8 + 1.3.29 @@ -62,6 +63,12 @@ mockwebserver test + + io.dropwizard.metrics + metrics-core + 4.2.7 + compile + From 4d1c079da98566d921fdf9eec8232f4cb9feb144 Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 17:49:53 -0400 Subject: [PATCH 27/56] forgot to uncomment --- .../gateway/ha/handler/QueryIdCachingProxyHandler.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index e272811b..ea1b1873 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -281,11 +281,11 @@ protected void postConnectionHook( } catch (Exception e) { log.error("Error in proxying falling back to super call", e); } -// if (response.getStatus() >= 500){ -// errorCounter5xx.inc(); -// } else if (response.getStatus() >= 400){ -// errorCounter4xx.inc(); -// } + if (response.getStatus() >= 500){ + errorCounter5xx.inc(); + } else if (response.getStatus() >= 400){ + errorCounter4xx.inc(); + } super.postConnectionHook(request, response, buffer, offset, length, callback); } From 29613c4cb4f288b6e17a07882098b5c12205ac13 Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 17:56:50 -0400 Subject: [PATCH 28/56] fix checkstyle --- .../gateway/ha/handler/QueryIdCachingProxyHandler.java | 8 ++++---- .../data/gateway/ha/module/HaGatewayProviderModule.java | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index ea1b1873..270fb2df 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -1,5 +1,7 @@ package com.lyft.data.gateway.ha.handler; +import static com.codahale.metrics.MetricRegistry.name; + import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; @@ -30,7 +32,6 @@ import org.eclipse.jetty.http.HttpStatus; import org.eclipse.jetty.util.Callback; -import static com.codahale.metrics.MetricRegistry.name; @Slf4j public class QueryIdCachingProxyHandler extends ProxyHandler { @@ -281,12 +282,11 @@ protected void postConnectionHook( } catch (Exception e) { log.error("Error in proxying falling back to super call", e); } - if (response.getStatus() >= 500){ + if (response.getStatus() >= 500) { errorCounter5xx.inc(); - } else if (response.getStatus() >= 400){ + } else if (response.getStatus() >= 400) { errorCounter4xx.inc(); } - super.postConnectionHook(request, response, buffer, offset, length, callback); } diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java index 6d4fbf99..b2b37c0f 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/module/HaGatewayProviderModule.java @@ -1,6 +1,5 @@ package com.lyft.data.gateway.ha.module; -import com.codahale.metrics.Counter; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.google.inject.Provides; @@ -25,7 +24,6 @@ import com.lyft.data.proxyserver.ProxyServerConfiguration; import io.dropwizard.setup.Environment; -import static com.codahale.metrics.MetricRegistry.name; public class HaGatewayProviderModule extends AppModule { From d9cfe8ddbb61986a9320abe8f254512911ab929a Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 18:29:02 -0400 Subject: [PATCH 29/56] update dropwizard ver --- .../data/gateway/ha/handler/QueryIdCachingProxyHandler.java | 3 +-- proxyserver/pom.xml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java index 270fb2df..19395718 100644 --- a/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java +++ b/gateway-ha/src/main/java/com/lyft/data/gateway/ha/handler/QueryIdCachingProxyHandler.java @@ -64,7 +64,6 @@ public class QueryIdCachingProxyHandler extends ProxyHandler { private final Counter errorCounter4xx; private final Counter errorCounter5xx; - private final MetricRegistry metrics = new MetricRegistry(); public QueryIdCachingProxyHandler( QueryHistoryManager queryHistoryManager, @@ -75,7 +74,7 @@ public QueryIdCachingProxyHandler( MetricRegistry metrics) { this.requestMeter = requestMeter; this.errorCounter4xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "4xx")); - this.errorCounter5xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "5xx"));; + this.errorCounter5xx = metrics.counter(name(QueryIdCachingProxyHandler.class, "5xx")); this.routingManager = routingManager; this.routingGroupSelector = routingGroupSelector; this.queryHistoryManager = queryHistoryManager; diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index 9ad19af2..5b318e41 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -18,7 +18,7 @@ 1.8 UTF-8 UTF-8 - 1.3.29 + 1.3.7 From 0fee702cf9ea44120a72373da4583b43b08e32f8 Mon Sep 17 00:00:00 2001 From: Jeana Date: Tue, 2 Jul 2024 18:33:50 -0400 Subject: [PATCH 30/56] updating dropwizard dependency --- proxyserver/pom.xml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index 5b318e41..b679c39b 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -23,6 +23,11 @@ + + io.dropwizard + dropwizard-core + ${dropwizard.version} + org.eclipse.jetty jetty-server @@ -63,12 +68,6 @@ mockwebserver test - - io.dropwizard.metrics - metrics-core - 4.2.7 - compile - From 94ed9d8115b2f1519d0a57b282da25b1a9eafab8 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 2 Jul 2024 20:21:21 -0700 Subject: [PATCH 31/56] update --- gateway-ha/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 8b321801..0c790da8 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -20,7 +20,7 @@ UTF-8 UTF-8 9.4.48.v20220622 - 1.3.29 + 1.3.7 4.1.0 23.0 4.1.0 From 2cb6d6695b1577f0c79b1d3f255546f3ca8ed938 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 3 Jul 2024 13:44:52 -0700 Subject: [PATCH 32/56] pom changes --- gateway-ha/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 0c790da8..8b321801 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -20,7 +20,7 @@ UTF-8 UTF-8 9.4.48.v20220622 - 1.3.7 + 1.3.29 4.1.0 23.0 4.1.0 From 1c25d758140a75e1b93a3a5264b3e269ca7aae1d Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 3 Jul 2024 15:07:12 -0700 Subject: [PATCH 33/56] try request logging --- .../com/lyft/data/proxyserver/ProxyServer.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index b3ac5a42..1fe5d647 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -10,16 +10,22 @@ import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.ResponseHandler; import org.apache.http.util.TextUtils; +import org.eclipse.jetty.http.HttpParser; import org.eclipse.jetty.http.HttpScheme; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.proxy.ConnectHandler; +import org.eclipse.jetty.server.CustomRequestLog; +import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.SslConnectionFactory; +import org.eclipse.jetty.server.handler.HandlerCollection; +import org.eclipse.jetty.server.handler.RequestLogHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; @@ -97,7 +103,16 @@ private void setupContext(ProxyServerConfiguration config) { // Setup proxy handler to handle CONNECT methods ConnectHandler proxyConnectHandler = new ConnectHandler(); - this.server.setHandler(proxyConnectHandler); + + HandlerCollection handlers = new HandlerCollection(); + + RequestLogHandler requestLogHandler = new RequestLogHandler(); + //possible not needed + //requestLogHandler.setRequestLog(customRequestLog); + handlers.setHandlers(new Handler[] { requestLogHandler, proxyConnectHandler }); + + + this.server.setHandler(handlers); if (proxyHandler != null) { proxy.setProxyHandler(proxyHandler); From 76f038b89587acde27a79df7b4d362b34bfd08a8 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 3 Jul 2024 15:51:29 -0700 Subject: [PATCH 34/56] add handler --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 1fe5d647..7e1d65cd 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -116,19 +116,20 @@ private void setupContext(ProxyServerConfiguration config) { if (proxyHandler != null) { proxy.setProxyHandler(proxyHandler); + } ServletHolder proxyServlet = new ServletHolder(config.getName(), proxy); - proxyServlet.setInitParameter("proxyTo", config.getProxyTo()); proxyServlet.setInitParameter("prefix", config.getPrefix()); proxyServlet.setInitParameter("trustAll", config.getTrustAll()); proxyServlet.setInitParameter("preserveHost", config.getPreserveHost()); proxyServlet.setInitParameter("timeout", "120000"); + // Setup proxy servlet this.context = - new ServletContextHandler(proxyConnectHandler, "/", ServletContextHandler.SESSIONS); + new ServletContextHandler(handlers, "/", ServletContextHandler.SESSIONS); this.context.addServlet(proxyServlet, "/*"); this.context.addFilter(RequestFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); } From 30a065be67381edb4a7a34b5b7991200bbcc8652 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 9 Jul 2024 11:55:56 -0700 Subject: [PATCH 35/56] changes --- baseapp/pom.xml | 2 +- pom.xml | 12 +++ proxyserver/pom.xml | 12 +-- .../lyft/data/proxyserver/ProxyHandler.java | 1 - .../lyft/data/proxyserver/ProxyServer.java | 86 ++++++++++++++++++- 5 files changed, 101 insertions(+), 12 deletions(-) diff --git a/baseapp/pom.xml b/baseapp/pom.xml index 3662cb1f..c6be2d49 100644 --- a/baseapp/pom.xml +++ b/baseapp/pom.xml @@ -17,7 +17,7 @@ 1.8 UTF-8 UTF-8 - 1.3.7 + 1.3.29 4.1.0 0.9.10 diff --git a/pom.xml b/pom.xml index 69331ab5..35ca51f5 100644 --- a/pom.xml +++ b/pom.xml @@ -118,4 +118,16 @@ + + + lyft-releases + release + https://artifactory.lyft.net/artifactory/virtual-maven-libs-release + + + lyft-snapshots + libs-snapshot + https://artifactory.lyft.net/artifactory/virtual-maven-libs-snapshot + + diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index b679c39b..e140e783 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -18,16 +18,10 @@ 1.8 UTF-8 UTF-8 - 1.3.7 - - io.dropwizard - dropwizard-core - ${dropwizard.version} - org.eclipse.jetty jetty-server @@ -68,6 +62,12 @@ mockwebserver test + + com.fasterxml.jackson.core + jackson-databind + 2.14.2 + compile + diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index b34153e3..59f14dcd 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -8,7 +8,6 @@ import java.util.Collection; import java.util.Enumeration; import java.util.zip.GZIPInputStream; - import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 7e1d65cd..83b484b6 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -1,15 +1,16 @@ package com.lyft.data.proxyserver; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.Closeable; import java.io.File; import java.util.EnumSet; +import java.util.Enumeration; import java.util.concurrent.TimeUnit; import javax.servlet.DispatcherType; import javax.servlet.Filter; - import lombok.extern.slf4j.Slf4j; - import org.apache.http.client.ResponseHandler; import org.apache.http.util.TextUtils; import org.eclipse.jetty.http.HttpParser; @@ -20,9 +21,12 @@ import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.Slf4jRequestLogWriter; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.HandlerCollection; import org.eclipse.jetty.server.handler.RequestLogHandler; @@ -106,12 +110,84 @@ private void setupContext(ProxyServerConfiguration config) { HandlerCollection handlers = new HandlerCollection(); + Slf4jRequestLogWriter slfjRequestLogWriter = new Slf4jRequestLogWriter(); + slfjRequestLogWriter.setLoggerName("request.log"); + //String format = "%{client}a - %u %t '%r' %s %O '%{Referer}i' '%{User-Agent}i' '%C'"; + String myFormat = + "ACCESS LOG %{client}a - %u %t \"%r\" %s %O \"%{Referer}i\" \"%{User-Agent}i\" **%T/%D**"; + + + CustomRequestLog customRequestLog1 = new CustomRequestLog(slfjRequestLogWriter,myFormat) { + @Override + public void log(Request request, Response response) { + // Access properties of the request object here + + String clientAddress = request.getRemoteAddr(); + String username = request.getRemoteUser(); + String requestTime = String.valueOf(request.getTimeStamp()); + String requestMethod = request.getMethod(); + String requestUri = request.getRequestURI(); + int responseStatus = response.getStatus(); + long responseSize = response.getContentCount(); + String referer = request.getHeader("Referer"); + String userAgent = request.getHeader("User-Agent"); + long requestDurationMs = System.currentTimeMillis() - request.getTimeStamp(); + + String logMessageString = "ACCESS LOG == " + clientAddress + " - " + + (username != null ? username : "-") + " " + requestTime + + " \"" + requestMethod + " " + requestUri + " " + request.getProtocol() + + "\" " + responseStatus + " " + responseSize + " \"" + + (referer != null ? referer : "-") + "\" \"" + + (userAgent != null ? userAgent : "-") + + "\" **" + (requestDurationMs / 1000) + "/" + requestDurationMs + "**"; + + // Prepare the JSON log message + ObjectMapper mapper = new ObjectMapper(); + ObjectNode logMessage = mapper.createObjectNode(); + logMessage.put("clientAddress", clientAddress); + logMessage.put("username", username != null ? username : "-"); + logMessage.put("requestTime", requestTime); + logMessage.put("requestMethod", requestMethod); + logMessage.put("requestURI", requestUri); + logMessage.put("protocol", request.getProtocol()); + logMessage.put("responseStatus", responseStatus); + logMessage.put("responseSize", responseSize); + logMessage.put("referer", referer != null ? referer : "-"); + logMessage.put("userAgent", userAgent != null ? userAgent : "-"); + logMessage.put("requestDurationMs", requestDurationMs); + try { + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String header = headerNames.nextElement(); + logMessage.put("request_header_" + header, request.getHeader(header)); + } + + for (String i: response.getHeaderNames()) { + logMessage.put("request_header_" + i, response.getHeader(i)); + } + log.info("Request message: {}", request.toString()); + log.info("Response message: {}", response.toString()); + log.info("Response message: {}", response.toString()); + log.info("ACCESS LOG: {} : {}", logMessage.toString(), + request.getHeader("proxytarget"), request.getHeaderNames()); + } catch (Exception e) { + log.error("Error logging access log message", e); + } + + + //log.info(logMessageString); + } + }; + + + RequestLogHandler requestLogHandler = new RequestLogHandler(); + + //CustomRequestLog customRequestLog = new CustomRequestLog(slfjRequestLogWriter, myFormat); + this.server.setRequestLog(customRequestLog1); //possible not needed //requestLogHandler.setRequestLog(customRequestLog); handlers.setHandlers(new Handler[] { requestLogHandler, proxyConnectHandler }); - - this.server.setHandler(handlers); if (proxyHandler != null) { @@ -127,10 +203,12 @@ private void setupContext(ProxyServerConfiguration config) { proxyServlet.setInitParameter("timeout", "120000"); + // Setup proxy servlet this.context = new ServletContextHandler(handlers, "/", ServletContextHandler.SESSIONS); this.context.addServlet(proxyServlet, "/*"); + this.context.addFilter(RequestFilter.class, "/*", EnumSet.allOf(DispatcherType.class)); } From 341c07a47be6fa639e49441aace76c0638644fc7 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 9 Jul 2024 12:01:18 -0700 Subject: [PATCH 36/56] changes --- .../lyft/data/proxyserver/ProxyHandler.java | 31 +++---------------- .../lyft/data/proxyserver/ProxyServer.java | 13 +------- 2 files changed, 6 insertions(+), 38 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 59f14dcd..b9ef496d 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -41,7 +41,7 @@ public void preConnectionHook(HttpServletRequest request, Request proxyRequest) + "toString {}, getContentLength {}, getRequestHeaderSize {}, requestHeaders {}", request.getRequestURL(), request.getRequestURI(), request.getServletPath(), request.toString(), request.getContentLength(), getRequestHeaderSize(request), - errorLogHeaders(request)); + logRequestHeaders(request)); } } @@ -68,29 +68,8 @@ protected void postConnectionHook( request.getRequestURL()); } response.getOutputStream().write(buffer, offset, length); - // [sev-16337] with a 10% probably, log the request and response headers - // and size for debugging - if (Math.random() < 0.10) { - log.debug("(postConnectionHook) Request URL: {} , request URI {} , servlet path {} , " - + "toString {}, getContentLength {}, getRequestHeaderSize {}, " - + "getResponseHeaderSize {}, requestHeaders {}, responseHeaders {}", - request.getRequestURL(), request.getRequestURI(), request.getServletPath(), - request.toString(), request.getContentLength(), getRequestHeaderSize(request), - getResponseHeaderSize(response), errorLogHeaders(request), - errorLogHeaders(response)); - } - callback.succeeded(); } catch (Throwable var9) { - log.error("(postConnectionHook) Exception occurred while processing request URL: {} , " - + "request URI {} , servlet path {} , toString {}, getContentLength {}, " - + "getRequestHeaderSize {}, getResponseHeaderSize {}, requestHeaders {}, " - + "responseHeaders {}", - request.getRequestURL(), request.getRequestURI(), request.getServletPath(), - request.toString(), request.getContentLength(), getRequestHeaderSize(request), - getResponseHeaderSize(response), errorLogHeaders(request), - errorLogHeaders(response), var9); - callback.failed(var9); } } @@ -115,8 +94,8 @@ private int getResponseHeaderSize(HttpServletResponse response) { return headerSize; } - protected String errorLogHeaders(HttpServletRequest request) { - StringBuilder sb = new StringBuilder("------- error HttpServletRequest headers---------"); + protected String logRequestHeaders(HttpServletRequest request) { + StringBuilder sb = new StringBuilder("------- HttpServletRequest headers---------"); Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String header = headers.nextElement(); @@ -126,8 +105,8 @@ protected String errorLogHeaders(HttpServletRequest request) { return sb.toString(); } - protected String errorLogHeaders(HttpServletResponse response) { - StringBuilder sb = new StringBuilder("------- error HttpServletResponse headers---------"); + protected String logResponseHeaders(HttpServletResponse response) { + StringBuilder sb = new StringBuilder("------- HttpServletResponse headers---------"); Collection headers = response.getHeaderNames(); for (String header : headers) { sb.append(header + "->" + response.getHeader(header) + "\n"); diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 83b484b6..724c8275 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -88,7 +88,6 @@ private void setupContext(ProxyServerConfiguration config) { src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); src.setStsIncludeSubDomains(true); httpsConfig.addCustomizer(src); - httpsConfig.addCustomizer(new org.eclipse.jetty.server.ForwardedRequestCustomizer()); connector = new ServerConnector( server, @@ -179,20 +178,12 @@ public void log(Request request, Response response) { } }; - - - RequestLogHandler requestLogHandler = new RequestLogHandler(); - - //CustomRequestLog customRequestLog = new CustomRequestLog(slfjRequestLogWriter, myFormat); this.server.setRequestLog(customRequestLog1); - //possible not needed - //requestLogHandler.setRequestLog(customRequestLog); - handlers.setHandlers(new Handler[] { requestLogHandler, proxyConnectHandler }); + handlers.setHandlers(new Handler[] { proxyConnectHandler }); this.server.setHandler(handlers); if (proxyHandler != null) { proxy.setProxyHandler(proxyHandler); - } ServletHolder proxyServlet = new ServletHolder(config.getName(), proxy); @@ -202,8 +193,6 @@ public void log(Request request, Response response) { proxyServlet.setInitParameter("preserveHost", config.getPreserveHost()); proxyServlet.setInitParameter("timeout", "120000"); - - // Setup proxy servlet this.context = new ServletContextHandler(handlers, "/", ServletContextHandler.SESSIONS); From 32145ed43fa89f1a122e8b3a6a487c141976d69c Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 10 Jul 2024 03:09:03 +0530 Subject: [PATCH 37/56] Update gateway-ha-config.yml --- gateway-ha/gateway-ha-config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gateway-ha/gateway-ha-config.yml b/gateway-ha/gateway-ha-config.yml index e8cd62e0..70f1ee92 100644 --- a/gateway-ha/gateway-ha-config.yml +++ b/gateway-ha/gateway-ha-config.yml @@ -6,10 +6,6 @@ requestRouter: port: 8080 name: prestoRouter historySize: 1000 - requestHeaderSize: 2048000 - responseHeaderSize: 2048000 - requestBufferSize: 2048000 - responseBufferSize: 2048000 dataStore: jdbcUrl: jdbc:mysql://127.0.0.1:3306/prestogateway From 2bb4d5c07cae98354c62071450ff86ac0a624f6f Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 10 Jul 2024 03:09:52 +0530 Subject: [PATCH 38/56] Update ProxyServer.java --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 724c8275..db5b2f8f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -79,10 +79,7 @@ private void setupContext(ProxyServerConfiguration config) { HttpConfiguration httpsConfig = new HttpConfiguration(); httpsConfig.setSecureScheme(HttpScheme.HTTPS.asString()); httpsConfig.setSecurePort(config.getLocalPort()); - httpsConfig.setIdleTimeout(150000); httpsConfig.setOutputBufferSize(32768); - httpsConfig.setRequestHeaderSize(2048000); - httpsConfig.setResponseHeaderSize(2048000); SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); From 25373fc7613794dba4cab867ceeb990beb1e94a9 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 10 Jul 2024 03:10:25 +0530 Subject: [PATCH 39/56] Update ProxyServer.java --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index db5b2f8f..b742c225 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -97,8 +97,6 @@ private void setupContext(ProxyServerConfiguration config) { connector.setPort(config.getLocalPort()); connector.setName(config.getName()); connector.setAccepting(true); - connector.setIdleTimeout(150000L); - connector.setAcceptQueueSize(1024); this.server.addConnector(connector); // Setup proxy handler to handle CONNECT methods From 4075810504799559168623186e0d67b6d7227290 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 10 Jul 2024 03:10:54 +0530 Subject: [PATCH 40/56] Update ProxyServer.java --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index b742c225..15e9a2e2 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -186,7 +186,6 @@ public void log(Request request, Response response) { proxyServlet.setInitParameter("prefix", config.getPrefix()); proxyServlet.setInitParameter("trustAll", config.getTrustAll()); proxyServlet.setInitParameter("preserveHost", config.getPreserveHost()); - proxyServlet.setInitParameter("timeout", "120000"); // Setup proxy servlet this.context = From 87c64f903ef1aa532fd327e330cc87969ff34161 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 10 Jul 2024 03:11:26 +0530 Subject: [PATCH 41/56] Update ProxyServletImpl.java --- .../java/com/lyft/data/proxyserver/ProxyServletImpl.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java index 8a1c36c2..48261769 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java @@ -45,10 +45,7 @@ protected HttpClient newHttpClient() { HttpClient httpClient = new HttpClient(sslFactory); httpClient.setMaxConnectionsPerDestination(10000); - httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(65)); - httpClient.setIdleTimeout(TimeUnit.SECONDS.toMillis(65)); - httpClient.setRequestBufferSize(2048000); - httpClient.setResponseBufferSize(2048000); + httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(60)); return httpClient; } From e11ede048509bf9163e8c86004cc50de47c8886a Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Tue, 9 Jul 2024 17:56:10 -0700 Subject: [PATCH 42/56] add configs --- .../java/com/lyft/data/proxyserver/ProxyServletImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java index 48261769..8a1c36c2 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java @@ -45,7 +45,10 @@ protected HttpClient newHttpClient() { HttpClient httpClient = new HttpClient(sslFactory); httpClient.setMaxConnectionsPerDestination(10000); - httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(60)); + httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(65)); + httpClient.setIdleTimeout(TimeUnit.SECONDS.toMillis(65)); + httpClient.setRequestBufferSize(2048000); + httpClient.setResponseBufferSize(2048000); return httpClient; } From 36fe69bb24ee9ecc662b10c03bbae85cf92af89a Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 10 Jul 2024 09:03:02 -0700 Subject: [PATCH 43/56] more configs --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 15e9a2e2..58694f79 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -80,6 +80,8 @@ private void setupContext(ProxyServerConfiguration config) { httpsConfig.setSecureScheme(HttpScheme.HTTPS.asString()); httpsConfig.setSecurePort(config.getLocalPort()); httpsConfig.setOutputBufferSize(32768); + httpsConfig.setRequestHeaderSize(2048000); + httpsConfig.setResponseHeaderSize(2048000); SecureRequestCustomizer src = new SecureRequestCustomizer(); src.setStsMaxAge(TimeUnit.SECONDS.toSeconds(2000)); From 0f4ad3c6cf71634ec4753b05797c144c94b81be1 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Wed, 10 Jul 2024 12:45:35 -0700 Subject: [PATCH 44/56] Revert "add configs" This reverts commit e11ede048509bf9163e8c86004cc50de47c8886a. --- .../java/com/lyft/data/proxyserver/ProxyServletImpl.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java index 8a1c36c2..48261769 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServletImpl.java @@ -45,10 +45,7 @@ protected HttpClient newHttpClient() { HttpClient httpClient = new HttpClient(sslFactory); httpClient.setMaxConnectionsPerDestination(10000); - httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(65)); - httpClient.setIdleTimeout(TimeUnit.SECONDS.toMillis(65)); - httpClient.setRequestBufferSize(2048000); - httpClient.setResponseBufferSize(2048000); + httpClient.setConnectTimeout(TimeUnit.SECONDS.toMillis(60)); return httpClient; } From e7084370739bf6ed7fc492246468f12387149e74 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Thu, 11 Jul 2024 21:45:46 +0530 Subject: [PATCH 45/56] Update ProxyServer.java --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 58694f79..dd1b101e 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -99,6 +99,8 @@ private void setupContext(ProxyServerConfiguration config) { connector.setPort(config.getLocalPort()); connector.setName(config.getName()); connector.setAccepting(true); + connector.setIdleTimeout(150000L); + connector.setAcceptQueueSize(1024); this.server.addConnector(connector); // Setup proxy handler to handle CONNECT methods From ee461847034cb5557ea83666fb7ddf1210ed89b1 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Fri, 26 Jul 2024 01:40:34 +0530 Subject: [PATCH 46/56] Update ProxyHandler.java --- .../java/com/lyft/data/proxyserver/ProxyHandler.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index b9ef496d..a56e9339 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -33,16 +33,6 @@ protected String rewriteTarget(HttpServletRequest request) { * @param proxyRequest */ public void preConnectionHook(HttpServletRequest request, Request proxyRequest) { - // you may override it. - - // [sev-16337] with a 10% probably, log the request headers for debugging - if (Math.random() < 0.10) { - log.debug("(preConnectionHook) Request URL: {} , request URI {} , servlet path {} ," - + "toString {}, getContentLength {}, getRequestHeaderSize {}, requestHeaders {}", - request.getRequestURL(), request.getRequestURI(), request.getServletPath(), - request.toString(), request.getContentLength(), getRequestHeaderSize(request), - logRequestHeaders(request)); - } } /** From 2bde674d55e5671774f5525ac5f93208b9690930 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Fri, 26 Jul 2024 01:41:20 +0530 Subject: [PATCH 47/56] Update ProxyHandler.java --- .../src/main/java/com/lyft/data/proxyserver/ProxyHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index a56e9339..34920a5d 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -33,6 +33,7 @@ protected String rewriteTarget(HttpServletRequest request) { * @param proxyRequest */ public void preConnectionHook(HttpServletRequest request, Request proxyRequest) { + // you may override it. } /** From 6833beb7264c618585a7701d71d3c6d21cf4f446 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Fri, 26 Jul 2024 01:42:26 +0530 Subject: [PATCH 48/56] Update ProxyHandler.java --- .../lyft/data/proxyserver/ProxyHandler.java | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java index 34920a5d..fb531b9d 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyHandler.java @@ -65,47 +65,6 @@ protected void postConnectionHook( } } - protected int getRequestHeaderSize(HttpServletRequest request) { - int headerSize = 0; - Enumeration headerNames = request.getHeaderNames(); - while (headerNames.hasMoreElements()) { - String headerName = headerNames.nextElement(); - String headerValue = request.getHeader(headerName); - headerSize += headerName.length() + headerValue.length(); - } - return headerSize; - } - - private int getResponseHeaderSize(HttpServletResponse response) { - int headerSize = 0; - for (String headerName : response.getHeaderNames()) { - String headerValue = response.getHeader(headerName); - headerSize += headerName.length() + headerValue.length(); - } - return headerSize; - } - - protected String logRequestHeaders(HttpServletRequest request) { - StringBuilder sb = new StringBuilder("------- HttpServletRequest headers---------"); - Enumeration headers = request.getHeaderNames(); - while (headers.hasMoreElements()) { - String header = headers.nextElement(); - sb.append(header + "->" + request.getHeader(header) + "\n"); - } - - return sb.toString(); - } - - protected String logResponseHeaders(HttpServletResponse response) { - StringBuilder sb = new StringBuilder("------- HttpServletResponse headers---------"); - Collection headers = response.getHeaderNames(); - for (String header : headers) { - sb.append(header + "->" + response.getHeader(header) + "\n"); - } - - return sb.toString(); - } - protected void debugLogHeaders(HttpServletRequest request) { if (log.isDebugEnabled()) { log.debug("-------HttpServletRequest headers---------"); From 11c58cdc87af0c79fa4fb6022b37fe6933ec874e Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Fri, 26 Jul 2024 01:49:04 +0530 Subject: [PATCH 49/56] Update ProxyServer.java --- .../lyft/data/proxyserver/ProxyServer.java | 137 +++++++++--------- 1 file changed, 69 insertions(+), 68 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index dd1b101e..931a3c3b 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -40,6 +40,7 @@ public class ProxyServer implements Closeable { private final ProxyServletImpl proxy; private final ProxyHandler proxyHandler; private ServletContextHandler context; + private boolean accessLogEnabled; public ProxyServer(ProxyServerConfiguration config, ProxyHandler proxyHandler) { this(config, proxyHandler, new ProxyServletImpl()); @@ -54,6 +55,8 @@ public ProxyServer(ProxyServerConfiguration config, ProxyHandler proxyHandler, this.proxy.setServerConfig(config); this.setupContext(config); + //cheap flag to turn on Jetty access logging + this.accessLogEnabled = false; } private void setupContext(ProxyServerConfiguration config) { @@ -108,76 +111,74 @@ private void setupContext(ProxyServerConfiguration config) { HandlerCollection handlers = new HandlerCollection(); - Slf4jRequestLogWriter slfjRequestLogWriter = new Slf4jRequestLogWriter(); - slfjRequestLogWriter.setLoggerName("request.log"); - //String format = "%{client}a - %u %t '%r' %s %O '%{Referer}i' '%{User-Agent}i' '%C'"; - String myFormat = - "ACCESS LOG %{client}a - %u %t \"%r\" %s %O \"%{Referer}i\" \"%{User-Agent}i\" **%T/%D**"; - - - CustomRequestLog customRequestLog1 = new CustomRequestLog(slfjRequestLogWriter,myFormat) { - @Override - public void log(Request request, Response response) { - // Access properties of the request object here - - String clientAddress = request.getRemoteAddr(); - String username = request.getRemoteUser(); - String requestTime = String.valueOf(request.getTimeStamp()); - String requestMethod = request.getMethod(); - String requestUri = request.getRequestURI(); - int responseStatus = response.getStatus(); - long responseSize = response.getContentCount(); - String referer = request.getHeader("Referer"); - String userAgent = request.getHeader("User-Agent"); - long requestDurationMs = System.currentTimeMillis() - request.getTimeStamp(); - - String logMessageString = "ACCESS LOG == " + clientAddress + " - " - + (username != null ? username : "-") + " " + requestTime - + " \"" + requestMethod + " " + requestUri + " " + request.getProtocol() - + "\" " + responseStatus + " " + responseSize + " \"" - + (referer != null ? referer : "-") + "\" \"" - + (userAgent != null ? userAgent : "-") - + "\" **" + (requestDurationMs / 1000) + "/" + requestDurationMs + "**"; - - // Prepare the JSON log message - ObjectMapper mapper = new ObjectMapper(); - ObjectNode logMessage = mapper.createObjectNode(); - logMessage.put("clientAddress", clientAddress); - logMessage.put("username", username != null ? username : "-"); - logMessage.put("requestTime", requestTime); - logMessage.put("requestMethod", requestMethod); - logMessage.put("requestURI", requestUri); - logMessage.put("protocol", request.getProtocol()); - logMessage.put("responseStatus", responseStatus); - logMessage.put("responseSize", responseSize); - logMessage.put("referer", referer != null ? referer : "-"); - logMessage.put("userAgent", userAgent != null ? userAgent : "-"); - logMessage.put("requestDurationMs", requestDurationMs); - try { - Enumeration headerNames = request.getHeaderNames(); - while (headerNames.hasMoreElements()) { - String header = headerNames.nextElement(); - logMessage.put("request_header_" + header, request.getHeader(header)); + if (this.accessLogEnabled) { + Slf4jRequestLogWriter slfjRequestLogWriter = new Slf4jRequestLogWriter(); + slfjRequestLogWriter.setLoggerName("request.log"); + String myFormat = + "ACCESS LOG %{client}a - %u %t \"%r\" %s %O \"%{Referer}i\" \"%{User-Agent}i\" **%T/%D**"; + + + CustomRequestLog requestLog = new CustomRequestLog(slfjRequestLogWriter,myFormat) { + @Override + public void log(Request request, Response response) { + String clientAddress = request.getRemoteAddr(); + String username = request.getRemoteUser(); + String requestTime = String.valueOf(request.getTimeStamp()); + String requestMethod = request.getMethod(); + String requestUri = request.getRequestURI(); + int responseStatus = response.getStatus(); + long responseSize = response.getContentCount(); + String referer = request.getHeader("Referer"); + String userAgent = request.getHeader("User-Agent"); + long requestDurationMs = System.currentTimeMillis() - request.getTimeStamp(); + + String logMessageString = "ACCESS LOG == " + clientAddress + " - " + + (username != null ? username : "-") + " " + requestTime + + " \"" + requestMethod + " " + requestUri + " " + request.getProtocol() + + "\" " + responseStatus + " " + responseSize + " \"" + + (referer != null ? referer : "-") + "\" \"" + + (userAgent != null ? userAgent : "-") + + "\" **" + (requestDurationMs / 1000) + "/" + requestDurationMs + "**"; + + ObjectMapper mapper = new ObjectMapper(); + ObjectNode logMessage = mapper.createObjectNode(); + logMessage.put("clientAddress", clientAddress); + logMessage.put("username", username != null ? username : "-"); + logMessage.put("requestTime", requestTime); + logMessage.put("requestMethod", requestMethod); + logMessage.put("requestURI", requestUri); + logMessage.put("protocol", request.getProtocol()); + logMessage.put("responseStatus", responseStatus); + logMessage.put("responseSize", responseSize); + logMessage.put("referer", referer != null ? referer : "-"); + logMessage.put("userAgent", userAgent != null ? userAgent : "-"); + logMessage.put("requestDurationMs", requestDurationMs); + try { + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String header = headerNames.nextElement(); + logMessage.put("request_header_" + header, request.getHeader(header)); + } + + for (String i: response.getHeaderNames()) { + logMessage.put("request_header_" + i, response.getHeader(i)); + } + + log.info("ACCESS LOG: {} : {}", logMessage.toString(), + request.getHeader("proxytarget"), request.getHeaderNames()); + + } catch (Exception e) { + log.error("Error logging access log message", e); } - - for (String i: response.getHeaderNames()) { - logMessage.put("request_header_" + i, response.getHeader(i)); - } - log.info("Request message: {}", request.toString()); - log.info("Response message: {}", response.toString()); - log.info("Response message: {}", response.toString()); - log.info("ACCESS LOG: {} : {}", logMessage.toString(), - request.getHeader("proxytarget"), request.getHeaderNames()); - } catch (Exception e) { - log.error("Error logging access log message", e); + + + //log.info(logMessageString); } - - - //log.info(logMessageString); - } - }; - - this.server.setRequestLog(customRequestLog1); + }; + + this.server.setRequestLog(customRequestLog); + } + handlers.setHandlers(new Handler[] { proxyConnectHandler }); this.server.setHandler(handlers); From 69cbd7fbb36ef8b00084ed3c2217fcb83fed685c Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Thu, 25 Jul 2024 13:21:06 -0700 Subject: [PATCH 50/56] final changes --- .../src/main/java/com/lyft/data/proxyserver/ProxyServer.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 931a3c3b..0a948a2f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -7,13 +7,10 @@ import java.util.EnumSet; import java.util.Enumeration; import java.util.concurrent.TimeUnit; - import javax.servlet.DispatcherType; import javax.servlet.Filter; import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.ResponseHandler; import org.apache.http.util.TextUtils; -import org.eclipse.jetty.http.HttpParser; import org.eclipse.jetty.http.HttpScheme; import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.proxy.ConnectHandler; @@ -29,7 +26,6 @@ import org.eclipse.jetty.server.Slf4jRequestLogWriter; import org.eclipse.jetty.server.SslConnectionFactory; import org.eclipse.jetty.server.handler.HandlerCollection; -import org.eclipse.jetty.server.handler.RequestLogHandler; import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; From ff0976c8e9e3b2ba3a9a29b8d4b7307f726b2b01 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Fri, 26 Jul 2024 01:51:30 +0530 Subject: [PATCH 51/56] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 35ca51f5..2f1ec24a 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ prestogateway-parent prestogateway-parent pom - 1.9.6-SNAPSHOT + 1.9.7 1.8 From 893eb2eb660e765ded000849157bb9b03f60bb51 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Thu, 25 Jul 2024 13:25:39 -0700 Subject: [PATCH 52/56] bug fix --- .../main/java/com/lyft/data/proxyserver/ProxyServer.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java index 0a948a2f..cb5d0a1f 100644 --- a/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java +++ b/proxyserver/src/main/java/com/lyft/data/proxyserver/ProxyServer.java @@ -166,13 +166,11 @@ public void log(Request request, Response response) { } catch (Exception e) { log.error("Error logging access log message", e); } - - - //log.info(logMessageString); + } }; - this.server.setRequestLog(customRequestLog); + this.server.setRequestLog(requestLog); } handlers.setHandlers(new Handler[] { proxyConnectHandler }); From e47fd6162ac734ea6d1adc59b6a88d01fd0d01f0 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Thu, 25 Jul 2024 13:57:51 -0700 Subject: [PATCH 53/56] updates --- baseapp/pom.xml | 2 +- gateway-ha/pom.xml | 2 +- proxyserver/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/baseapp/pom.xml b/baseapp/pom.xml index c6be2d49..11a326d7 100644 --- a/baseapp/pom.xml +++ b/baseapp/pom.xml @@ -5,7 +5,7 @@ com.lyft.data prestogateway-parent - 1.9.6-SNAPSHOT + ${project.parent.version} ../ diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 8b321801..2cea9e44 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -6,11 +6,11 @@ com.lyft.data prestogateway-parent - 1.9.6-SNAPSHOT ../ gateway-ha + ${project.parent.version} jar gateway-ha diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index e140e783..ab95579e 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -6,11 +6,11 @@ com.lyft.data prestogateway-parent - 1.9.6-SNAPSHOT ../ proxyserver + ${project.parent.version} jar From 79be7f013ec66ef0a618c454a610874b3ec6d727 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Thu, 25 Jul 2024 13:59:38 -0700 Subject: [PATCH 54/56] version fix --- baseapp/pom.xml | 2 +- gateway-ha/pom.xml | 1 + proxyserver/pom.xml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/baseapp/pom.xml b/baseapp/pom.xml index 11a326d7..dc6e90da 100644 --- a/baseapp/pom.xml +++ b/baseapp/pom.xml @@ -5,7 +5,7 @@ com.lyft.data prestogateway-parent - ${project.parent.version} + 1.9.7 ../ diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 2cea9e44..471abb99 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -6,6 +6,7 @@ com.lyft.data prestogateway-parent + 1.9.7 ../ diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index ab95579e..bb55c7f1 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -6,6 +6,7 @@ com.lyft.data prestogateway-parent + 1.9.7 ../ From 4b1c0a7b86d7f2c5afd0e1a9cab1827663830426 Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Fri, 26 Jul 2024 02:31:55 +0530 Subject: [PATCH 55/56] Update pom.xml --- gateway-ha/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 471abb99..72f9ccf2 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -11,7 +11,6 @@ gateway-ha - ${project.parent.version} jar gateway-ha From cb485dfc3000455d4fcaa4cb2eb1dbed9d9117df Mon Sep 17 00:00:00 2001 From: Ritesh Varyani Date: Fri, 26 Jul 2024 02:32:07 +0530 Subject: [PATCH 56/56] Update pom.xml --- proxyserver/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/proxyserver/pom.xml b/proxyserver/pom.xml index bb55c7f1..46e93d4b 100644 --- a/proxyserver/pom.xml +++ b/proxyserver/pom.xml @@ -11,7 +11,6 @@ proxyserver - ${project.parent.version} jar