From c6609f0cd9315217727d44b3166f705acc4da0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Werner?= <4639399+jowerner@users.noreply.github.com> Date: Tue, 8 Aug 2023 18:04:16 +0200 Subject: [PATCH 1/7] #379: Report menu hover marker exceeds menu bar vertically On wide screens, the font size increases a bit. Ensure the height of the navigation bar is adjusted accordingly. --- config/testreport/css/default.css | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/config/testreport/css/default.css b/config/testreport/css/default.css index 699efd3db..f50bbc708 100644 --- a/config/testreport/css/default.css +++ b/config/testreport/css/default.css @@ -380,7 +380,6 @@ template { --navigation-dropdown-hover-bg-color: #e0e0e0; --navigation-line-height: 2; - --navigation-height: 25px; --header-full-height: 70px; --header-small-height: 30px; @@ -434,11 +433,17 @@ body } @media screen and (max-width:1399px) { + :root { + --navigation-height: 25px; + } html { font-size: 14px; } } @media screen and (min-width:1400px) { + :root { + --navigation-height: 28px; + } html { font-size: 16px; } From e7886afb925d43c1f3f6b68da3ec23cc653feeab Mon Sep 17 00:00:00 2001 From: Joerg Werner <4639399+jowerner@users.noreply.github.com> Date: Mon, 21 Aug 2023 12:06:01 +0200 Subject: [PATCH 2/7] - no longer add the "samples" subdirectory when creating the distribution archive - cleaned up some rules for dirty-checking or creating the archive --- build.xml | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/build.xml b/build.xml index 1ed1d0913..eff2432f4 100644 --- a/build.xml +++ b/build.xml @@ -271,16 +271,12 @@ - - - - - - - + + + @@ -337,40 +333,21 @@ - - - - - - - - - - - - - - - + - - - - - From 48541e3626b03e915e329abc8fcce650b2f9673c Mon Sep 17 00:00:00 2001 From: Joerg Werner <4639399+jowerner@users.noreply.github.com> Date: Mon, 21 Aug 2023 13:10:58 +0200 Subject: [PATCH 3/7] changed name of Chromium binary from "chromium-browser" to "chromium" --- build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.properties b/build.properties index dac729755..63eb4f2f2 100644 --- a/build.properties +++ b/build.properties @@ -31,7 +31,7 @@ resultbrowser.dir.source = resultbrowser/dist resultbrowser.dir.target = ${classes.dir}/com/xceptance/xlt/engine/resultbrowser/assets # Linux -timerrecorder.chrome.executable = chromium-browser +timerrecorder.chrome.executable = chromium # Windows #timerrecorder.chrome.executable = C:/Program Files (x86)/Google/Chrome/Application/chrome.exe # macOS From 3fef9f07ab944023b4ea9351f231cf771483ef79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Werner?= <4639399+jowerner@users.noreply.github.com> Date: Wed, 23 Aug 2023 11:02:16 +0200 Subject: [PATCH 4/7] #400: Implement chunked downloads of results (#409) * - Use a series of partial GETs to download/serve an archive file in chunks. - Fall back to a full download in case the agent controller does not support chunked download yet. - Retry downloading a file (chunk) in case of I/O errors. - Made chunk size and download attempts per chunk configurable. * changed name of Chromium binary from "chromium-browser" to "chromium" * feedback from reviewer * feedback from reviewer (part 2) * feedback from reviewer (part 3) * Fixed small typo in comment line. * feedback from reviewer (part 4) --------- Co-authored-by: Hartmut Arlt --- config/mastercontroller.properties | 10 + .../xceptance/common/io/IoActionHandler.java | 77 +++++++ .../agentcontroller/AgentControllerProxy.java | 43 +++- .../xlt/agentcontroller/FileManagerProxy.java | 206 ++++++++++++++++-- .../agentcontroller/FileManagerServlet.java | 99 ++++++++- .../xlt/agentcontroller/PartialGetUtils.java | 158 ++++++++++++++ .../MasterControllerConfiguration.java | 110 ++++++---- .../MasterControllerMain.java | 4 +- ...artialGetUtils_ContentRangeHeaderTest.java | 92 ++++++++ .../PartialGetUtils_RangeHeaderTest.java | 88 ++++++++ 10 files changed, 818 insertions(+), 69 deletions(-) create mode 100644 src/main/java/com/xceptance/common/io/IoActionHandler.java create mode 100644 src/main/java/com/xceptance/xlt/agentcontroller/PartialGetUtils.java create mode 100644 src/test/java/com/xceptance/xlt/agentcontroller/PartialGetUtils_ContentRangeHeaderTest.java create mode 100644 src/test/java/com/xceptance/xlt/agentcontroller/PartialGetUtils_RangeHeaderTest.java diff --git a/config/mastercontroller.properties b/config/mastercontroller.properties index 3dced773a..2bef089e3 100644 --- a/config/mastercontroller.properties +++ b/config/mastercontroller.properties @@ -63,6 +63,16 @@ com.xceptance.xlt.mastercontroller.password = xceptance #com.xceptance.xlt.mastercontroller.maxParallelUploads = -1 #com.xceptance.xlt.mastercontroller.maxParallelDownloads = -1 +## The size of a file chunk (in bytes, defaults to 100 MB; minimum value: 1000) +## when downloading test result archives from the agent controllers in chunks. +## The chunked mode will be used automatically unless the agent controller does +## not support it. +#com.xceptance.xlt.mastercontroller.download.chunkSize = 100000000 + +## The maximum number of retries if downloading a file (chunk) from an agent +## controller failed because of an I/O error (defaults to 1; minimum value: 0). +#com.xceptance.xlt.mastercontroller.download.maxRetries = 1 + # ================== # Result Storage # ================== diff --git a/src/main/java/com/xceptance/common/io/IoActionHandler.java b/src/main/java/com/xceptance/common/io/IoActionHandler.java new file mode 100644 index 000000000..25ae46ad1 --- /dev/null +++ b/src/main/java/com/xceptance/common/io/IoActionHandler.java @@ -0,0 +1,77 @@ +package com.xceptance.common.io; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An {@link IoActionHandler} runs an {@link IoAction}, and if that action fails with an {@link IOException} (or a + * subclass), repeats the action until the action eventually succeeds or the maximum number of retries is reached. + */ +public class IoActionHandler +{ + /** + * Functional interface describing an action that performs I/O and may fail with {@link IOException}. + * + * @param + * the type of the result of the action + */ + @FunctionalInterface + public interface IoAction + { + public T run() throws IOException; + } + + private static final Logger log = LoggerFactory.getLogger(IoActionHandler.class); + + /** + * The maximum number of retries. + */ + private final int maxRetries; + + /** + * Creates a new {@link IoActionHandler} initialized with the given number of retries. + * + * @param maxRetries + * the maximum number of retries + */ + public IoActionHandler(final int maxRetries) + { + this.maxRetries = maxRetries; + } + + /** + * Runs the passed {@link IoAction} performing retries if needed. + * + * @param action + * the action to perform + * @return the result of the action upon successful attempt + * @throws IOException + * the exception thrown by the underlying action if the maximum number of retries was reached + */ + public T run(final IoAction action) throws IOException + { + int remainingRetries = maxRetries; + + while (true) + { + try + { + return action.run(); + } + catch (final IOException e) + { + if (remainingRetries > 0) + { + log.debug("Retry action because of: {}", e.toString()); + remainingRetries--; + } + else + { + throw e; + } + } + } + } +} diff --git a/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerProxy.java b/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerProxy.java index 50646f9f6..3f904515c 100644 --- a/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerProxy.java +++ b/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerProxy.java @@ -39,6 +39,16 @@ public class AgentControllerProxy extends AgentControllerImpl { private static final Logger log = LoggerFactory.getLogger(AgentControllerProxy.class); + /** + * The default size of a file chunk when downloading a result archive from an agent controller. + */ + public static final long DEFAULT_DOWNLOAD_CHUNK_SIZE = 100_000_000L; + + /** + * The default maximum number of retries in case downloading a result file (chunk) failed because of an I/O error. + */ + public static final int DEFAULT_DOWNLOAD_MAX_RETRIES = 1; + /** * The client-side agent controller implementation. */ @@ -59,6 +69,16 @@ public class AgentControllerProxy extends AgentControllerImpl */ private final UrlConnectionFactory urlConnectionFactory; + /** + * The size of a file chunk when downloading a result archive from an agent controller. + */ + private final long downloadChunkSize; + + /** + * The maximum number of retries in case downloading a result file (chunk) failed because of an I/O error. + */ + private final int downloadMaxRetries; + /** * Creates a new AgentControllerProxy object. * @@ -69,11 +89,31 @@ public class AgentControllerProxy extends AgentControllerImpl public AgentControllerProxy(final Properties commandLineProperties, final HessianProxyFactory proxyFactory, final UrlConnectionFactory urlConnectionFactory) throws Exception + { + this(commandLineProperties, proxyFactory, urlConnectionFactory, DEFAULT_DOWNLOAD_CHUNK_SIZE, DEFAULT_DOWNLOAD_MAX_RETRIES); + } + + /** + * Creates a new AgentControllerProxy object. + * + * @param commandLineProperties + * @param proxyFactory + * @param urlConnectionFactory + * @param downloadChunkSize + * the size of a file chunk + * @param downloadMaxRetries + * the maximum number of download retries + */ + public AgentControllerProxy(final Properties commandLineProperties, final HessianProxyFactory proxyFactory, + final UrlConnectionFactory urlConnectionFactory, final long downloadChunkSize, final int downloadMaxRetries) + throws Exception { super(commandLineProperties); this.proxyFactory = proxyFactory; this.urlConnectionFactory = urlConnectionFactory; + this.downloadChunkSize = downloadChunkSize; + this.downloadMaxRetries = downloadMaxRetries; } /** @@ -130,8 +170,9 @@ public void setTotalAgentCount(final int totalAgentCount) public void startProxy(final URL url) throws MalformedURLException { log.info("start proxy for " + getName()); + // start file manager proxy - fileManager = new FileManagerProxy(url, urlConnectionFactory); + fileManager = new FileManagerProxy(url, urlConnectionFactory, downloadChunkSize, downloadMaxRetries); // start agent controller proxy agentController = (AgentController) proxyFactory.create(AgentController.class, diff --git a/src/main/java/com/xceptance/xlt/agentcontroller/FileManagerProxy.java b/src/main/java/com/xceptance/xlt/agentcontroller/FileManagerProxy.java index 7357d3582..e9d912ed5 100644 --- a/src/main/java/com/xceptance/xlt/agentcontroller/FileManagerProxy.java +++ b/src/main/java/com/xceptance/xlt/agentcontroller/FileManagerProxy.java @@ -21,17 +21,22 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.net.URLConnection; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.xceptance.common.io.IoActionHandler; import com.xceptance.common.net.UrlConnectionFactory; +import com.xceptance.xlt.agentcontroller.PartialGetUtils.ContentRangeHeaderData; +import com.xceptance.xlt.api.util.XltException; +import com.xceptance.xlt.engine.httprequest.HttpRequestHeaders; +import com.xceptance.xlt.engine.httprequest.HttpResponseHeaders; /** * The FileManagerProxy class is the client-side implementation of the FileManager interface, i.e. it runs on the master @@ -39,26 +44,63 @@ */ public class FileManagerProxy implements FileManager { + /** + * The result returned when downloading a chunk. + */ + private static class ChunkInfo + { + /** The total size of the resource that is currently being downloaded. */ + public final long totalSize; + + /** The size of the current chunk. */ + public final long chunkSize; + + public ChunkInfo(final long totalSize, final long chunkSize) + { + super(); + this.totalSize = totalSize; + this.chunkSize = chunkSize; + } + } + private static final Logger log = LoggerFactory.getLogger(FileManagerProxy.class); private final URL url; private final UrlConnectionFactory urlConnectionFactory; + /** + * The size of a file chunk when downloading a result archive from an agent controller. + */ + private final long downloadChunkSize; + + /** + * The maximum number of retries in case downloading a result file (chunk) failed because of an I/O error. + */ + private final int downloadMaxRetries; + /** * Creates a new FileManagerProxy object. - * + * * @param url * the agent controller's URL * @param urlConnectionFactory * the URL connection factory to use + * @param downloadChunkSize + * the size of a file chunk + * @param downloadMaxRetries + * the maximum number of download retries * @throws MalformedURLException * if the file manager's URL cannot be created */ - public FileManagerProxy(final URL url, final UrlConnectionFactory urlConnectionFactory) throws MalformedURLException + public FileManagerProxy(final URL url, final UrlConnectionFactory urlConnectionFactory, final long downloadChunkSize, + final int downloadMaxRetries) + throws MalformedURLException { this.url = new URL(url + FileManagerServlet.SERVLET_PATH); this.urlConnectionFactory = urlConnectionFactory; + this.downloadChunkSize = downloadChunkSize; + this.downloadMaxRetries = downloadMaxRetries; } /** @@ -75,28 +117,156 @@ public void deleteFile(final String remoteFileName) throws IOException @Override public void downloadFile(final File localFile, final String remoteFileName) throws IOException { - InputStream cin = null; - OutputStream fout = null; + final URL downloadUrl = new URL(url + remoteFileName); - try + log.debug("Downloading file from '{}' to '{}' ...", downloadUrl, localFile); + + // make sure the target directory exists + FileUtils.forceMkdir(localFile.getParentFile()); + + // prepare retry handling in case of I/O errors + final IoActionHandler ioActionHandler = new IoActionHandler(downloadMaxRetries); + + // download the file content in chunks + long bytesRead = 0; + long totalBytes = Long.MAX_VALUE; + + do + { + final long offset = bytesRead; + final long bytes = Math.min(downloadChunkSize, totalBytes - offset); + + final ChunkInfo chunkInfo = ioActionHandler.run(() -> downloadFileChunk(localFile, downloadUrl, offset, bytes)); + + bytesRead += chunkInfo.chunkSize; + totalBytes = chunkInfo.totalSize; + } + while (bytesRead < totalBytes); + } + + /** + * Downloads a file chunk from the given URL using a partial GET, falling back to downloading the full file if the + * agent controller does not support partial GETs. + * + * @param localFile + * the file to download the chunk to + * @param downloadUrl + * the URL to download the chunk from + * @param offset + * the position in the remote file to start the download from + * @param bytes + * the number of bytes to download + * @return a {@link ChunkInfo} object containing the download details + * @throws IOException + * if anything went wrong + */ + private ChunkInfo downloadFileChunk(final File localFile, final URL downloadUrl, final long offset, final long bytes) throws IOException + { + final long startPos = offset; + final long endPos = offset + bytes - 1; + + // request data with a partial GET + final HttpURLConnection conn = (HttpURLConnection) urlConnectionFactory.open(downloadUrl); + final String rangeHeaderValue = PartialGetUtils.formatRangeHeader(startPos, endPos); + conn.setRequestProperty(HttpRequestHeaders.RANGE, rangeHeaderValue); + + // check what type of response we got + final int statusCode = conn.getResponseCode(); + if (statusCode == HttpURLConnection.HTTP_OK) { - final URL downloadUrl = new URL(url + remoteFileName); + /* + * Response with the full content. + */ - log.debug("Downloading file from '" + downloadUrl + "' to '" + localFile + "' ..."); + log.debug("Downloading complete file from '{}' ...", downloadUrl); - // make sure the target directory exists - FileUtils.forceMkdir(localFile.getParentFile()); + final long bytesCopied = copyBytes(conn, localFile, false); - // download file - fout = new FileOutputStream(localFile); - final URLConnection conn = urlConnectionFactory.open(downloadUrl); - cin = conn.getInputStream(); - IOUtils.copy(cin, fout); + return new ChunkInfo(bytesCopied, bytesCopied); } - finally + else if (statusCode == HttpURLConnection.HTTP_PARTIAL) + { + /* + * Response with only a part of the content. + */ + + log.debug("Downloading chunk {}-{} from '{}' ...", startPos, endPos, downloadUrl); + + // validate Content-Range response header value and extract the total size of the file + final String contentRangeHeaderValue = conn.getHeaderField(HttpResponseHeaders.CONTENT_RANGE); + final ContentRangeHeaderData contentRangeHeaderData = PartialGetUtils.parseContentRangeHeader(contentRangeHeaderValue); + if (contentRangeHeaderData == null) + { + throw new XltException("Received invalid Content-Range header: " + contentRangeHeaderValue); + } + + // truncate the file to undo any previous attempt to append the current chunk + truncateFile(localFile, offset); + + final long bytesCopied = copyBytes(conn, localFile, true); + + return new ChunkInfo(contentRangeHeaderData.totalBytes, bytesCopied); + } + else { - IOUtils.closeQuietly(fout); - IOUtils.closeQuietly(cin); + /* + * Unexpected response. + */ + + throw new XltException("Received unexpected status code: " + statusCode); + } + } + + /** + * Truncates the given file to the given size. + * + * @param file + * the file to truncate + * @param newSize + * the size to truncate the file to + * @throws IOException + * if anything went wrong + */ + private void truncateFile(final File file, final long newSize) throws IOException + { + if (file.length() > newSize) + { + try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) + { + raf.setLength(newSize); + } + } + } + + /** + * Copies all available data from the URL connection to the file, either appending the data to the file or + * overwriting it. + * + * @param conn + * the URL connection to read from + * @param file + * the file to write to + * @param append + * whether to append the data to the file or overwrite it + * @return the number of bytes copied + * @throws IOException + * if anything went wrong + */ + private long copyBytes(final HttpURLConnection conn, final File file, final boolean append) throws IOException + { + try (final InputStream cin = conn.getInputStream(); final FileOutputStream fout = new FileOutputStream(file, append)) + { + // copy what is available + final long bytesCopied = IOUtils.copyLarge(cin, fout); + + // check whether we copied the expected number of bytes + final long bytesAnnounced = conn.getContentLengthLong(); // -1 if not set + if (bytesAnnounced != -1 && bytesAnnounced != bytesCopied) + { + throw new IOException(String.format("Expected %d bytes to copy but got %d bytes", bytesAnnounced, bytesCopied)); + } + + return bytesCopied; } } diff --git a/src/main/java/com/xceptance/xlt/agentcontroller/FileManagerServlet.java b/src/main/java/com/xceptance/xlt/agentcontroller/FileManagerServlet.java index 06fb3b3de..a692deb75 100644 --- a/src/main/java/com/xceptance/xlt/agentcontroller/FileManagerServlet.java +++ b/src/main/java/com/xceptance/xlt/agentcontroller/FileManagerServlet.java @@ -31,9 +31,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.xceptance.xlt.agentcontroller.PartialGetUtils.RangeHeaderData; +import com.xceptance.xlt.engine.httprequest.HttpRequestHeaders; +import com.xceptance.xlt.engine.httprequest.HttpResponseHeaders; + /** * The FileManagerServlet handles all file requests made from the master controller. - * + * * @author Jörg Werner (Xceptance Software Technologies GmbH) */ public class FileManagerServlet extends HttpServlet @@ -65,7 +69,7 @@ public class FileManagerServlet extends HttpServlet /** * Creates a new FileManagerServlet object. - * + * * @param rootDirectory * the local directory that is the web root */ @@ -76,7 +80,7 @@ public FileManagerServlet(final File rootDirectory) /** * Handles all download requests. - * + * * @param req * the servlet request * @param resp @@ -92,21 +96,94 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res { log.debug("File being downloaded: " + fileName); + // paranoia check if (fileName == null) { resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); + return; } - else + + final File file = new File(rootDirectory, fileName); + + // check if the file does not exist + if (!file.isFile()) { - final File file = new File(rootDirectory, fileName); - in = new FileInputStream(file); + // handle file does not exist + resp.setStatus(HttpServletResponse.SC_NOT_FOUND); + return; + } + + // check if the file is empty + final long fileLength = file.length(); + if (fileLength == 0) + { + // handle empty file + resp.setStatus(HttpServletResponse.SC_OK); + resp.setContentLengthLong(0); + return; + } + + // open the file for reading + in = new FileInputStream(file); + + // check for a partial GET request + final String rangeHeaderValue = req.getHeader(HttpRequestHeaders.RANGE); + if (rangeHeaderValue == null) + { + /* + * No partial request -> serve the full file content in one go. + */ + + log.debug("Serving full content from file '{}' ...", file); resp.setStatus(HttpServletResponse.SC_OK); - resp.setContentLengthLong(file.length()); - + resp.setContentLengthLong(fileLength); + final OutputStream out = resp.getOutputStream(); - IOUtils.copy(in, out); + IOUtils.copyLarge(in, out); + } + else + { + /* + * Partial request -> serve only the requested part of the file. + */ + + // validate the Range request header + final RangeHeaderData rangeHeaderData = PartialGetUtils.parseRangeHeader(rangeHeaderValue); + if (rangeHeaderData == null) + { + resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); + return; + } + + // extract and validate the start/end position passed in the Range header + final long startPos = rangeHeaderData.startPos; + final long endPos = rangeHeaderData.endPos; + + if (startPos > endPos || startPos > fileLength - 1) + { + resp.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); + return; + } + + // determine how many bytes can be served at all + final long finalEndPos = Math.min(endPos, fileLength - 1); + final long bytes = finalEndPos - startPos + 1; + + // prepare the Content-Range response header + final String contentRangeHeaderValue = PartialGetUtils.formatContentRangeHeader(startPos, finalEndPos, fileLength); + + // serve the requested byte range + log.debug("Serving chunk {}-{} from file '{}' ...", startPos, endPos, file); + + resp.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + resp.setContentLengthLong(bytes); + resp.setHeader(HttpResponseHeaders.CONTENT_RANGE, contentRangeHeaderValue); + + final OutputStream out = resp.getOutputStream(); + + IOUtils.copyLarge(in, out, startPos, bytes); } } catch (final Exception ex) @@ -122,7 +199,7 @@ protected void doGet(final HttpServletRequest req, final HttpServletResponse res /** * Handles all upload requests. - * + * * @param req * the servlet request * @param resp @@ -167,7 +244,7 @@ protected void doPut(final HttpServletRequest req, final HttpServletResponse res /** * Returns the file name from the URL parameters. - * + * * @param req * the servlet request * @return the file name, or null if not found diff --git a/src/main/java/com/xceptance/xlt/agentcontroller/PartialGetUtils.java b/src/main/java/com/xceptance/xlt/agentcontroller/PartialGetUtils.java new file mode 100644 index 000000000..75f388e73 --- /dev/null +++ b/src/main/java/com/xceptance/xlt/agentcontroller/PartialGetUtils.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2005-2023 Xceptance Software Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.xceptance.xlt.agentcontroller; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.xceptance.common.lang.ParseNumbers; + +/** + * Helper methods to parse HTTP headers involved when performing partial GET requests. + */ +class PartialGetUtils +{ + /** + * A pattern to validate a Range request header (for example, bytes=1000-1999) and extract values from + * it. + */ + private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=(\\d+)-(\\d+)"); + + /** + * A pattern to validate a Content-Range response header (for example, bytes 1000-1999/12345) and + * extract values from it. + */ + private static final Pattern CONTENT_RANGE_PATTERN = Pattern.compile("bytes (\\d+)-(\\d+)/(\\d+)"); + + /** + * The values passed in a Range request header value. + */ + static class RangeHeaderData + { + /** The start position of the requested part. **/ + public final long startPos; + + /** The end position (inclusive) of the requested part. **/ + public final long endPos; + + public RangeHeaderData(final long startPos, final long endPos) + { + this.startPos = startPos; + this.endPos = endPos; + } + } + + /** + * The values passed in a Content-Range response header value. + */ + static class ContentRangeHeaderData + { + /** The start position of the returned part. */ + public final long startPos; + + /** The end position (inclusive) of the returned part. */ + public final long endPos; + + /** The total size of the resource. */ + public final long totalBytes; + + public ContentRangeHeaderData(final long startPos, final long endPos, final long totalBytes) + { + this.startPos = startPos; + this.endPos = endPos; + this.totalBytes = totalBytes; + } + } + + /** + * Formats the given values as a valid Range request header value. + * + * @param startPos + * the start position of the requested part + * @param endPos + * the end position (inclusive) of the requested part + * @return the formatted header value + */ + static String formatRangeHeader(final long startPos, final long endPos) + { + return "bytes=" + startPos + "-" + endPos; + } + + /** + * Parses the given header value as a Range header and returns the extracted data. + * + * @param rangeHeaderValue + * the header value to parse + * @return the extracted data if the header could be parsed successfully, null otherwise + */ + static RangeHeaderData parseRangeHeader(final String rangeHeaderValue) + { + if (rangeHeaderValue != null) + { + final Matcher matcher = RANGE_PATTERN.matcher(rangeHeaderValue); + if (matcher.matches()) + { + final long startPos = ParseNumbers.parseLong(matcher.group(1)); + final long endPos = ParseNumbers.parseLong(matcher.group(2)); + + return new RangeHeaderData(startPos, endPos); + } + } + + return null; + } + + /** + * Formats the given values as a valid Content-Range response header value. + * + * @param startPos + * the start position of the returned part + * @param endPos + * the end position (inclusive) of the returned part + * @param totalBytes + * the total size of the resource + * @return the formatted header value + */ + static String formatContentRangeHeader(final long startPos, final long endPos, final long totalBytes) + { + return "bytes " + startPos + "-" + endPos + "/" + totalBytes; + } + + /** + * Parses the given header value as a Content-Range header and returns the extracted data. + * + * @param contentRangeHeaderValue + * the header value to parse + * @return the extracted data if the header could be parsed successfully, null otherwise + */ + static ContentRangeHeaderData parseContentRangeHeader(final String contentRangeHeaderValue) + { + if (contentRangeHeaderValue != null) + { + final Matcher matcher = CONTENT_RANGE_PATTERN.matcher(contentRangeHeaderValue); + if (matcher.matches()) + { + final long startPos = ParseNumbers.parseLong(matcher.group(1)); + final long endPos = ParseNumbers.parseLong(matcher.group(2)); + final long totalBytes = ParseNumbers.parseLong(matcher.group(3)); + + return new ContentRangeHeaderData(startPos, endPos, totalBytes); + } + } + + return null; + } +} diff --git a/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerConfiguration.java b/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerConfiguration.java index 63405fae2..a1213490a 100644 --- a/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerConfiguration.java +++ b/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerConfiguration.java @@ -28,13 +28,14 @@ import org.apache.commons.io.FileUtils; import com.xceptance.common.util.AbstractConfiguration; +import com.xceptance.xlt.agentcontroller.AgentControllerProxy; import com.xceptance.xlt.common.XltConstants; import com.xceptance.xlt.engine.XltExecutionContext; /** * The MasterControllerConfiguration is the central place where all configuration information of the master controller * can be retrieved from. - * + * * @author Jörg Werner (Xceptance Software Technologies GmbH) */ public class MasterControllerConfiguration extends AbstractConfiguration @@ -102,7 +103,11 @@ public class MasterControllerConfiguration extends AbstractConfiguration private static final String PROP_PASSWORD = PROP_PREFIX + "password"; private static final String PROP_COMPRESSED_TIMER_FILES = PROP_PREFIX + "compressedTimerFiles"; - + + private static final String PROP_DOWNLOAD_CHUNK_SIZE = PROP_PREFIX + "download.chunkSize"; + + private static final String PROP_DOWNLOAD_MAX_RETRIES = PROP_PREFIX + "download.maxRetries"; + private final List agentControllerConnectionInfos; private File agentFilesDirectory; @@ -156,10 +161,14 @@ public class MasterControllerConfiguration extends AbstractConfiguration private final boolean isEmbedded; private final boolean compressedTimerFiles; - + + private final long downloadChunkSize; + + private final int downloadMaxRetries; + /** * Creates a new MasterControllerConfiguration object. - * + * * @param commandLineProperties * the properties specified on the command line * @param isEmbeddedMode @@ -168,7 +177,8 @@ public class MasterControllerConfiguration extends AbstractConfiguration * if an I/O error occurs */ public MasterControllerConfiguration(final File overridePropertyFile, final Properties commandLineProperties, - final boolean isEmbeddedMode) throws IOException + final boolean isEmbeddedMode) + throws IOException { isEmbedded = isEmbeddedMode; homeDirectory = XltExecutionContext.getCurrent().getXltHomeDir(); @@ -272,14 +282,18 @@ else if (!(tempDirectory.isDirectory() && tempDirectory.canWrite())) // user name/password userName = XltConstants.USER_NAME; password = getStringProperty(PROP_PASSWORD, null); - - // do we want to keep the timer files compressed for efficency + + // do we want to keep the timer files compressed for efficiency compressedTimerFiles = getBooleanProperty(PROP_COMPRESSED_TIMER_FILES, true); + + // download options + downloadChunkSize = Math.max(1000, getLongProperty(PROP_DOWNLOAD_CHUNK_SIZE, AgentControllerProxy.DEFAULT_DOWNLOAD_CHUNK_SIZE)); + downloadMaxRetries = Math.max(0, getIntProperty(PROP_DOWNLOAD_MAX_RETRIES, AgentControllerProxy.DEFAULT_DOWNLOAD_MAX_RETRIES)); } /** * Returns the list of all configured agent controllers. - * + * * @return the agent controllers */ public List getAgentControllerConnectionInfos() @@ -289,7 +303,7 @@ public List getAgentControllerConnectionInfos() /** * Returns the directory where the agent files are located. - * + * * @return the agent files directory */ public File getAgentFilesDirectory() @@ -299,7 +313,7 @@ public File getAgentFilesDirectory() /** * Returns the directory where the master controller's configuration is located. - * + * * @return the config directory */ public File getConfigDirectory() @@ -309,7 +323,7 @@ public File getConfigDirectory() /** * Returns the master controller's home directory. - * + * * @return the home directory */ public File getHomeDirectory() @@ -319,7 +333,7 @@ public File getHomeDirectory() /** * Returns the master controller's temp directory. - * + * * @return the temp directory */ public File getTempDirectory() @@ -329,7 +343,7 @@ public File getTempDirectory() /** * Returns the root directory of all test reports. - * + * * @return the test reports directory */ public File getTestReportsRootDirectory() @@ -339,7 +353,7 @@ public File getTestReportsRootDirectory() /** * Returns the root directory of all test result files. - * + * * @return the test results directory */ public File getTestResultsRootDirectory() @@ -349,12 +363,12 @@ public File getTestResultsRootDirectory() /** * Reads and returns the list of all configured agent controllers. - * + * * @return the list of agent controllers */ private List readAgentControllerConnectionInfos() { - final List infos = new ArrayList(); + final List infos = new ArrayList<>(); defaultAgentCount = getIntProperty(PROP_AGENT_CONTROLLER_DEFAULT_AGENTS, defaultAgentCount); defaultWeight = getIntProperty(PROP_AGENT_CONTROLLER_DEFAULT_WEIGHT, defaultWeight); @@ -362,7 +376,7 @@ private List readAgentControllerConnectionInfos() final boolean defaultCP = getBooleanProperty(PROP_AGENT_CONTROLLER_DEFAULT_CP, false); final Set agentControllerNames = getPropertyKeyFragment(PROP_AGENT_CONTROLLERS_PREFIX); - final HashMap urlToNameMap = new HashMap(agentControllerNames.size()); + final HashMap urlToNameMap = new HashMap<>(agentControllerNames.size()); for (final String name : agentControllerNames) { // skip "default" agent controller settings @@ -423,7 +437,7 @@ private List readAgentControllerConnectionInfos() /** * Returns whether to display detailed status information for each simulated test user, or whether status * information will be aggregated into one line per user type. - * + * * @return whether to show detailed information */ public boolean getShowDetailedStatusList() @@ -433,7 +447,7 @@ public boolean getShowDetailedStatusList() /** * Returns the number of seconds to wait before the status list is updated again. - * + * * @return the update interval */ public int getStatusListUpdateInterval() @@ -444,7 +458,7 @@ public int getStatusListUpdateInterval() /** * In case of initial connection problems with a agent controller the load of the test is distributed to the * remaining agent controllers if the connection is relaxed. - * + * * @return true if the agent controller connection is relaxed; false otherwise */ public boolean isAgentControllerConnectionRelaxed() @@ -454,7 +468,7 @@ public boolean isAgentControllerConnectionRelaxed() /** * Tells to use a proxy or not. - * + * * @return true if using a proxy is enabled explicitly; false otherwise */ public boolean isHttpsProxyEnabled() @@ -464,7 +478,7 @@ public boolean isHttpsProxyEnabled() /** * Returns the https proxy host. - * + * * @return https proxy host */ public String getHttpsProxyHost() @@ -474,7 +488,7 @@ public String getHttpsProxyHost() /** * Returns the https proxy port. - * + * * @return https proxy port */ public String getHttpsProxyPort() @@ -484,7 +498,7 @@ public String getHttpsProxyPort() /** * Returns the hosts to bypass the proxy connection. - * + * * @return hosts to bypass the proxy connection */ public String getHttpsProxyBypassHosts() @@ -494,7 +508,7 @@ public String getHttpsProxyBypassHosts() /** * Returns the default agent count. - * + * * @return default agent count */ public int getDefaultAgentCount() @@ -504,7 +518,7 @@ public int getDefaultAgentCount() /** * Returns the default agent controller weight. - * + * * @return default agent controller weight */ public int getDefaultWeight() @@ -514,7 +528,7 @@ public int getDefaultWeight() /** * Returns the number of maximum parallel agent controller communication limit - * + * * @return the number of maximum parallel agent controller communication limit */ public int getParallelCommunicationLimit() @@ -524,7 +538,7 @@ public int getParallelCommunicationLimit() /** * Returns the number of maximum parallel uploads - * + * * @return the number of maximum parallel uploads */ public int getParallelUploadLimit() @@ -534,7 +548,7 @@ public int getParallelUploadLimit() /** * Returns the number of maximum parallel downloads - * + * * @return the number of maximum parallel downloads */ public int getParallelDownloadLimit() @@ -544,7 +558,7 @@ public int getParallelDownloadLimit() /** * Returns the configured agent-controller connection timeout. - * + * * @return agent-controller connection timeout */ public int getAgentControllerConnectTimeout() @@ -554,7 +568,7 @@ public int getAgentControllerConnectTimeout() /** * Returns the configured agent-controller read timeout. - * + * * @return agent-controller read timeout */ public int getAgentControllerReadTimeout() @@ -564,7 +578,7 @@ public int getAgentControllerReadTimeout() /** * Returns the configured agent controller initial response timeout. - * + * * @return agent controller initial response timeout */ public int getAgentControllerInitialResponseTimeout() @@ -574,7 +588,7 @@ public int getAgentControllerInitialResponseTimeout() /** * Returns the configured user name. - * + * * @return the user name */ public String getUserName() @@ -584,7 +598,7 @@ public String getUserName() /** * Returns the configured password. - * + * * @return the password */ public String getPassword() @@ -594,7 +608,7 @@ public String getPassword() /** * Returns the result output directory override as specified on command line. - * + * * @return result output directory override */ public File getResultOutputDirectory() @@ -605,7 +619,7 @@ public File getResultOutputDirectory() /** * Sets the result output directory override. If the given directory name denotes a relative file then it will be * rooted at the test results root directory. - * + * * @param outputDirectory * the result output directory name to use as override */ @@ -638,14 +652,34 @@ public boolean isEmbedded() { return isEmbedded; } - + /** * How shall we handle timer files after the download - * + * * @return true, keep them compressed, false, classic expanded storage */ public boolean isCompressedTimerFiles() { return compressedTimerFiles; } + + /** + * Returns the size of a file chunk when downloading a result archive from an agent controller. + * + * @return the chunk size (in bytes) + */ + public long getDownloadChunkSize() + { + return downloadChunkSize; + } + + /** + * Returns the maximum number of retries in case downloading a result file (chunk) failed because of an I/O error. + * + * @return the maximum number of retries + */ + public int getDownloadMaxRetries() + { + return downloadMaxRetries; + } } diff --git a/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerMain.java b/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerMain.java index 6c6ee50e7..0b44cf0df 100644 --- a/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerMain.java +++ b/src/main/java/com/xceptance/xlt/mastercontroller/MasterControllerMain.java @@ -274,7 +274,9 @@ private void startAgentControllerRemote(final Map agent for (final AgentControllerConnectionInfo info : connectionInfos) { final int agentCount = info.getNumberOfAgents(); - final AgentControllerProxy agentController = new AgentControllerProxy(commandLineProps, proxyFactory, urlConnectionFactory); + final AgentControllerProxy agentController = new AgentControllerProxy(commandLineProps, proxyFactory, urlConnectionFactory, + config.getDownloadChunkSize(), + config.getDownloadMaxRetries()); try { diff --git a/src/test/java/com/xceptance/xlt/agentcontroller/PartialGetUtils_ContentRangeHeaderTest.java b/src/test/java/com/xceptance/xlt/agentcontroller/PartialGetUtils_ContentRangeHeaderTest.java new file mode 100644 index 000000000..75412d2b2 --- /dev/null +++ b/src/test/java/com/xceptance/xlt/agentcontroller/PartialGetUtils_ContentRangeHeaderTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2005-2023 Xceptance Software Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.xceptance.xlt.agentcontroller; + +import java.util.Arrays; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import com.xceptance.xlt.agentcontroller.PartialGetUtils.ContentRangeHeaderData; + +@RunWith(Parameterized.class) +public class PartialGetUtils_ContentRangeHeaderTest +{ + @Parameters(name = "{index}: {0}") + public static Iterable data() + { + return Arrays.asList(new Object[][] + { + { + null, null + }, + { + "", null + }, + { + " ", null + }, + { + "bytes", null + }, + { + "bytes ", null + }, + { + "bytes 0", null + }, + { + "bytes 0-0", null + }, + { + "bytes 0-0/1", new ContentRangeHeaderData(0, 0, 1) + }, + { + "bytes 0-999/12345", new ContentRangeHeaderData(0, 999, 12345) + }, + { + "bytes 1000-1999/12345", new ContentRangeHeaderData(1000, 1999, 12345) + } + }); + } + + @Parameter(value = 0) + public String headerValue; + + @Parameter(value = 1) + public ContentRangeHeaderData expectedHeaderData; + + @Test + public void parseContentRangeHeader() + { + final ContentRangeHeaderData actualHeaderData = PartialGetUtils.parseContentRangeHeader(headerValue); + + if (expectedHeaderData == null) + { + Assert.assertNull(actualHeaderData); + } + else + { + Assert.assertEquals(expectedHeaderData.startPos, actualHeaderData.startPos); + Assert.assertEquals(expectedHeaderData.endPos, actualHeaderData.endPos); + Assert.assertEquals(expectedHeaderData.totalBytes, actualHeaderData.totalBytes); + } + } +} diff --git a/src/test/java/com/xceptance/xlt/agentcontroller/PartialGetUtils_RangeHeaderTest.java b/src/test/java/com/xceptance/xlt/agentcontroller/PartialGetUtils_RangeHeaderTest.java new file mode 100644 index 000000000..ad53524e0 --- /dev/null +++ b/src/test/java/com/xceptance/xlt/agentcontroller/PartialGetUtils_RangeHeaderTest.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2005-2023 Xceptance Software Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.xceptance.xlt.agentcontroller; + +import java.util.Arrays; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +import com.xceptance.xlt.agentcontroller.PartialGetUtils.RangeHeaderData; + +@RunWith(Parameterized.class) +public class PartialGetUtils_RangeHeaderTest +{ + @Parameters(name = "{index}: {0}") + public static Iterable data() + { + return Arrays.asList(new Object[][] + { + { + null, null + }, + { + "", null + }, + { + " ", null + }, + { + "bytes", null + }, + { + "bytes=", null + }, + { + "bytes=0", null + }, + { + "bytes=0-0", new RangeHeaderData(0, 0) + }, + { + "bytes=0-999", new RangeHeaderData(0, 999) + }, + { + "bytes=1000-1999", new RangeHeaderData(1000, 1999) + } + }); + } + + @Parameter(value = 0) + public String headerValue; + + @Parameter(value = 1) + public RangeHeaderData expectedHeaderData; + + @Test + public void parseRangeHeader() + { + final RangeHeaderData actualHeaderData = PartialGetUtils.parseRangeHeader(headerValue); + + if (expectedHeaderData == null) + { + Assert.assertNull(actualHeaderData); + } + else + { + Assert.assertEquals(expectedHeaderData.startPos, actualHeaderData.startPos); + Assert.assertEquals(expectedHeaderData.endPos, actualHeaderData.endPos); + } + } +} From d4d7a0e4f641e56f396fa23c8a463536e8d7f07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Werner?= <4639399+jowerner@users.noreply.github.com> Date: Wed, 23 Aug 2023 13:41:16 +0200 Subject: [PATCH 5/7] - turned JSON tree view into a Web component (#404) - extracted inline JSON viewer stuff into another Web component - added a new tab "Request Body (JSON)" with an own JSON viewer instance - did the needed adjustments to make it work again --- resultbrowser/src/css/default.css | 38 ----- .../src/css/{jsonview.css => json-tree.css} | 36 +++-- resultbrowser/src/css/json-viewer.css | 39 +++++ resultbrowser/src/index.html | 28 ++-- .../src/js/{jsonview.js => json-tree.js} | 139 +++++++++++------- resultbrowser/src/js/json-viewer.js | 101 +++++++++++++ resultbrowser/src/js/resultbrowser.js | 44 +++--- 7 files changed, 276 insertions(+), 149 deletions(-) rename resultbrowser/src/css/{jsonview.css => json-tree.css} (79%) create mode 100644 resultbrowser/src/css/json-viewer.css rename resultbrowser/src/js/{jsonview.js => json-tree.js} (77%) create mode 100644 resultbrowser/src/js/json-viewer.js diff --git a/resultbrowser/src/css/default.css b/resultbrowser/src/css/default.css index 1084fe7e1..f13dcb257 100644 --- a/resultbrowser/src/css/default.css +++ b/resultbrowser/src/css/default.css @@ -553,44 +553,6 @@ table.key-value-table td.value .csep { visibility: hidden !important; } - - -/******************************************************************************************** - * - * JSON Viewer - * - ********************************************************************************************/ - -#jsonViewer {} - -#jsonViewerActions { - font-size: 13px; - background-color: #f8f8f8; - border-top: 1px solid gray; - padding: 10px; -} - -#jsonViewerActions button { - padding: 0px 4px; - height: 26px; -} - -#jsonViewerActions input[type="text"] { - padding: 0px 4px; - height: 26px; - box-sizing: border-box; -} - -#jsonViewerContent { - font-size: 14px; - background-color: #fff; - color: #808080; - height: 100%; - overflow: auto; - padding: 10px; -} - - /******************************************************************************************** * * Miscellaneous diff --git a/resultbrowser/src/css/jsonview.css b/resultbrowser/src/css/json-tree.css similarity index 79% rename from resultbrowser/src/css/jsonview.css rename to resultbrowser/src/css/json-tree.css index 67b46285a..56d2fb79c 100644 --- a/resultbrowser/src/css/jsonview.css +++ b/resultbrowser/src/css/json-tree.css @@ -19,13 +19,17 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -.json-line { +json-tree { + display: block; +} + +json-tree .json-line { margin: 4px 0; display: flex; justify-content: flex-start; } -.json-caret::after { +json-tree .json-caret::after { min-width: 16px; cursor: pointer; font-style: normal; @@ -36,67 +40,67 @@ transform: translateY(-10%) rotate(-45deg); } -.json-caret-expanded::after { +json-tree .json-caret-expanded::after { transform: translateY(-10%) rotate(0deg); } -.json-caret-empty::after { +json-tree .json-caret-empty::after { cursor: default; display: none; } -.json-key { +json-tree .json-key { color: #444; margin-right: 4px; margin-left: 4px; } -.json-size { +json-tree .json-size { color: #808080; margin-right: 4px; margin-left: 4px; } -.json-index { +json-tree .json-index { color: #444; margin-right: 4px; margin-left: 4px; } -.json-separator { +json-tree .json-separator { color: #808080; } -.json-value { +json-tree .json-value { margin-left: 8px; } -.json-number { +json-tree .json-number { color: #f9ae58; } -.json-boolean { +json-tree .json-boolean { color: #ec5f66; } -.json-string { +json-tree .json-string { color: #86b25c; } -.json-null { +json-tree .json-null { color: #808080; } -.json-match { +json-tree .json-match { background-color: #DDD; color: #0c46dd; } -.json-match-current { +json-tree .json-match-current { background-color: #444; color: #FFF; } -.json-hide { +json-tree .json-hide { display: none; } diff --git a/resultbrowser/src/css/json-viewer.css b/resultbrowser/src/css/json-viewer.css new file mode 100644 index 000000000..f6b908aa5 --- /dev/null +++ b/resultbrowser/src/css/json-viewer.css @@ -0,0 +1,39 @@ +json-viewer { + display: block; +} + +json-viewer .tree { + font-size: 14px; + background-color: #fff; + color: #808080; + height: 100%; + overflow: auto; + padding: 10px; +} + +json-viewer .actions { + font-size: 13px; + background-color: #f8f8f8; + border-top: 1px groove gray; + padding: 10px; +} + +json-viewer .actions button { + padding: 0px 4px; + height: 26px; +} + +json-viewer .actions input[type="text"] { + padding: 0px 4px; + height: 26px; + box-sizing: border-box; +} + +json-viewer .actions .separator { + display: inline-block; + height: 20px; + border-left: 1px groove gray; + margin: 0px 8px; + vertical-align: middle; +} + diff --git a/resultbrowser/src/index.html b/resultbrowser/src/index.html index c4df9099d..90e89bb5a 100644 --- a/resultbrowser/src/index.html +++ b/resultbrowser/src/index.html @@ -6,7 +6,8 @@ XLT Result Browser - + + @@ -15,7 +16,6 @@ - @@ -84,9 +84,10 @@

Result Browser ☰

  • Request/Response Information
    • -
  • Request Body (Raw)
    • -
  • Response Content
    • -
  • JSON
    • +
  • Request Body
    • +
  • Request Body (JSON)
    • +
  • Response Body
    • +
  • Response Body (JSON)

Request

@@ -151,6 +152,9 @@

Response Headers


                         

+                        

+                            
+                        

                         

                             

                                 
@@ -174,17 +178,7 @@ 
Response Headers

                             

                         

                         

-                            

-                                

-                                

-                                    
-                                     | Search:
-                                    
-                                      
-                                      
-                                     | 0 Match(es) | JSON Path: 
-                                

-                            

+                            
                         

                     

 
@@ -350,6 +344,8 @@ 
Stored Test Parameters and Result Data

             
         

     
+    
+    
 
 
 
diff --git a/resultbrowser/src/js/jsonview.js b/resultbrowser/src/js/json-tree.js
similarity index 77%
rename from resultbrowser/src/js/jsonview.js
rename to resultbrowser/src/js/json-tree.js
index d25a44b8d..efb7735b4 100644
--- a/resultbrowser/src/js/jsonview.js
+++ b/resultbrowser/src/js/json-tree.js
@@ -115,8 +115,7 @@
                 node = node.parent;
             } while (node != null);
 
-            // TODO
-            document.querySelector('#jsonPath').textContent = path;
+            return path;
         };
 
         this.render = function (targetElem) {
@@ -142,7 +141,7 @@
 
             // change appearance of this node accordingly
             this.setVisible(found || !searchState.filter);
-            found ? this.setExpanded(true) : this.collapse();
+            found && this.setExpanded(true);
 
             return found;
         };
@@ -284,12 +283,6 @@
         const lineElement = createElement('div', { className: 'json-line', children: childElements });
         lineElement.style = 'margin-left: ' + node.depth * 24 + 'px;';
 
-        // attach event handlers to the line element
-        const handleClick = node.toggle.bind(node);
-        const handleClick2 = node.getJsonPath.bind(node);
-        lineElement.addEventListener('click', handleClick);
-        lineElement.addEventListener('click', handleClick2);
-
         return lineElement;
     }
 
@@ -320,55 +313,74 @@
         return htmlElement;
     }
 
-    /** The tree. */
-    let tree = null;
+    /**
+     * Attaches an event handler to the HTML element that represents the given node. The same handler is attached to all child nodes recursively.
+     *
+     * @param {TreeNode} node - a tree node
+     * @param {(TreeNode) => void} handler - the handler to attach 
+     */
+    function attachEventHandlers(node, handler) {
+        node.elem.addEventListener('click', () => handler(node));
+        node.forEachChildNode((child) => attachEventHandlers(child, handler));
+    }
+
+    /**
+     * Renders JSON data in a tree-like view.
+     */
+    class JsonTree extends HTMLElement {
+
+        /** The tree. */
+        tree = null;
 
-    /**  */
-    let searchState = null;
+        /**  */
+        searchState = null;
+
+        /**
+         * Clears any rendered JSON.
+         */
+        clear() {
+            this.tree = null;
+            this.searchState = null;
+            this.innerHTML = "";
+        }
 
-    /* Export jsonView object */
-    window.jsonView = {
         /**
-           * Creates a tree from the JSON data and renders it into a DOM container.
-           *
-           * @param {string} jsonData - the JSON data in string form
-           * @param {string} targetElementSelector - the CSS selector specifying the target element
-           */
-        format: function (jsonData, targetElementSelector) {
-            const targetElement = document.querySelector(targetElementSelector) || document.body;
+         * Creates a tree from the JSON data and renders it into the JsonTree element.
+         *
+         * @param {string} jsonData - the JSON data in string form
+         */
+        load(jsonData) {
+            this.clear();
 
             try {
                 const parsedData = JSON.parse(jsonData);
 
-                tree = createTree(parsedData);
+                this.tree = createTree(parsedData);
+                this.tree.render(this);
 
-                tree.render(targetElement);
-                tree.collapseAll();
-                tree.expand();
+                attachEventHandlers(this.tree, (node) => node.toggle());
+
+                this.tree.collapseAll();
+                this.tree.expand();
             }
             catch (error) {
-                targetElement.textContent = error;
+                this.textContent = error;
             }
-        },
+        }
 
         /**
          * Collapses all JSON nodes recursively.
          */
-        collapseAll: function () {
-            tree && tree.collapseAll();
-        },
+        collapseAll() {
+            this.tree && this.tree.collapseAll();
+        }
 
         /**
          * Expands all JSON nodes recursively.
          */
-        expandAll: function () {
-            tree && tree.expandAll();
-        },
-
-        /**
-         * The CSS selector for the DOM element which holds the number of matches found.
-         */
-        matchesElementSelector: '#matches',
+        expandAll() {
+            this.tree && this.tree.expandAll();
+        }
 
         /**
          * Searches the JSON tree for entries matching the search phrase and marks any match.
@@ -376,45 +388,60 @@
          * @param {string} searchPhrase - the text to search for
          * @param {boolean} ignoreCase - wehther the search is case insensitive
          * @param {boolean} filter - whether non-matching lines should be filtered out
+         * @return {number} the number of matches found
          */
-        search: function (searchPhrase, ignoreCase, filter) {
-            if (tree) {
+        search(searchPhrase, ignoreCase, filter) {
+            if (this.tree) {
                 searchPhrase = searchPhrase || '';
-                searchState = { matches: [], currentMatch: undefined, ignoreCase: ignoreCase, filter: filter, };
+                this.searchState = { matches: [], currentMatch: undefined, ignoreCase: ignoreCase, filter: filter, };
 
-                tree.search(searchPhrase, searchState);
+                this.tree.search(searchPhrase, this.searchState);
                 this.highlightNextMatch(true);
 
-                // update number of matches found
-                document.querySelector(this.matchesElementSelector).textContent = searchState.matches.length;
+                return this.searchState.matches.length;
+            }
+            else {
+                return 0;
             }
-        },
+        }
 
         /**
          * Highlights the next occurrence of the search phrase in the JSON tree.
          *
          * @param {boolean} forward - whether to go forward or backward
          */
-        highlightNextMatch: function (forward) {
-            if (searchState && searchState.matches.length > 0) {
-                if (searchState.currentMatch === undefined) {
-                    searchState.currentMatch = 0;
+        highlightNextMatch(forward) {
+            if (this.searchState && this.searchState.matches.length > 0) {
+                if (this.searchState.currentMatch === undefined) {
+                    this.searchState.currentMatch = 0;
                 }
                 else {
-                    searchState.matches[searchState.currentMatch].classList.remove('json-match-current');
+                    this.searchState.matches[this.searchState.currentMatch].classList.remove('json-match-current');
 
                     if (forward) {
-                        searchState.currentMatch = (searchState.currentMatch == searchState.matches.length - 1) ? 0 : searchState.currentMatch + 1;
+                        this.searchState.currentMatch = (this.searchState.currentMatch == this.searchState.matches.length - 1) ? 0 : this.searchState.currentMatch + 1;
                     }
                     else {
-                        searchState.currentMatch = (searchState.currentMatch == 0) ? searchState.matches.length - 1 : searchState.currentMatch - 1;
+                        this.searchState.currentMatch = (this.searchState.currentMatch == 0) ? this.searchState.matches.length - 1 : this.searchState.currentMatch - 1;
                     }
                 }
 
-                searchState.matches[searchState.currentMatch].scrollIntoView(!forward);
-                searchState.matches[searchState.currentMatch].classList.add('json-match-current');
+                this.searchState.matches[this.searchState.currentMatch].scrollIntoView(!forward);
+                this.searchState.matches[this.searchState.currentMatch].classList.add('json-match-current');
             }
-        },
+        }
+
+        /**
+         * Adds an event handler that is called with the JSON path of a node whenever a node in the JSON tree is clicked.
+         *
+         * @param {(string) => void} handler - consumes the JSON path of the target node
+         */
+        onJsonNodeSelected(handler) {
+            attachEventHandlers(this.tree, (node) => handler(node.getJsonPath()));
+        }   
     }
 
+    // register the JsonTree web component
+    window.customElements.define('json-tree', JsonTree);
+
 })();
diff --git a/resultbrowser/src/js/json-viewer.js b/resultbrowser/src/js/json-viewer.js
new file mode 100644
index 000000000..784db881a
--- /dev/null
+++ b/resultbrowser/src/js/json-viewer.js
@@ -0,0 +1,101 @@
+(function () {
+    'use strict';
+
+    /** 
+     * The sub elements of the JsonViewer as a template. 
+     */
+    const template = document.createElement('template');
+
+    template.innerHTML = `
+        
+        

+            
+            
+            
+            
+            
+            
+            
+            0 Match(es) 
+            
+            JSON Path: 
+        
`;
+
+    /**
+     * A viewer for JSON data that renders the JSON as a tree and offers ways to search and filter the JSON data.
+     */
+    class JsonViewer extends HTMLElement {
+
+        /** The JsonTree element. */
+        jsonTree = null;
+
+        constructor() {
+            super();
+
+            // create the sub elements from the template
+            this.appendChild(template.content.cloneNode(true));
+
+            // remember the json-tree element
+            this.jsonTree = this.querySelector("json-tree");
+
+            // get some UI elements
+            const searchInput = this.querySelector(".search");
+            const ignoreCaseCheckBox = this.querySelector(".ignoreCase");
+            const filterCheckBox = this.querySelector(".filter");
+            const matchesSpan = this.querySelector(".matches");
+
+            // the search event handler
+            const search = () => {
+                const searchPhrase = searchInput.value;
+                const ignoreCase = !!ignoreCaseCheckBox.checked;
+                const filter = !!filterCheckBox.checked;
+
+                const matches = this.jsonTree.search(searchPhrase, ignoreCase, filter);
+
+                matchesSpan.textContent = matches;                  
+            }
+
+            // add event handlers
+            this.querySelector(".expandAll").addEventListener('click', () => this.jsonTree.expandAll());
+            this.querySelector(".collapseAll").addEventListener('click', () => this.jsonTree.collapseAll());
+            this.querySelector(".search").addEventListener('keyup', search);
+            this.querySelector(".ignoreCase").addEventListener('click', search);
+            this.querySelector(".filter").addEventListener('click', search);
+            this.querySelector(".previous").addEventListener('click', () => this.jsonTree.highlightNextMatch(false));
+            this.querySelector(".next").addEventListener('click', () => this.jsonTree.highlightNextMatch(true));
+        }
+
+        connectedCallback() {
+            // nothing to do here for now
+        }
+
+        /**
+         * Populates the viewer with the JSON data.
+         *
+         * @param {string} jsonData - the JSON data in string form
+         */
+        load(json) {
+            // pass data on to the JSON tree
+            this.jsonTree.load(json);
+
+            // add a handler to the JSON tree that is called whenever the currently selected node changes
+            const jsonPath = this.querySelector(".jsonPath");
+            this.jsonTree.onJsonNodeSelected((s) => jsonPath.textContent = s);
+        }
+
+        /**
+         * Clears any rendered JSON and resets the UI state.
+         */
+        clear() {
+            this.jsonTree.clear();
+            this.querySelector(".search").value = "";
+            this.querySelector(".matches").textContent = "0";
+            this.querySelector(".jsonPath").textContent = "";
+        }
+    }
+
+    // register the JsonViewer web component
+    window.customElements.define('json-viewer', JsonViewer);
+
+})()
+
diff --git a/resultbrowser/src/js/resultbrowser.js b/resultbrowser/src/js/resultbrowser.js
index 00688dea6..58cfa60b5 100644
--- a/resultbrowser/src/js/resultbrowser.js
+++ b/resultbrowser/src/js/resultbrowser.js
@@ -131,6 +131,9 @@
         postRequestParam = getElementById("postrequestparameters");
         requestBodySmall = getElementById("requestBodySmall");
 
+        requestBodyJsonViewer = getElementById("requestBodyJson");
+        responseBodyJsonViewer = getElementById("responseBodyJson");
+
         transactionContent = getElementById("transactionContent");
         valueLog = getElementById("valueLog");
 
@@ -250,23 +253,6 @@
 
         // transaction page
         transaction.addEventListener("click", showTransaction);
-
-        // JSON viewer
-        queryFirst("#jsonViewerActions .expandAll").addEventListener("click", function () { jsonView.expandAll(); });
-        queryFirst("#jsonViewerActions .collapseAll").addEventListener("click", function () { jsonView.collapseAll(); });
-        queryFirst("#jsonViewerActions .search").addEventListener("keyup", search);
-        queryFirst("#jsonViewerActions .ignoreCase").addEventListener("click", search);
-        queryFirst("#jsonViewerActions .filter").addEventListener("click", search);
-        queryFirst("#jsonViewerActions .previous").addEventListener("click", function () { jsonView.highlightNextMatch(false); });
-        queryFirst("#jsonViewerActions .next").addEventListener("click", function () { jsonView.highlightNextMatch(true); });
-    }
-
-    function search() {
-        const searchPhrase = queryFirst("#jsonViewerActions .search").value;
-        const ignoreCase = !!queryFirst("#jsonViewerActions .ignoreCase").checked;
-        const filter = !!queryFirst("#jsonViewerActions .filter").checked;
-
-        jsonView.search(searchPhrase, ignoreCase, filter);
     }
 
     function toggleContent(element) {
@@ -506,6 +492,11 @@
         }
     }
 
+    function isJsonContent(contentType) {
+        //  e.g. "application/json" or "application/<...>+json"
+        return /^application\/(.+\+)?json$/.test(contentType);
+    }
+
     function showRequest(element) {
         // get action parent element
         const action = filterElements(getParents(element), parent => parent.classList.contains("action"))[0];
@@ -518,9 +509,9 @@
 
             hide(getElementById("loadErrorContent"));
 
-            queryFirst("#jsonViewerActions .search").value = "";
-
-            setText(getElementById("jsonViewerContent"), "");
+            // clear JSON viewers
+            requestBodyJsonViewer.clear();
+            responseBodyJsonViewer.clear();
 
             // retrieve the request data
             const requestData = dataStore.fetchData(element);
@@ -570,9 +561,9 @@
                             requestText.classList.add(...(lang ? [`language-${lang}`, lang] : ['text']));
                             show(requestText);
 
-                            // feed the json viewer if the mime type indicates json-ish content (e.g. "application/json" or "application/<...>+json")
-                            if (/^application\/(.+\+)?json$/.test(requestData.mimeType)) {
-                                jsonView.format(data, '#jsonViewerContent');
+                            // feed the response body json viewer if the mime type indicates json-ish content
+                            if (isJsonContent(requestData.mimeType)) {
+                                responseBodyJsonViewer.load(data);
                             }
                         }).catch(() => {
                             hide(requestText);
@@ -629,6 +620,13 @@
             // update the request content tab
             setText(getElementById("requestbody"), requestData.requestBodyRaw || '');
 
+            // feed the request body json viewer if the content type indicates json-ish content
+            const requestContentTypeHeader = requestData.requestHeaders.find(e => e.name_.toLowerCase() === "content-type");
+            const requestContentType = requestContentTypeHeader ? requestContentTypeHeader.value_ : "";
+            if (isJsonContent(requestContentType)) {
+                requestBodyJsonViewer.load(requestData.requestBodyRaw);
+            }
+
             // update the response information tab
             setText(getElementById("protocol"), requestData.protocol);
             setText(getElementById("status"), parseStatusLine(requestData.status));

From 36c4fdbf9558fc7d2aac1263301d7a7ca04ca24b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Werner?=
 <4639399+jowerner@users.noreply.github.com>
Date: Wed, 23 Aug 2023 15:01:01 +0200
Subject: [PATCH 6/7] - Don't escape text nodes and attributes when cloning
 them. (#412)

- Escape name and value of attributes when outputting them. Text nodes are already escaped.
---
 .../com/xceptance/common/xml/AbstractDomPrinter.java   |  3 ++-
 src/main/java/com/xceptance/common/xml/DomUtils.java   |  5 ++---
 .../xceptance/xlt/engine/resultbrowser/DomUtils.java   | 10 +++-------
 3 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/main/java/com/xceptance/common/xml/AbstractDomPrinter.java b/src/main/java/com/xceptance/common/xml/AbstractDomPrinter.java
index 025a76ca5..d878bc1d9 100644
--- a/src/main/java/com/xceptance/common/xml/AbstractDomPrinter.java
+++ b/src/main/java/com/xceptance/common/xml/AbstractDomPrinter.java
@@ -154,7 +154,8 @@ protected void printAttributes(final Element element, final PrintWriter printWri
         {
             final Attr attribute = (Attr) attributes.item(i);
 
-            printWriter.print(" " + attribute.getName() + "=\"" + attribute.getValue() + "\"");
+            printWriter.print(" " + StringEscapeUtils.escapeXml10(attribute.getName()) + "=\"" +
+                              StringEscapeUtils.escapeXml10(attribute.getValue()) + "\"");
         }
     }
 
diff --git a/src/main/java/com/xceptance/common/xml/DomUtils.java b/src/main/java/com/xceptance/common/xml/DomUtils.java
index 1b832773c..a017ba04d 100644
--- a/src/main/java/com/xceptance/common/xml/DomUtils.java
+++ b/src/main/java/com/xceptance/common/xml/DomUtils.java
@@ -18,7 +18,6 @@
 import java.io.OutputStream;
 import java.io.Writer;
 
-import org.apache.commons.text.StringEscapeUtils;
 import org.w3c.dom.Attr;
 import org.w3c.dom.CDATASection;
 import org.w3c.dom.Comment;
@@ -163,7 +162,7 @@ private static Node cloneElementNode(final Element node, final Document document
         for (int i = 0; i < attributes.getLength(); i++)
         {
             final Attr attribute = (Attr) attributes.item(i);
-            clone.setAttribute(attribute.getName(), StringEscapeUtils.escapeXml10(attribute.getValue()));
+            clone.setAttribute(attribute.getName(), attribute.getValue());
         }
 
         // clone the children
@@ -201,7 +200,7 @@ private static Node cloneCDATA(final CDATASection node, final Document document)
      */
     private static Node cloneText(final Text node, final Document document)
     {
-        return document.createTextNode(StringEscapeUtils.escapeXml10(node.getData()));
+        return document.createTextNode(node.getData());
     }
 
     /**
diff --git a/src/main/java/com/xceptance/xlt/engine/resultbrowser/DomUtils.java b/src/main/java/com/xceptance/xlt/engine/resultbrowser/DomUtils.java
index 51f88c28f..617a2a9ba 100644
--- a/src/main/java/com/xceptance/xlt/engine/resultbrowser/DomUtils.java
+++ b/src/main/java/com/xceptance/xlt/engine/resultbrowser/DomUtils.java
@@ -38,7 +38,6 @@
 import org.w3c.dom.NodeList;
 import org.w3c.dom.Text;
 
-import com.google.common.html.HtmlEscapers;
 import com.xceptance.common.util.ParameterCheckUtils;
 
 /**
@@ -299,18 +298,15 @@ private static Node cloneElementNode(final Element node, final PageDOMClone page
             final Attr attribute = (Attr) attributes.item(i);
             try
             {
-                // XLT#1954: Attribute values of the clone have to be escaped correctly since the raw value of the
-                // original attribute is not available anymore and their node value is already unescaped.
-                final String value = HtmlEscapers.htmlEscaper().escape(attribute.getValue());
                 // GH#88: Use namespaceURI of attribute and fall back to namespaceURI of owner element node if not set.
-                clone.setAttributeNS(ObjectUtils.defaultIfNull(attribute.getNamespaceURI(), nodeNS), attribute.getName(), value);
+                clone.setAttributeNS(ObjectUtils.defaultIfNull(attribute.getNamespaceURI(), nodeNS), attribute.getName(),
+                                     attribute.getValue());
             }
             catch (final DOMException dex)
             {
                 if (LOG.isWarnEnabled())
                 {
-                    LOG.warn(String.format("Failed to set attribute <%s> to value <%s>", attribute.getName(),
-                                                               attribute.getValue()));
+                    LOG.warn(String.format("Failed to set attribute <%s> to value <%s>", attribute.getName(), attribute.getValue()));
                 }
             }
         }

From 6b2eac2005a63c19d81c7cb0b6243bcbfa9bd043 Mon Sep 17 00:00:00 2001
From: Joerg Werner <4639399+jowerner@users.noreply.github.com>
Date: Wed, 23 Aug 2023 15:16:17 +0200
Subject: [PATCH 7/7] prepare release 7.2.0

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 1e6021afe..789ae1e0d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
 
     com.xceptance
     xlt
-    7.1.1
+    7.2.0
     jar
 
     XLT