diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServer.java b/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServer.java index d0a26feb2..f07cd9995 100644 --- a/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServer.java +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServer.java @@ -1,8 +1,6 @@ package ru.vk.itmo.test.smirnovandrew; import one.nio.http.Header; -import one.nio.http.HttpClient; -import one.nio.http.HttpException; import one.nio.http.HttpServer; import one.nio.http.HttpSession; import one.nio.http.Param; @@ -10,35 +8,32 @@ import one.nio.http.Request; import one.nio.http.RequestMethod; import one.nio.http.Response; -import one.nio.pool.PoolException; import ru.vk.itmo.ServiceConfig; import ru.vk.itmo.test.reference.dao.ReferenceDao; import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.logging.Logger; -import java.util.stream.Collectors; public class MyServer extends HttpServer { - private static final String ROOT = "/v0/entity"; - private static final String X_SENDER_NODE = "X-SenderNode"; - private static final String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; - private static final long DURATION = 1000L; - private static final int OK_STATUS = 300; - private static final int NOT_FOUND_STATUS = 404; - private static final String HEADER_DELIMITER = ": "; private final MyServerDao dao; private final MyExecutor executor; private final Logger logger; - private final Map httpClients; + private final HttpClient httpClient; private final RendezvousClusterManager rendezvousClustersManager; private final ServiceConfig config; @@ -64,15 +59,13 @@ public MyServer( this.dao = new MyServerDao(dao); this.executor = new MyExecutor(corePoolSize, availableProcessors); this.logger = Logger.getLogger(MyServer.class.getName()); - this.httpClients = config.clusterUrls().stream() - .filter(url -> !Objects.equals(url, config.selfUrl())) - .collect(Collectors.toMap(s -> s, MyServerUtil::createClient, (c, c1) -> c)); + this.httpClient = HttpClient.newHttpClient(); } @Override public void handleRequest(Request request, HttpSession session) throws IOException { try { - long exp = System.currentTimeMillis() + DURATION; + long exp = System.currentTimeMillis() + MyServerUtil.DURATION; executor.execute(() -> { try { if (System.currentTimeMillis() > exp) { @@ -98,27 +91,31 @@ private static int quorum(int from) { return from / 2 + 1; } - private Response sendToAnotherNode( + private HttpRequest toHttpRequest(Request request, String nodeUrl, String params) { + return HttpRequest.newBuilder(URI.create(nodeUrl + MyServerUtil.ROOT + "?" + params)) + .method(request.getMethodName(), request.getBody() == null + ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofByteArray(request.getBody())) + .setHeader(MyServerUtil.X_SENDER_NODE, config.selfUrl()) + .build(); + } + + private CompletableFuture sendToAnotherNode( Request request, + int ack, + int from, + String id, String clusterUrl, Function operation ) { if (Objects.equals(clusterUrl, config.selfUrl())) { - return operation.apply(dao); + return CompletableFuture.completedFuture(operation.apply(dao)); } - var httpClient = httpClients.get(clusterUrl); + var httpRequest = toHttpRequest(request, clusterUrl, String.format("id=%s&from=%d&ack=%d", id, from, ack)); - try { - return httpClient.invoke(request); - } catch (InterruptedException e) { - logger.info(e.getMessage()); - Thread.currentThread().interrupt(); - return new Response(Response.INTERNAL_ERROR, Response.EMPTY); - } catch (HttpException | IOException | PoolException e1) { - logger.info(e1.getMessage()); - return new Response(Response.INTERNAL_ERROR, Response.EMPTY); - } + return httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofByteArray()) + .thenApplyAsync(MyServerUtil::processingResponse); } private Response handleLocalRequest( @@ -128,15 +125,19 @@ private Response handleLocalRequest( Integer ackParam, String senderNode, Function operation - ) { - Integer from = fromParam; - if (Objects.isNull(from)) { + ) throws ExecutionException, InterruptedException, TimeoutException { + final int from; + if (Objects.isNull(fromParam)) { from = config.clusterUrls().size(); + } else { + from = fromParam; } - Integer ack = ackParam; - if (Objects.isNull(ack)) { + final int ack; + if (Objects.isNull(ackParam)) { ack = quorum(from); + } else { + ack = ackParam; } String paramError = getParametersError(id, from, ack); @@ -154,26 +155,29 @@ private Response handleLocalRequest( return new Response(Response.BAD_REQUEST, Response.EMPTY); } - var sortedNodes = RendezvousClusterManager.getSortedNodes(from, config); + var sortedNodes = RendezvousClusterManager.getSortedNodes(id, from, config); if (sortedNodes.stream().map(config.clusterUrls()::get).noneMatch(config.selfUrl()::equals)) { - return sendToAnotherNode(request, clusterUrl, operation); + return sendToAnotherNode(request, ack, from, id, clusterUrl, operation) + .get(MyServerUtil.DURATION, TimeUnit.MILLISECONDS); } - request.addHeader(String.join(HEADER_DELIMITER, X_SENDER_NODE, config.selfUrl())); - var responses = new ArrayList(); - for (int nodeNumber : sortedNodes) { - var r = sendToAnotherNode(request, config.clusterUrls().get(nodeNumber), operation); - if (r.getStatus() < OK_STATUS - || (r.getStatus() == NOT_FOUND_STATUS && request.getMethod() == Request.METHOD_GET)) { - responses.add(r); - } - } - - if (responses.size() < ack) { - return new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY); - } - return MyServerUtil.getMaxTimestampResponse(responses); + var completableResults = sortedNodes.stream() + .map(nodeNumber -> sendToAnotherNode( + request, + ack, + from, + id, + config.clusterUrls().get(nodeNumber), + operation)) + .toList(); + + return MyServerUtil.getResults( + from, + ack, + completableResults, + logger + ); } private String getParametersError(String id, Integer from, Integer ack) { @@ -201,15 +205,15 @@ private String getParametersError(String id, Integer from, Integer ack) { return null; } - @Path(ROOT) + @Path(MyServerUtil.ROOT) @RequestMethod(Request.METHOD_GET) public Response get( @Param(value = "id", required = true) String id, @Param(value = "from") Integer from, @Param(value = "ack") Integer ack, - @Header(value = X_SENDER_NODE) String senderNode, + @Header(value = MyServerUtil.X_SENDER_NODE) String senderNode, Request request - ) { + ) throws ExecutionException, InterruptedException, TimeoutException { return handleLocalRequest( request, id, @@ -220,15 +224,15 @@ public Response get( ); } - @Path(ROOT) + @Path(MyServerUtil.ROOT) @RequestMethod(Request.METHOD_DELETE) public Response delete( @Param(value = "id", required = true) String id, @Param(value = "from") Integer from, @Param(value = "ack") Integer ack, - @Header(value = X_SENDER_NODE) String senderNode, + @Header(value = MyServerUtil.X_SENDER_NODE) String senderNode, Request request - ) { + ) throws ExecutionException, InterruptedException, TimeoutException { return handleLocalRequest( request, id, @@ -239,15 +243,15 @@ public Response delete( ); } - @Path(ROOT) + @Path(MyServerUtil.ROOT) @RequestMethod(Request.METHOD_PUT) public Response put( @Param(value = "id", required = true) String id, @Param(value = "from") Integer from, @Param(value = "ack") Integer ack, - @Header(value = X_SENDER_NODE) String senderNode, + @Header(value = MyServerUtil.X_SENDER_NODE) String senderNode, Request request - ) { + ) throws ExecutionException, InterruptedException, TimeoutException { request.addHeader("Content-Length: " + request.getBody().length); request.setBody(request.getBody()); @@ -275,6 +279,6 @@ public void handleDefault(Request request, HttpSession session) throws IOExcepti public synchronized void stop() { this.executor.shutdown(); super.stop(); - httpClients.values().forEach(HttpClient::close); + httpClient.close(); } } diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServerDao.java b/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServerDao.java index f63805c8a..88829c970 100644 --- a/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServerDao.java +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServerDao.java @@ -31,11 +31,11 @@ Response getEntryFromDao(String id) { ValWithTime valueWithTimestamp = byteArrayToObject(entry.value().toArray(ValueLayout.JAVA_BYTE)); if (valueWithTimestamp.value() == null) { Response response = new Response(Response.NOT_FOUND, Response.EMPTY); - response.addHeader(MyServerUtil.X_TIMESTAMP + valueWithTimestamp.timestamp()); + response.addHeader("X-Timestamp: " + valueWithTimestamp.timestamp()); return response; } Response response = new Response(Response.OK, valueWithTimestamp.value()); - response.addHeader(MyServerUtil.X_TIMESTAMP + valueWithTimestamp.timestamp()); + response.addHeader("X-Timestamp: " + valueWithTimestamp.timestamp()); return response; } catch (IOException | ClassNotFoundException e) { return new Response(Response.INTERNAL_ERROR, Response.EMPTY); diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServerUtil.java b/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServerUtil.java index 9c6397bbf..420dd62f7 100644 --- a/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServerUtil.java +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServerUtil.java @@ -1,21 +1,40 @@ package ru.vk.itmo.test.smirnovandrew; -import one.nio.http.HttpClient; import one.nio.http.HttpServerConfig; import one.nio.http.HttpSession; import one.nio.http.Response; -import one.nio.net.ConnectionString; import one.nio.server.AcceptorConfig; import ru.vk.itmo.ServiceConfig; import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.http.HttpResponse; +import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; import java.util.logging.Logger; public final class MyServerUtil { - private static final int CONNECTION_TIMEOUT = 1000; - - public static final String X_TIMESTAMP = "X-TimeStamp: "; + public static final String ROOT = "/v0/entity"; + public static final String X_SENDER_NODE = "X-SenderNode"; + public static final String X_TIMESTAMP = "X-TimeStamp"; + public static final Map HTTP_CODES = Map.of( + HttpURLConnection.HTTP_OK, Response.OK, + HttpURLConnection.HTTP_ACCEPTED, Response.ACCEPTED, + HttpURLConnection.HTTP_CREATED, Response.CREATED, + HttpURLConnection.HTTP_NOT_FOUND, Response.NOT_FOUND, + HttpURLConnection.HTTP_BAD_REQUEST, Response.BAD_REQUEST, + HttpURLConnection.HTTP_INTERNAL_ERROR, Response.INTERNAL_ERROR + ); + public static final String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; + public static final long DURATION = 1000L; private MyServerUtil() { } @@ -32,12 +51,6 @@ public static HttpServerConfig generateServerConfig(ServiceConfig config) { return serverConfig; } - public static HttpClient createClient(String url) { - var client = new HttpClient(new ConnectionString(url)); - client.setConnectTimeout(CONNECTION_TIMEOUT); - return client; - } - public static void sendEmpty(HttpSession session, Logger logger, String message) { try { session.sendResponse(new Response(message, Response.EMPTY)); @@ -46,24 +59,68 @@ public static void sendEmpty(HttpSession session, Logger logger, String message) } } - private static long headerTimestampToLong(Response r) { - String header = r.getHeader(X_TIMESTAMP); + public static long headerTimestampToLong(Response r) { + String header = r.getHeader("X-TimeStamp: "); if (header == null) { return Long.MIN_VALUE; } return Long.parseLong(header); } - public static Response getMaxTimestampResponse(List responses) { - long maxTimestamp = Long.MIN_VALUE; - Response maxResponse = null; - for (var response: responses) { - long timestamp = headerTimestampToLong(response); - if (timestamp >= maxTimestamp) { - maxResponse = response; - maxTimestamp = timestamp; + public static Response processingResponse(HttpResponse response) { + String statusCode = HTTP_CODES.getOrDefault(response.statusCode(), null); + if (statusCode == null) { + return new Response(Response.INTERNAL_ERROR, response.body()); + } else { + Response newResponse = new Response(statusCode, response.body()); + long timestamp = response.headers().firstValueAsLong(X_TIMESTAMP).orElse(0); + newResponse.addHeader(X_TIMESTAMP + ": " + timestamp); + return newResponse; + } + } + + public static Response getResults( + int from, + int ack, + List> completableResults, + Logger logger + ) throws ExecutionException, InterruptedException { + var okResponses = new ConcurrentLinkedDeque(); + var okResponsesCount = new AtomicInteger(); + var failedResponsesCount = new AtomicInteger(); + var answer = new CompletableFuture(); + + BiConsumer whenComplete = (r, throwable) -> { + if (throwable == null || r.getStatus() < 500) { + okResponsesCount.incrementAndGet(); + okResponses.add(r); + } else { + failedResponsesCount.incrementAndGet(); } + + if (okResponsesCount.get() >= ack) { + answer.complete(okResponses.stream() + .max(Comparator.comparingLong(MyServerUtil::headerTimestampToLong)) + .get()); + } + + if (failedResponsesCount.get() >= from - ack + 1) { + answer.complete(new Response(MyServerUtil.NOT_ENOUGH_REPLICAS, Response.EMPTY)); + } + }; + + completableResults.forEach(completableFuture -> { + var responseFuture = completableFuture.whenComplete(whenComplete); + if (responseFuture == null) { + logger.info("Error completable future is null!"); + } + }); + + try { + return answer.get(MyServerUtil.DURATION, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + logger.info("Too long waiting for response: " + e.getMessage()); + return new Response(MyServerUtil.NOT_ENOUGH_REPLICAS, Response.EMPTY); } - return maxResponse; } } diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServiceFactory.java b/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServiceFactory.java index 81564c3f4..e41656d9a 100644 --- a/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServiceFactory.java +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/MyServiceFactory.java @@ -4,7 +4,7 @@ import ru.vk.itmo.ServiceConfig; import ru.vk.itmo.test.ServiceFactory; -@ServiceFactory(stage = 4) +@ServiceFactory(stage = 5) public class MyServiceFactory implements ServiceFactory.Factory { @Override public Service create(ServiceConfig config) { diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/RendezvousClusterManager.java b/src/main/java/ru/vk/itmo/test/smirnovandrew/RendezvousClusterManager.java index 0f6f40f70..92799e11d 100644 --- a/src/main/java/ru/vk/itmo/test/smirnovandrew/RendezvousClusterManager.java +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/RendezvousClusterManager.java @@ -3,8 +3,10 @@ import one.nio.util.Hash; import ru.vk.itmo.ServiceConfig; -import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; public class RendezvousClusterManager { @@ -18,7 +20,7 @@ public String getCluster(String key) { int resIdx = -1; int maxHash = Integer.MIN_VALUE; for (int i = 0; i < availableClusters.size(); ++i) { - var hash = Hash.murmur3(key + availableClusters.get(i)); + var hash = Hash.murmur3(String.join("", availableClusters.get(i), key)); if (hash > maxHash) { resIdx = i; maxHash = hash; @@ -32,11 +34,12 @@ public String getCluster(String key) { return availableClusters.get(resIdx); } - public static List getSortedNodes(int amount, ServiceConfig config) { - var result = new ArrayList(); - for (int i = 0; i < config.clusterUrls().size() && result.size() < amount; i++) { - result.add(i); - } - return result; + public static List getSortedNodes(String key, int amount, ServiceConfig config) { + return IntStream.range(0, config.clusterUrls().size()) + .mapToObj(i -> Map.entry(i, Hash.murmur3(key + i))) + .sorted(Comparator.>comparingInt(Map.Entry::getValue).reversed()) + .map(Map.Entry::getKey) + .limit(amount) + .toList(); } } diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5.md b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5.md new file mode 100644 index 000000000..face2e83a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5.md @@ -0,0 +1,81 @@ +# Отчет о тестировании +## Использованные методы +Все методы реализованы так же как и методы в stage1, +как именно они реализованы можно посмотреть либо в отчете +stage1.md или в папке lua + +После прогрева выяснилось, что точка разладки происходит где-то +в районе 3000-3500, поэтому будем тестировать на 3000 rps + +### PUT запросы +``` +./wrk -d 120 -t 1 -c 64 -R 3000 -L -s /home/andrew/my-dht/2024-highload-dht/src/main/java/ru/vk/itmo/test/smirnovandrew/lua/put.lua http://localhost:8080 + ``` +```dtd +./profiler.sh --fdtransfer -d 120 -e cpu -f put_cpu_stage3.html jps +``` + +![put3000.png](stage5%2Fput3000.png) +Здесь интересно, что latency начинает расти где-то около 99.9% + +Анализ CPU: +![put_cpu.png](stage5%2Fput_cpu.png) +Наибольшая нагрузка приходится на ожидание свободных потоков + +Также на асинхронное отправление запроса на другой шард + + +Анализ ALLOC: +![put_alloc.png](stage5%2Fput_alloc.png) +Большее количество памяти так же, как и в предыдущих частях +тратится на конвертацию MemorySegment в byte[] и обратно + +Также тратится на расшифрофку headerов и на посылку Completable future + + +Анализ LOCK: +![put_lock.png](stage5%2Fput_lock.png) +Наибольная нагрузка тут приходится на ожидание таски из очереди + +Также тратится на ожидание асинхронного запроса + +### GET запросы +``` +./wrk -d 60 -t 1 -c 64 -R 3000 -L -s /home/andrew/my-dht/2024-highload-dht/src/main/java/ru/vk/itmo/test/smirnovandrew/lua/get.lua http://localhost:8080 + ``` +```dtd +./profiler.sh --fdtransfer -d 120 -e cpu -f get_cpu_stage3.html jps +``` + +![get3000.png](stage5%2Fget3000.png) +Здесь также latency серьезно растет около 99.9% + +Анализ CPU: +![get_cpu.png](stage5%2Fget_cpu.png) +Наибольшая нагрузка приходится на ожидание свободных потоков + +Также на асинхронное отправление запроса на другой шард + +Анализ ALLOC: +![get_alloc.png](stage5%2Fget_alloc.png) +Большее количество памяти так же, как и в предыдущих частях +тратится на конвертацию MemorySegment в byte[] и обратно + +Также тратится на расшифрофку headerов и на посылку Completable future + + + +Анализ LOCK: +![get_lock.png](stage5%2Fget_lock.png) +Наибольная нагрузка тут приходится на ожидание таски из очереди + +Также тратится на ожидание асинхронного запроса + + +### Сравнение с предыдущей реализацией +![compare.png](stage5%2Fcompare.png) +К сожалению я не делал графики на stage4, поэтому могу сравнить +показатели только со stage3. Здесь по графикам заметно, что относительно +stage3 есть значительные улучшения по latency + + diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/compare.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/compare.png new file mode 100644 index 000000000..f240f4a81 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/compare.png differ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000.png new file mode 100644 index 000000000..d73a18162 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000.png differ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000.txt b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000.txt new file mode 100644 index 000000000..18be6dd37 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000.txt @@ -0,0 +1,117 @@ +❯ ./wrk -d 60 -t 1 -c 64 -R 3000 -L -s /Users/sandrew2003/2024-highload-dht/src/main/java/ru/vk/itmo/test/smirnovandrew/lua/get.lua http://localhost:8080 + +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 1.691ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.67ms 805.87us 25.26ms 78.75% + Req/Sec 3.16k 306.18 7.20k 79.01% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.55ms + 75.000% 2.10ms + 90.000% 2.54ms + 99.000% 4.02ms + 99.900% 7.93ms + 99.990% 20.06ms + 99.999% 24.93ms +100.000% 25.28ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.273 0.000000 1 1.00 + 0.876 0.100000 14961 1.11 + 1.046 0.200000 29907 1.25 + 1.201 0.300000 44746 1.43 + 1.366 0.400000 59625 1.67 + 1.546 0.500000 74525 2.00 + 1.644 0.550000 82012 2.22 + 1.746 0.600000 89445 2.50 + 1.854 0.650000 96880 2.86 + 1.974 0.700000 104362 3.33 + 2.101 0.750000 111826 4.00 + 2.169 0.775000 115567 4.44 + 2.237 0.800000 119290 5.00 + 2.307 0.825000 123074 5.71 + 2.379 0.850000 126718 6.67 + 2.457 0.875000 130463 8.00 + 2.497 0.887500 132279 8.89 + 2.541 0.900000 134163 10.00 + 2.591 0.912500 136016 11.43 + 2.647 0.925000 137866 13.33 + 2.713 0.937500 139750 16.00 + 2.749 0.943750 140665 17.78 + 2.793 0.950000 141637 20.00 + 2.841 0.956250 142535 22.86 + 2.903 0.962500 143470 26.67 + 2.991 0.968750 144390 32.00 + 3.053 0.971875 144854 35.56 + 3.139 0.975000 145325 40.00 + 3.263 0.978125 145790 45.71 + 3.419 0.981250 146250 53.33 + 3.599 0.984375 146719 64.00 + 3.709 0.985938 146950 71.11 + 3.817 0.987500 147182 80.00 + 3.935 0.989062 147415 91.43 + 4.071 0.990625 147649 106.67 + 4.235 0.992188 147887 128.00 + 4.307 0.992969 148000 142.22 + 4.427 0.993750 148116 160.00 + 4.547 0.994531 148230 182.86 + 4.679 0.995313 148346 213.33 + 4.843 0.996094 148464 256.00 + 4.967 0.996484 148524 284.44 + 5.095 0.996875 148579 320.00 + 5.275 0.997266 148637 365.71 + 5.531 0.997656 148695 426.67 + 5.911 0.998047 148753 512.00 + 6.147 0.998242 148783 568.89 + 6.415 0.998437 148812 640.00 + 6.843 0.998633 148841 731.43 + 7.403 0.998828 148870 853.33 + 7.995 0.999023 148899 1024.00 + 8.271 0.999121 148915 1137.78 + 8.551 0.999219 148928 1280.00 + 9.015 0.999316 148943 1462.86 + 9.935 0.999414 148957 1706.67 + 12.255 0.999512 148972 2048.00 + 13.287 0.999561 148979 2275.56 + 13.935 0.999609 148986 2560.00 + 15.399 0.999658 148994 2925.71 + 16.159 0.999707 149001 3413.33 + 17.023 0.999756 149008 4096.00 + 17.311 0.999780 149012 4551.11 + 17.519 0.999805 149015 5120.00 + 18.191 0.999829 149019 5851.43 + 18.655 0.999854 149023 6826.67 + 19.055 0.999878 149026 8192.00 + 19.487 0.999890 149028 9102.22 + 20.095 0.999902 149030 10240.00 + 20.959 0.999915 149032 11702.86 + 21.743 0.999927 149034 13653.33 + 21.807 0.999939 149035 16384.00 + 21.839 0.999945 149036 18204.44 + 22.431 0.999951 149037 20480.00 + 22.559 0.999957 149038 23405.71 + 22.863 0.999963 149039 27306.67 + 23.407 0.999969 149040 32768.00 + 23.407 0.999973 149040 36408.89 + 23.647 0.999976 149041 40960.00 + 23.647 0.999979 149041 46811.43 + 24.175 0.999982 149042 54613.33 + 24.175 0.999985 149042 65536.00 + 24.175 0.999986 149042 72817.78 + 24.927 0.999988 149043 81920.00 + 24.927 0.999989 149043 93622.86 + 24.927 0.999991 149043 109226.67 + 24.927 0.999992 149043 131072.00 + 24.927 0.999993 149043 145635.56 + 25.279 0.999994 149044 163840.00 + 25.279 1.000000 149044 inf +#[Mean = 1.670, StdDeviation = 0.806] +#[Max = 25.264, Total count = 149044] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 179556 requests in 1.00m, 24.30MB read +Requests/sec: 2992.55 +Transfer/sec: 414.66KB diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000_stage3.txt b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000_stage3.txt new file mode 100644 index 000000000..2cab705e2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get3000_stage3.txt @@ -0,0 +1,115 @@ +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 1.893ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.81ms 1.45ms 44.51ms 87.58% + Req/Sec 3.17k 379.09 9.56k 76.06% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.59ms + 75.000% 2.36ms + 90.000% 3.17ms + 99.000% 5.27ms + 99.900% 17.63ms + 99.990% 38.53ms + 99.999% 43.33ms +100.000% 44.54ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.073 0.000000 1 1.00 + 0.591 0.100000 14907 1.11 + 0.851 0.200000 29845 1.25 + 1.081 0.300000 44735 1.43 + 1.321 0.400000 59643 1.67 + 1.587 0.500000 74547 2.00 + 1.726 0.550000 82004 2.22 + 1.869 0.600000 89446 2.50 + 2.021 0.650000 96885 2.86 + 2.179 0.700000 104352 3.33 + 2.357 0.750000 111827 4.00 + 2.453 0.775000 115573 4.44 + 2.559 0.800000 119272 5.00 + 2.677 0.825000 122973 5.71 + 2.809 0.850000 126687 6.67 + 2.967 0.875000 130432 8.00 + 3.063 0.887500 132302 8.89 + 3.167 0.900000 134137 10.00 + 3.283 0.912500 136030 11.43 + 3.407 0.925000 137883 13.33 + 3.557 0.937500 139750 16.00 + 3.637 0.943750 140659 17.78 + 3.735 0.950000 141605 20.00 + 3.845 0.956250 142532 22.86 + 3.971 0.962500 143458 26.67 + 4.143 0.968750 144397 32.00 + 4.239 0.971875 144848 35.56 + 4.343 0.975000 145327 40.00 + 4.455 0.978125 145793 45.71 + 4.591 0.981250 146255 53.33 + 4.755 0.984375 146716 64.00 + 4.875 0.985938 146954 71.11 + 4.995 0.987500 147186 80.00 + 5.155 0.989062 147410 91.43 + 5.355 0.990625 147645 106.67 + 5.643 0.992188 147875 128.00 + 5.891 0.992969 147994 142.22 + 6.179 0.993750 148108 160.00 + 6.543 0.994531 148224 182.86 + 7.091 0.995313 148341 213.33 + 7.759 0.996094 148458 256.00 + 8.367 0.996484 148516 284.44 + 9.231 0.996875 148575 320.00 + 10.839 0.997266 148632 365.71 + 12.231 0.997656 148691 426.67 + 13.727 0.998047 148748 512.00 + 14.335 0.998242 148778 568.89 + 15.015 0.998437 148808 640.00 + 15.671 0.998633 148836 731.43 + 16.447 0.998828 148865 853.33 + 17.759 0.999023 148894 1024.00 + 19.487 0.999121 148909 1137.78 + 20.575 0.999219 148923 1280.00 + 22.815 0.999316 148938 1462.86 + 24.319 0.999414 148952 1706.67 + 26.431 0.999512 148967 2048.00 + 27.391 0.999561 148974 2275.56 + 28.911 0.999609 148981 2560.00 + 30.607 0.999658 148989 2925.71 + 31.535 0.999707 148996 3413.33 + 32.799 0.999756 149003 4096.00 + 33.087 0.999780 149007 4551.11 + 34.399 0.999805 149010 5120.00 + 36.031 0.999829 149014 5851.43 + 36.511 0.999854 149018 6826.67 + 37.695 0.999878 149021 8192.00 + 38.463 0.999890 149023 9102.22 + 38.751 0.999902 149025 10240.00 + 39.519 0.999915 149027 11702.86 + 40.159 0.999927 149029 13653.33 + 40.671 0.999939 149030 16384.00 + 41.087 0.999945 149031 18204.44 + 41.183 0.999951 149032 20480.00 + 41.343 0.999957 149034 23405.71 + 41.343 0.999963 149034 27306.67 + 42.015 0.999969 149036 32768.00 + 42.015 0.999973 149036 36408.89 + 42.015 0.999976 149036 40960.00 + 42.015 0.999979 149036 46811.43 + 43.167 0.999982 149037 54613.33 + 43.167 0.999985 149037 65536.00 + 43.167 0.999986 149037 72817.78 + 43.327 0.999988 149038 81920.00 + 43.327 0.999989 149038 93622.86 + 43.327 0.999991 149038 109226.67 + 43.327 0.999992 149038 131072.00 + 43.327 0.999993 149038 145635.56 + 44.543 0.999994 149039 163840.00 + 44.543 1.000000 149039 inf +#[Mean = 1.809, StdDeviation = 1.453] +#[Max = 44.512, Total count = 149039] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 179550 requests in 1.00m, 19.16MB read +Requests/sec: 2992.53 +Transfer/sec: 326.97KB diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_alloc.html b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_alloc.html new file mode 100644 index 000000000..251971c21 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_alloc.html @@ -0,0 +1,2576 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_alloc.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_alloc.png new file mode 100644 index 000000000..2dd8d5ef4 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_alloc.png differ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_cpu.html b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_cpu.html new file mode 100644 index 000000000..5ed9c8365 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_cpu.html @@ -0,0 +1,2472 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_cpu.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_cpu.png new file mode 100644 index 000000000..e20a78d0c Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_cpu.png differ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_lock.html b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_lock.html new file mode 100644 index 000000000..b33e30cdc --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_lock.html @@ -0,0 +1,913 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_lock.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_lock.png new file mode 100644 index 000000000..1dbbfd81d Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/get_lock.png differ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put3000.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put3000.png new file mode 100644 index 000000000..efa52124e Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put3000.png differ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put3000.txt b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put3000.txt new file mode 100644 index 000000000..08f7ea989 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put3000.txt @@ -0,0 +1,117 @@ +❯ ./wrk -d 60 -t 1 -c 64 -R 3000 -L -s /Users/sandrew2003/2024-highload-dht/src/main/java/ru/vk/itmo/test/smirnovandrew/lua/put.lua http://localhost:8080 + +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 1.292ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.51ms 1.08ms 62.02ms 94.14% + Req/Sec 3.16k 308.45 11.89k 86.27% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.39ms + 75.000% 1.89ms + 90.000% 2.34ms + 99.000% 3.57ms + 99.900% 12.49ms + 99.990% 40.38ms + 99.999% 60.41ms +100.000% 62.05ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.186 0.000000 1 1.00 + 0.735 0.100000 14926 1.11 + 0.912 0.200000 29884 1.25 + 1.065 0.300000 44785 1.43 + 1.223 0.400000 59642 1.67 + 1.393 0.500000 74567 2.00 + 1.484 0.550000 82017 2.22 + 1.576 0.600000 89447 2.50 + 1.675 0.650000 96905 2.86 + 1.780 0.700000 104340 3.33 + 1.892 0.750000 111835 4.00 + 1.951 0.775000 115533 4.44 + 2.015 0.800000 119250 5.00 + 2.085 0.825000 123042 5.71 + 2.161 0.850000 126688 6.67 + 2.245 0.875000 130432 8.00 + 2.291 0.887500 132306 8.89 + 2.341 0.900000 134164 10.00 + 2.395 0.912500 136002 11.43 + 2.453 0.925000 137885 13.33 + 2.519 0.937500 139773 16.00 + 2.557 0.943750 140696 17.78 + 2.597 0.950000 141601 20.00 + 2.639 0.956250 142530 22.86 + 2.691 0.962500 143464 26.67 + 2.751 0.968750 144406 32.00 + 2.785 0.971875 144864 35.56 + 2.831 0.975000 145334 40.00 + 2.881 0.978125 145796 45.71 + 2.947 0.981250 146258 53.33 + 3.049 0.984375 146714 64.00 + 3.123 0.985938 146947 71.11 + 3.245 0.987500 147182 80.00 + 3.405 0.989062 147412 91.43 + 3.673 0.990625 147645 106.67 + 3.929 0.992188 147877 128.00 + 4.091 0.992969 147996 142.22 + 4.295 0.993750 148110 160.00 + 4.547 0.994531 148227 182.86 + 4.815 0.995313 148343 213.33 + 5.191 0.996094 148459 256.00 + 5.435 0.996484 148518 284.44 + 5.835 0.996875 148578 320.00 + 6.155 0.997266 148635 365.71 + 6.539 0.997656 148695 426.67 + 7.019 0.998047 148751 512.00 + 7.527 0.998242 148782 568.89 + 8.011 0.998437 148809 640.00 + 9.047 0.998633 148838 731.43 + 10.343 0.998828 148867 853.33 + 12.743 0.999023 148896 1024.00 + 14.167 0.999121 148911 1137.78 + 15.983 0.999219 148925 1280.00 + 17.343 0.999316 148940 1462.86 + 18.799 0.999414 148954 1706.67 + 20.415 0.999512 148969 2048.00 + 21.135 0.999561 148976 2275.56 + 22.047 0.999609 148983 2560.00 + 24.847 0.999658 148991 2925.71 + 27.263 0.999707 148998 3413.33 + 32.591 0.999756 149005 4096.00 + 33.087 0.999780 149009 4551.11 + 35.167 0.999805 149012 5120.00 + 37.631 0.999829 149016 5851.43 + 39.519 0.999854 149020 6826.67 + 39.967 0.999878 149023 8192.00 + 40.319 0.999890 149025 9102.22 + 41.663 0.999902 149027 10240.00 + 42.847 0.999915 149029 11702.86 + 45.279 0.999927 149031 13653.33 + 45.567 0.999939 149032 16384.00 + 49.759 0.999945 149033 18204.44 + 50.111 0.999951 149034 20480.00 + 53.119 0.999957 149035 23405.71 + 57.951 0.999963 149036 27306.67 + 58.975 0.999969 149037 32768.00 + 58.975 0.999973 149037 36408.89 + 59.071 0.999976 149038 40960.00 + 59.071 0.999979 149038 46811.43 + 59.967 0.999982 149039 54613.33 + 59.967 0.999985 149039 65536.00 + 59.967 0.999986 149039 72817.78 + 60.415 0.999988 149040 81920.00 + 60.415 0.999989 149040 93622.86 + 60.415 0.999991 149040 109226.67 + 60.415 0.999992 149040 131072.00 + 60.415 0.999993 149040 145635.56 + 62.047 0.999994 149041 163840.00 + 62.047 1.000000 149041 inf +#[Mean = 1.514, StdDeviation = 1.085] +#[Max = 62.016, Total count = 149041] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 179556 requests in 1.00m, 14.21MB read +Requests/sec: 2992.62 +Transfer/sec: 242.57KB diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_alloc.html b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_alloc.html new file mode 100644 index 000000000..ef92a5045 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_alloc.html @@ -0,0 +1,3310 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_alloc.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_alloc.png new file mode 100644 index 000000000..8880afbd3 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_alloc.png differ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_cpu.html b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_cpu.html new file mode 100644 index 000000000..97fdd646a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_cpu.html @@ -0,0 +1,3786 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_cpu.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_cpu.png new file mode 100644 index 000000000..941280d7f Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_cpu.png differ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_lock.html b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_lock.html new file mode 100644 index 000000000..5a244831f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_lock.html @@ -0,0 +1,870 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_lock.png b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_lock.png new file mode 100644 index 000000000..fb85f9a2a Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/smirnovandrew/readme/stage5/put_lock.png differ