diff --git a/.codeclimate.yml b/.codeclimate.yml index 6cf45b498..d193d57fa 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -79,3 +79,4 @@ exclude_patterns: - "src/test/" - "src/main/java/ru/vk/itmo/test/reference/" - "src/main/java/ru/vk/itmo/test/smirnovdmitrii/application/properties" + - "src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ProcessResult.java" diff --git a/README.md b/README.md index 1f026ec7b..5935b793a 100644 --- a/README.md +++ b/README.md @@ -147,3 +147,19 @@ HTTP API расширяется query-параметрами `from` и `ack`, с Когда всё будет готово, присылайте pull request с изменениями, результатами нагрузочного тестирования и профилирования, а также **анализом результатов по сравнению с предыдущей** (синхронной) версией. На всех этапах **оценивается и код, и анализ (отчёт)** -- без анализа полученных результатов работа оценивается минимальным количеством баллов. Не забывайте **отвечать на комментарии в PR** (в том числе автоматизированные) и **исправлять замечания**! + +## Этап 6. Range-запросы (soft deadline 2024-04-25 18:29:59 MSK, hard deadline 2024-05-01 23:59:59 MSK) + +Реализуйте получение **диапазона данных текущего узла** с помощью HTTP `GET /v0/entities?start=[&end=]`, который возвращает: +* Статус код `200 OK` +* Возможно пустой **отсортированный** (по ключу) набор **ключей** и **значений** в диапазоне ключей от **обязательного** `start` (включая) до **опционального** `end` (не включая) +* Используйте [Chunked transfer encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding) +* Чанки в формате `\n` + +Диапазон должен отдаваться в **потоковом режиме** без формирования всего ответа в памяти. +Проверить корректность можно, запросив весь диапазон данных предварительно наполненной БД размером больше Java Heap. + +### Report +После прохождения модульных тестов, присылайте pull request с изменениями. +Наполните БД большим объёмом данных и отпрофилируйте cpu, alloc и lock при получении range всей базы одним запросом curl'ом. +Присылайте отчёт с анализом результатов и оптимизаций. diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/ProxyRequestHandler.java b/src/main/java/ru/vk/itmo/test/asvistukhin/ProxyRequestHandler.java index c194627e2..f4808594e 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/ProxyRequestHandler.java +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/ProxyRequestHandler.java @@ -8,88 +8,102 @@ import ru.vk.itmo.ServiceConfig; import java.io.IOException; +import java.net.ConnectException; +import java.net.HttpURLConnection; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.util.HashMap; -import java.util.SortedMap; -import java.util.TreeMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; public class ProxyRequestHandler { private static final Logger log = LoggerFactory.getLogger(ProxyRequestHandler.class); - private final HashMap nodes; - private final SortedMap hashRing = new TreeMap<>(); - private final String selfUrl; + private final Map clients; + private final Map urlHashes; public ProxyRequestHandler(ServiceConfig serviceConfig) { - this.nodes = new HashMap<>(serviceConfig.clusterUrls().size()); - this.selfUrl = serviceConfig.selfUrl(); + this.clients = HashMap.newHashMap(serviceConfig.clusterUrls().size() - 1); - for (int i = 0; i < serviceConfig.clusterUrls().size(); i++) { - String url = serviceConfig.clusterUrls().get(i); - nodes.put(url, HttpClient.newHttpClient()); - int hash = Hash.murmur3(url); - hashRing.put(hash, url); + for (String url : serviceConfig.clusterUrls()) { + if (!Objects.equals(url, serviceConfig.selfUrl())) { + clients.put(url, HttpClient.newHttpClient()); + } + } + this.urlHashes = HashMap.newHashMap(serviceConfig.clusterUrls().size()); + for (String url : serviceConfig.clusterUrls()) { + urlHashes.put(url, Hash.murmur3(url)); } } public synchronized void close() { - nodes.values().forEach(HttpClient::close); + clients.values().forEach(HttpClient::close); } - public Response proxyRequest(Request request) { - String id = request.getParameter("id="); - String nodeUrl = getNodeByKey(id); + public Map proxyRequests(Request request, List nodeUrls) throws IOException { + Map responses = HashMap.newHashMap(nodeUrls.size()); + for (String url : nodeUrls) { + Response response = proxyRequest(request, url); + responses.put(url, response); + } - HttpRequest.BodyPublisher bodyPublisher = request.getBody() == null - ? HttpRequest.BodyPublishers.noBody() : HttpRequest.BodyPublishers.ofByteArray(request.getBody()); + return responses; + } - URI uri = URI.create(nodeUrl + request.getPath() + "?id=" + id); - log.debug("Proxy request to {}", uri); + public Response proxyRequest(Request request, String proxiedNodeUrl) throws IOException { + String id = request.getParameter("id="); + byte[] body = request.getBody(); + URI uri = URI.create(proxiedNodeUrl + request.getPath() + "?id=" + id); try { - HttpResponse response = nodes.get(nodeUrl).send( + HttpResponse httpResponse = clients.get(proxiedNodeUrl).send( HttpRequest.newBuilder() .uri(uri) - .method(request.getMethodName(), bodyPublisher) + .method( + request.getMethodName(), + body == null + ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofByteArray(body) + ) + .header(RequestWrapper.SELF_HEADER, "true") .build(), - HttpResponse.BodyHandlers.ofByteArray() - ); - return parseResponse(response); + HttpResponse.BodyHandlers.ofByteArray()); + Response response = new Response(proxyResponseCode(httpResponse), httpResponse.body()); + long timestamp = httpResponse.headers().firstValueAsLong(RequestWrapper.NIO_TIMESTAMP_HEADER).orElse(0); + response.addHeader(RequestWrapper.NIO_TIMESTAMP_STRING_HEADER + timestamp); + return response; } catch (InterruptedException ex) { log.error("Proxy request thread interrupted", ex); Thread.currentThread().interrupt(); return new Response(Response.INTERNAL_ERROR, Response.EMPTY); - } catch (IOException ex) { - log.error("IOException during proxy request to another node", ex); + } catch (IllegalArgumentException ex) { + log.error("IllegalArgumentException during proxy request to another node", ex); + return new Response(Response.INTERNAL_ERROR, Response.EMPTY); + } catch (ConnectException ex) { + log.error("ConnectException during proxy request to another node", ex); return new Response(Response.INTERNAL_ERROR, Response.EMPTY); } } - public boolean isNeedProxy(String id) { - return !getNodeByKey(id).equals(selfUrl); - } - - private Response parseResponse(HttpResponse response) { - String responseHttpCode = switch (response.statusCode()) { - case 200 -> Response.OK; - case 201 -> Response.CREATED; - case 202 -> Response.ACCEPTED; - case 400 -> Response.BAD_REQUEST; - case 404 -> Response.NOT_FOUND; - case 405 -> Response.METHOD_NOT_ALLOWED; - case 503 -> Response.SERVICE_UNAVAILABLE; + private String proxyResponseCode(HttpResponse response) { + return switch (response.statusCode()) { + case HttpURLConnection.HTTP_OK -> Response.OK; + case HttpURLConnection.HTTP_CREATED -> Response.CREATED; + case HttpURLConnection.HTTP_ACCEPTED -> Response.ACCEPTED; + case HttpURLConnection.HTTP_BAD_REQUEST -> Response.BAD_REQUEST; + case HttpURLConnection.HTTP_NOT_FOUND -> Response.NOT_FOUND; default -> Response.INTERNAL_ERROR; }; - - return new Response(responseHttpCode, response.body()); } - private String getNodeByKey(String id) { - int hash = Hash.murmur3(id); - SortedMap tailMap = hashRing.tailMap(hash); - hash = tailMap.isEmpty() ? hashRing.firstKey() : tailMap.firstKey(); - return hashRing.get(hash); + public List getNodesByHash(int numOfNodes) { + return urlHashes.entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .limit(numOfNodes) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); } } diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/RequestHandler.java b/src/main/java/ru/vk/itmo/test/asvistukhin/RequestHandler.java new file mode 100644 index 000000000..01cfd1040 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/RequestHandler.java @@ -0,0 +1,78 @@ +package ru.vk.itmo.test.asvistukhin; + +import one.nio.http.Param; +import one.nio.http.Request; +import one.nio.http.Response; +import ru.vk.itmo.test.asvistukhin.dao.Dao; +import ru.vk.itmo.test.asvistukhin.dao.TimestampEntry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; + +public class RequestHandler { + private final Dao> dao; + + public RequestHandler(Dao> dao) { + this.dao = dao; + } + + public Response handle(Request request) { + String id = request.getParameter("id="); + return switch (request.getMethod()) { + case Request.METHOD_GET -> get(id); + case Request.METHOD_PUT -> put(id, request); + case Request.METHOD_DELETE -> delete(id); + default -> new Response(Response.BAD_REQUEST, Response.EMPTY); + }; + } + + public Response get(@Param(value = "id", required = true) String id) { + if (RequestWrapper.isEmptyParam(id)) { + return new Response(Response.BAD_REQUEST, Response.EMPTY); + } + + TimestampEntry entry = dao.get(MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8))); + + Response response; + long timestamp; + if (entry == null || entry.value() == null) { + timestamp = (entry != null ? entry.timestamp() : 0); + response = new Response(Response.NOT_FOUND, Response.EMPTY); + response.addHeader(RequestWrapper.TIMESTAMP_STRING_HEADER + timestamp); + } else { + timestamp = entry.timestamp(); + response = Response.ok(entry.value().toArray(ValueLayout.JAVA_BYTE)); + response.addHeader(RequestWrapper.TIMESTAMP_STRING_HEADER + timestamp); + } + + return response; + } + + public Response put(@Param(value = "id", required = true) String id, Request request) { + if (RequestWrapper.isEmptyParam(id) || RequestWrapper.isEmptyRequest(request)) { + return new Response(Response.BAD_REQUEST, Response.EMPTY); + } + + MemorySegment key = MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)); + MemorySegment value = MemorySegment.ofArray(request.getBody()); + dao.upsert(new TimestampEntry<>(key, value, System.currentTimeMillis())); + + return new Response(Response.CREATED, Response.EMPTY); + } + + public Response delete(@Param(value = "id", required = true) String id) { + if (RequestWrapper.isEmptyParam(id)) { + return new Response(Response.BAD_REQUEST, Response.EMPTY); + } + + dao.upsert( + new TimestampEntry<>( + MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)), + null, + System.currentTimeMillis()) + ); + + return new Response(Response.ACCEPTED, Response.EMPTY); + } +} diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/RequestWrapper.java b/src/main/java/ru/vk/itmo/test/asvistukhin/RequestWrapper.java new file mode 100644 index 000000000..5a9ebc909 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/RequestWrapper.java @@ -0,0 +1,44 @@ +package ru.vk.itmo.test.asvistukhin; + +import one.nio.http.Request; + +public class RequestWrapper { + private static final String ID_PARAM = "id="; + private static final String FROM_PARAM = "from="; + private static final String ACK_PARAM = "ack="; + public static final String SELF_HEADER = "X-Self"; + public static final String NIO_TIMESTAMP_HEADER = "x-timestamp"; + public static final String TIMESTAMP_STRING_HEADER = "X-Timestamp:"; + public static final String NIO_TIMESTAMP_STRING_HEADER = "x-timestamp:"; + + public final String id; + public final int from; + public final int ack; + + public RequestWrapper(Request request, int clusterSize) throws IllegalArgumentException { + String idString = request.getParameter(ID_PARAM); + String fromString = request.getParameter(FROM_PARAM); + String ackString = request.getParameter(ACK_PARAM); + if (isEmptyParam(idString)) throw new IllegalArgumentException(); + + id = idString; + try { + from = isEmptyParam(fromString) ? clusterSize : Integer.parseInt(fromString); + ack = isEmptyParam(ackString) ? (from + 1) / 2 : Integer.parseInt(ackString); + } catch (NumberFormatException ex) { + throw new IllegalArgumentException(ex); + } + + if (from <= 0 || ack <= 0 || from < ack || clusterSize < from) { + throw new IllegalArgumentException(); + } + } + + public static boolean isEmptyParam(String param) { + return param == null || param.isEmpty(); + } + + public static boolean isEmptyRequest(Request request) { + return request.getBody() == null; + } +} diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/ServerImpl.java b/src/main/java/ru/vk/itmo/test/asvistukhin/ServerImpl.java index 6429d43e2..602c5c40e 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/ServerImpl.java +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/ServerImpl.java @@ -11,26 +11,46 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.vk.itmo.ServiceConfig; +import ru.vk.itmo.test.asvistukhin.dao.PersistentDao; import java.io.IOException; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; public class ServerImpl extends HttpServer { private static final Logger log = LoggerFactory.getLogger(ServerImpl.class); + private static final List ALLOWED_METHODS = List.of( + Request.METHOD_GET, + Request.METHOD_PUT, + Request.METHOD_DELETE + ); private static final String HTTP_SERVICE_NOT_AVAILABLE = "503"; + private static final String NOT_ENOUGH_REPLICAS_RESPONSE = "504 Not Enough Replicas"; private static final int QUEUE_CAPACITY = 3000; private final ThreadPoolExecutor executor; + private final ServiceConfig serviceConfig; + private final RequestHandler requestHandler; + private final ProxyRequestHandler proxyRequestHandler; private final AtomicBoolean isServerStopped = new AtomicBoolean(false); - public ServerImpl(ServiceConfig config) throws IOException { - super(createHttpServerConfig(config)); + public ServerImpl( + ServiceConfig serviceConfig, + PersistentDao persistentDao, + ProxyRequestHandler proxyRequestHandler + ) throws IOException { + super(createHttpServerConfig(serviceConfig)); + this.serviceConfig = serviceConfig; + this.requestHandler = new RequestHandler(persistentDao); + this.proxyRequestHandler = proxyRequestHandler; executor = new ThreadPoolExecutor( 50, 150, @@ -49,11 +69,7 @@ public void handleRequest(Request request, HttpSession session) { @Override public void handleDefault(Request request, HttpSession session) throws IOException { - Response response = List.of( - Request.METHOD_GET, - Request.METHOD_PUT, - Request.METHOD_DELETE - ).contains(request.getMethod()) + Response response = ALLOWED_METHODS.contains(request.getMethod()) ? new Response(Response.BAD_REQUEST, Response.EMPTY) : new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); session.sendResponse(response); @@ -71,7 +87,17 @@ public synchronized void stop() { private void wrapHandleRequest(Request request, HttpSession session) { try { - super.handleRequest(request, session); + if (!ALLOWED_METHODS.contains(request.getMethod())) { + handleDefault(request, session); + return; + } + + RequestWrapper parameters = new RequestWrapper(request, serviceConfig.clusterUrls().size()); + if (request.getHeader(RequestWrapper.SELF_HEADER) == null) { + processFirstRequest(request, session, parameters); + } else { + session.sendResponse(requestHandler.handle(request)); + } } catch (RejectedExecutionException executionException) { try { log.error("Rejected execution new request.", executionException); @@ -83,7 +109,9 @@ private void wrapHandleRequest(Request request, HttpSession session) { } } catch (Exception ex) { try { - String response = ex.getClass() == HttpException.class ? Response.BAD_REQUEST : Response.INTERNAL_ERROR; + String response = ex.getClass() == HttpException.class + || ex.getClass() == IllegalArgumentException.class + ? Response.BAD_REQUEST : Response.INTERNAL_ERROR; session.sendError(response, null); } catch (IOException ioEx) { log.error("Failed send error response to client.", ioEx); @@ -93,6 +121,49 @@ private void wrapHandleRequest(Request request, HttpSession session) { } } + private void processFirstRequest( + Request request, + HttpSession session, + RequestWrapper parameters + ) throws IOException { + List nodeUrls = proxyRequestHandler.getNodesByHash(parameters.from); + if (nodeUrls.size() < parameters.from) { + session.sendResponse(new Response(NOT_ENOUGH_REPLICAS_RESPONSE, Response.EMPTY)); + } + + boolean isSelfProcessing = nodeUrls.remove(serviceConfig.selfUrl()); + Map responses = proxyRequestHandler.proxyRequests(request, nodeUrls); + + if (isSelfProcessing) { + responses.put(serviceConfig.selfUrl(), requestHandler.handle(request)); + } + + List validResponses = responses.values().stream() + .filter(response -> isSuccessProcessed(response.getStatus())) + .collect(Collectors.toList()); + if (validResponses.size() >= parameters.ack) { + if (request.getMethod() == Request.METHOD_GET) { + validResponses.sort( + Comparator.comparingLong(r -> { + String timestamp = r.getHeader(RequestWrapper.NIO_TIMESTAMP_STRING_HEADER); + return timestamp == null ? 0 : Long.parseLong(timestamp); + } + ) + ); + session.sendResponse(validResponses.getLast()); + } else { + session.sendResponse(validResponses.getFirst()); + } + } else { + session.sendResponse(new Response(NOT_ENOUGH_REPLICAS_RESPONSE, Response.EMPTY)); + } + } + + private boolean isSuccessProcessed(int status) { + // not server and time limit errors + return status < 500; + } + private static HttpServerConfig createHttpServerConfig(ServiceConfig serviceConfig) { HttpServerConfig serverConfig = new HttpServerConfig(); AcceptorConfig acceptorConfig = new AcceptorConfig(); diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/ServiceImpl.java b/src/main/java/ru/vk/itmo/test/asvistukhin/ServiceImpl.java index 54e1ac068..3bc2fb0cd 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/ServiceImpl.java +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/ServiceImpl.java @@ -1,22 +1,12 @@ package ru.vk.itmo.test.asvistukhin; -import one.nio.http.Param; -import one.nio.http.Path; -import one.nio.http.Request; -import one.nio.http.RequestMethod; -import one.nio.http.Response; import ru.vk.itmo.Service; import ru.vk.itmo.ServiceConfig; -import ru.vk.itmo.dao.BaseEntry; import ru.vk.itmo.dao.Config; -import ru.vk.itmo.dao.Entry; import ru.vk.itmo.test.ServiceFactory; import ru.vk.itmo.test.asvistukhin.dao.PersistentDao; import java.io.IOException; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; -import java.nio.charset.StandardCharsets; import java.util.concurrent.CompletableFuture; public class ServiceImpl implements Service { @@ -33,8 +23,8 @@ public ServiceImpl(ServiceConfig serviceConfig) { @Override public CompletableFuture start() throws IOException { dao = new PersistentDao(new Config(serviceConfig.workingDir(), 1024 * 1024 * 5L)); - server = new ServerImpl(serviceConfig); proxyRequestHandler = new ProxyRequestHandler(serviceConfig); + server = new ServerImpl(serviceConfig, dao, proxyRequestHandler); server.addRequestHandlers(this); server.start(); @@ -49,68 +39,7 @@ public CompletableFuture stop() throws IOException { return CompletableFuture.completedFuture(null); } - @Path("/v0/entity") - @RequestMethod(Request.METHOD_GET) - public Response get(@Param(value = "id", required = true) String id, Request request) { - if (id.isEmpty()) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); - } - - if (proxyRequestHandler.isNeedProxy(id)) { - return proxyRequestHandler.proxyRequest(request); - } - - Entry entry = dao.get(MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8))); - if (entry == null) { - return new Response(Response.NOT_FOUND, Response.EMPTY); - } - - return new Response(Response.OK, entry.value().toArray(ValueLayout.JAVA_BYTE)); - } - - @Path("/v0/entity") - @RequestMethod(Request.METHOD_PUT) - public Response upsert(@Param(value = "id", required = true) String id, Request request) { - if (id.isEmpty()) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); - } - - if (proxyRequestHandler.isNeedProxy(id)) { - return proxyRequestHandler.proxyRequest(request); - } - - dao.upsert( - new BaseEntry<>( - MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)), - MemorySegment.ofArray(request.getBody()) - ) - ); - - return new Response(Response.CREATED, Response.EMPTY); - } - - @Path("/v0/entity") - @RequestMethod(Request.METHOD_DELETE) - public Response delete(@Param(value = "id", required = true) String id, Request request) { - if (id.isEmpty()) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); - } - - if (proxyRequestHandler.isNeedProxy(id)) { - return proxyRequestHandler.proxyRequest(request); - } - - dao.upsert( - new BaseEntry<>( - MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)), - null - ) - ); - - return new Response(Response.ACCEPTED, Response.EMPTY); - } - - @ServiceFactory(stage = 3) + @ServiceFactory(stage = 4) public static class Factory implements ServiceFactory.Factory { @Override diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/Dao.java b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/Dao.java new file mode 100644 index 000000000..976310404 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/Dao.java @@ -0,0 +1,87 @@ +package ru.vk.itmo.test.asvistukhin.dao; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; + +public interface Dao> extends Closeable { + /** + * Returns ordered iterator of entries with keys between from (inclusive) and to (exclusive). + * @param from lower bound of range (inclusive) + * @param to upper bound of range (exclusive) + * @return entries [from;to) + */ + Iterator get(D from, D to); + + /** + * Returns entry by key. Note: default implementation is far from optimal. + * @param key entry`s key + * @return entry + */ + default E get(D key) { + Iterator iterator = get(key, null); + if (!iterator.hasNext()) { + return null; + } + + E next = iterator.next(); + if (next.key().equals(key)) { + return next; + } + return null; + } + + /** + * Returns ordered iterator of all entries with keys from (inclusive). + * @param from lower bound of range (inclusive) + * @return entries with key >= from + */ + default Iterator allFrom(D from) { + return get(from, null); + } + + /** + * Returns ordered iterator of all entries with keys < to. + * @param to upper bound of range (exclusive) + * @return entries with key < to + */ + default Iterator allTo(D to) { + return get(null, to); + } + + /** + * Returns ordered iterator of all entries. + * @return all entries + */ + default Iterator all() { + return get(null, null); + } + + /** + * Inserts of replaces entry. + * @param entry element to upsert + */ + void upsert(E entry); + + /** + * Persists data (no-op by default). + */ + default void flush() throws IOException { + // Do nothing + } + + /** + * Compacts data (no-op by default). + */ + default void compact() throws IOException { + // Do nothing + } + + /* + * Releases Dao (calls flush by default). + */ + @Override + default void close() throws IOException { + flush(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/DiskStorage.java b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/DiskStorage.java index 5b9e215e7..f2b317847 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/DiskStorage.java +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/DiskStorage.java @@ -1,6 +1,5 @@ package ru.vk.itmo.test.asvistukhin.dao; -import ru.vk.itmo.dao.BaseEntry; import ru.vk.itmo.dao.Entry; import java.io.IOException; @@ -36,10 +35,10 @@ public void mapSSTableAfterFlush(Path storagePath, String fileName, Arena arena) Path file = storagePath.resolve(fileName); try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { MemorySegment fileSegment = fileChannel.map( - FileChannel.MapMode.READ_WRITE, - 0, - Files.size(file), - arena + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena ); this.segmentList.add(fileSegment); } @@ -50,12 +49,11 @@ public void mapSSTableAfterCompaction(Path storagePath, Arena arena) throws IOEx mapSSTableAfterFlush(storagePath, DATA_FILE_AFTER_COMPACTION, arena); } - public Iterator> range( - StorageState storageState, - MemorySegment from, - MemorySegment to + public Iterator> range( + StorageState storageState, + MemorySegment from, MemorySegment to ) { - List>> iterators = new ArrayList<>(segmentList.size() + 1); + List>> iterators = new ArrayList<>(segmentList.size() + 1); for (MemorySegment memorySegment : segmentList) { iterators.add(iterator(memorySegment, from, to)); } @@ -65,19 +63,16 @@ public Iterator> range( } iterators.add(storageState.getActiveSSTable().get(from, to)); - return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, MemorySegmentUtils::compare)) { + return new MergeIterator<>(iterators, Comparator.comparing(TimestampEntry::key, MemorySegmentUtils::compare)) { @Override - protected boolean shouldSkip(Entry memorySegmentEntry) { + protected boolean shouldSkip(TimestampEntry memorySegmentEntry) { return memorySegmentEntry.value() == null; } }; } - public Iterator> rangeFromDisk( - MemorySegment from, - MemorySegment to - ) { - List>> iterators = new ArrayList<>(segmentList.size() + 1); + public Iterator> rangeFromDisk(MemorySegment from, MemorySegment to) { + List>> iterators = new ArrayList<>(segmentList.size() + 1); iterators.add(Collections.emptyIterator()); for (MemorySegment memorySegment : segmentList) { iterators.add(iterator(memorySegment, from, to)); @@ -85,110 +80,86 @@ public Iterator> rangeFromDisk( return new MergeIterator<>(iterators, Comparator.comparing(Entry::key, MemorySegmentUtils::compare)) { @Override - protected boolean shouldSkip(Entry memorySegmentEntry) { + protected boolean shouldSkip(TimestampEntry memorySegmentEntry) { return memorySegmentEntry.value() == null; } }; } - public static String save( - Path storagePath, - Iterable> iterable - ) throws IOException { + public static String save(Path storagePath, Iterable> iterable) throws IOException { final Path indexTmp = storagePath.resolve("index.tmp"); final Path indexFile = storagePath.resolve("index.idx"); - try { Files.createFile(indexFile); } catch (FileAlreadyExistsException ignored) { // it is ok, actually it is normal state } List existedFiles = Files.readAllLines(indexFile, StandardCharsets.UTF_8); - String newFileName = String.valueOf(existedFiles.size()); - long dataSize = 0; long count = 0; - for (Entry entry : iterable) { + for (TimestampEntry entry : iterable) { dataSize += entry.key().byteSize(); MemorySegment value = entry.value(); if (value != null) { dataSize += value.byteSize(); } + dataSize += Long.BYTES; count++; } long indexSize = count * 2 * Long.BYTES; - try ( - FileChannel fileChannel = FileChannel.open( - storagePath.resolve(newFileName), - StandardOpenOption.WRITE, - StandardOpenOption.READ, - StandardOpenOption.CREATE - ); - Arena writeArena = Arena.ofConfined() + FileChannel fileChannel = FileChannel.open( + storagePath.resolve(newFileName), + StandardOpenOption.WRITE, + StandardOpenOption.READ, + StandardOpenOption.CREATE + ); + Arena writeArena = Arena.ofConfined() ) { MemorySegment fileSegment = fileChannel.map( - FileChannel.MapMode.READ_WRITE, - 0, - indexSize + dataSize, - writeArena + FileChannel.MapMode.READ_WRITE, + 0, + indexSize + dataSize, + writeArena ); - - // index: - // |key0_Start|value0_Start|key1_Start|value1_Start|key2_Start|value2_Start|... - // key0_Start = data start = end of index long dataOffset = indexSize; int indexOffset = 0; - for (Entry entry : iterable) { + for (TimestampEntry entry : iterable) { fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); dataOffset += entry.key().byteSize(); indexOffset += Long.BYTES; - MemorySegment value = entry.value(); if (value == null) { fileSegment.set( - ValueLayout.JAVA_LONG_UNALIGNED, - indexOffset, - MemorySegmentUtils.tombstone(dataOffset) + ValueLayout.JAVA_LONG_UNALIGNED, + indexOffset, + MemorySegmentUtils.tombstone(dataOffset) ); } else { fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, indexOffset, dataOffset); dataOffset += value.byteSize(); } + dataOffset += Long.BYTES; indexOffset += Long.BYTES; } - // data: - // |key0|value0|key1|value1|... dataOffset = indexSize; - for (Entry entry : iterable) { + for (TimestampEntry entry : iterable) { MemorySegment key = entry.key(); MemorySegment.copy(key, 0, fileSegment, dataOffset, key.byteSize()); dataOffset += key.byteSize(); - MemorySegment value = entry.value(); if (value != null) { MemorySegment.copy(value, 0, fileSegment, dataOffset, value.byteSize()); dataOffset += value.byteSize(); } + fileSegment.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, entry.timestamp()); + dataOffset += Long.BYTES; } } - Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - - List list = new ArrayList<>(existedFiles.size() + 1); - list.addAll(existedFiles); - list.add(newFileName); - Files.write( - indexFile, - list, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING - ); - - Files.delete(indexTmp); + manageDataFiles(indexFile, indexTmp, newFileName, existedFiles); return newFileName; } @@ -212,10 +183,10 @@ public static List loadOrRecover(Path storagePath, Arena arena) t Path file = storagePath.resolve(fileName); try (FileChannel fileChannel = FileChannel.open(file, StandardOpenOption.READ, StandardOpenOption.WRITE)) { MemorySegment fileSegment = fileChannel.map( - FileChannel.MapMode.READ_WRITE, - 0, - Files.size(file), - arena + FileChannel.MapMode.READ_WRITE, + 0, + Files.size(file), + arena ); result.add(fileSegment); } @@ -238,10 +209,35 @@ public static void deleteObsoleteData(Path directory) throws IOException { } } - private static Iterator> iterator(MemorySegment page, MemorySegment from, MemorySegment to) { + private static void manageDataFiles( + Path indexFile, + Path indexTmp, + String newFileName, + List existedFiles + ) throws IOException { + Files.move(indexFile, indexTmp, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + List list = new ArrayList<>(existedFiles.size() + 1); + list.addAll(existedFiles); + list.add(newFileName); + Files.write( + indexFile, + list, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING + ); + Files.delete(indexTmp); + } + + private static Iterator> iterator( + MemorySegment page, + MemorySegment from, + MemorySegment to + ) { long recordIndexFrom = from == null ? 0 : MemorySegmentUtils.normalize(MemorySegmentUtils.indexOf(page, from)); - long recordIndexTo = to == null ? MemorySegmentUtils.recordsCount(page) - : MemorySegmentUtils.normalize(MemorySegmentUtils.indexOf(page, to)); + long recordIndexTo = to == null + ? MemorySegmentUtils.recordsCount(page) + : MemorySegmentUtils.normalize(MemorySegmentUtils.indexOf(page, to)); long recordsCount = MemorySegmentUtils.recordsCount(page); return new Iterator<>() { @@ -253,23 +249,22 @@ public boolean hasNext() { } @Override - public Entry next() { + public TimestampEntry next() { if (!hasNext()) { throw new NoSuchElementException(); } MemorySegment key = MemorySegmentUtils.slice( - page, - MemorySegmentUtils.startOfKey(page, index), - MemorySegmentUtils.endOfKey(page, index) + page, + MemorySegmentUtils.startOfKey(page, index), + MemorySegmentUtils.endOfKey(page, index) ); long startOfValue = MemorySegmentUtils.startOfValue(page, index); - MemorySegment value = startOfValue < 0 ? null : MemorySegmentUtils.slice( - page, - startOfValue, - MemorySegmentUtils.endOfValue(page, index, recordsCount) - ); + long endOfValue = MemorySegmentUtils.endOfValue(page, index, recordsCount); + MemorySegment value = startOfValue < 0 + ? null : MemorySegmentUtils.slice(page, startOfValue, endOfValue); + long timestamp = page.get(ValueLayout.JAVA_LONG_UNALIGNED, endOfValue); index++; - return new BaseEntry<>(key, value); + return new TimestampEntry<>(key, value, timestamp); } }; } diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/ITimestampEntry.java b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/ITimestampEntry.java new file mode 100644 index 000000000..e891bed33 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/ITimestampEntry.java @@ -0,0 +1,13 @@ +package ru.vk.itmo.test.asvistukhin.dao; + +import ru.vk.itmo.dao.Entry; + +public interface ITimestampEntry extends Entry { + @Override + T key(); + + @Override + T value(); + + long timestamp(); +} diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/MemorySegmentUtils.java b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/MemorySegmentUtils.java index 0cec15b08..a5aa585dd 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/MemorySegmentUtils.java +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/MemorySegmentUtils.java @@ -71,7 +71,7 @@ public static long endOfValue(MemorySegment segment, long recordIndex, long reco if (recordIndex < recordsCount - 1) { return startOfKey(segment, recordIndex + 1); } - return segment.byteSize(); + return segment.byteSize() - Long.BYTES; } public static long normalize(long value) { diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/PersistentDao.java b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/PersistentDao.java index cd00762f6..5f15285af 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/PersistentDao.java +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/PersistentDao.java @@ -1,8 +1,6 @@ package ru.vk.itmo.test.asvistukhin.dao; import ru.vk.itmo.dao.Config; -import ru.vk.itmo.dao.Dao; -import ru.vk.itmo.dao.Entry; import java.io.IOException; import java.io.UncheckedIOException; @@ -16,7 +14,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantReadWriteLock; -public class PersistentDao implements Dao> { +public class PersistentDao implements Dao> { private final StorageState storageState; private final Arena arena; @@ -42,12 +40,12 @@ public PersistentDao(Config config) throws IOException { } @Override - public Iterator> get(MemorySegment from, MemorySegment to) { + public Iterator> get(MemorySegment from, MemorySegment to) { return diskStorage.range(storageState, from, to); } @Override - public void upsert(Entry entry) { + public void upsert(TimestampEntry entry) { if (storageState.getActiveSSTable().getStorageSize() >= config.getFlushThresholdBytes() && storageState.getFlushingSSTable() != null ) { @@ -71,21 +69,18 @@ public void upsert(Entry entry) { } @Override - public Entry get(MemorySegment key) { - Entry entry = storageState.getActiveSSTable().get(key); + public TimestampEntry get(MemorySegment key) { + TimestampEntry entry = storageState.getActiveSSTable().get(key); if (entry != null) { - if (entry.value() == null) { - return null; - } return entry; } - Iterator> iterator = diskStorage.rangeFromDisk(key, null); + Iterator> iterator = diskStorage.rangeFromDisk(key, null); if (!iterator.hasNext()) { return null; } - Entry next = iterator.next(); + TimestampEntry next = iterator.next(); if (MemorySegmentUtils.isSameKey(next.key(), key)) { return next; } @@ -101,7 +96,7 @@ public synchronized void compact() { executor.execute(() -> { try { - Iterable> compactValues = () -> diskStorage.rangeFromDisk(null, null); + Iterable> compactValues = () -> diskStorage.rangeFromDisk(null, null); if (compactValues.iterator().hasNext()) { Files.createDirectories(config.getCompactTempPath()); DiskStorage.save(config.getCompactTempPath(), compactValues); @@ -133,7 +128,7 @@ public synchronized void flush() throws IOException { } @Override - public synchronized void close() throws IOException { + public synchronized void close() { if (!arena.scope().isAlive()) { return; } @@ -156,7 +151,7 @@ public synchronized void close() throws IOException { private void flushToDisk() { try { - Iterable> valuesToFlush = () -> storageState.getFlushingSSTable().getAll(); + Iterable> valuesToFlush = () -> storageState.getFlushingSSTable().getAll(); String filename = DiskStorage.save(config.getDataPath(), valuesToFlush); storageState.removeFlushingSSTable(); diskStorage.mapSSTableAfterFlush(config.getDataPath(), filename, arena); diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/SSTable.java b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/SSTable.java index d686ed49a..f5d0fd955 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/SSTable.java +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/SSTable.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.asvistukhin.dao; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.util.Comparator; import java.util.Iterator; @@ -11,14 +9,15 @@ public class SSTable { private final Comparator comparator = MemorySegmentUtils::compare; - private final NavigableMap> storage = new ConcurrentSkipListMap<>(comparator); + private final NavigableMap> storage = + new ConcurrentSkipListMap<>(comparator); private final AtomicLong sizeInBytes = new AtomicLong(0); - public Iterator> getAll() { + public Iterator> getAll() { return storage.values().iterator(); } - public Iterator> get(MemorySegment from, MemorySegment to) { + public Iterator> get(MemorySegment from, MemorySegment to) { if (from == null && to == null) { return storage.values().iterator(); } @@ -32,12 +31,12 @@ public Iterator> get(MemorySegment from, MemorySegment to) return storage.subMap(from, to).values().iterator(); } - public Entry get(MemorySegment key) { + public TimestampEntry get(MemorySegment key) { return storage.get(key); } - public void upsert(Entry entry) { - Entry oldValue = storage.get(entry.key()); + public void upsert(TimestampEntry entry) { + TimestampEntry oldValue = storage.get(entry.key()); storage.put(entry.key(), entry); if (oldValue != null) { sizeInBytes.addAndGet(-byteSizeOfEntry(oldValue)); @@ -45,7 +44,7 @@ public void upsert(Entry entry) { sizeInBytes.addAndGet(byteSizeOfEntry(entry)); } - public NavigableMap> getStorage() { + public NavigableMap> getStorage() { return storage; } @@ -53,7 +52,7 @@ public long getStorageSize() { return sizeInBytes.get(); } - public static long byteSizeOfEntry(Entry entry) { + public static long byteSizeOfEntry(TimestampEntry entry) { long valueSize = entry.value() == null ? 0L : entry.value().byteSize(); return entry.key().byteSize() + valueSize; } diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/dao/TimestampEntry.java b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/TimestampEntry.java new file mode 100644 index 000000000..0102bad89 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/dao/TimestampEntry.java @@ -0,0 +1,10 @@ +package ru.vk.itmo.test.asvistukhin.dao; + +public record TimestampEntry(T key, T value, long timestamp) implements ITimestampEntry { + + @Override + public String toString() { + return "{" + key + ":" + value + " - " + timestamp; + } + +} diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/get_alloc.html b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/get_alloc.html new file mode 100644 index 000000000..a4fdb856b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/get_alloc.html @@ -0,0 +1,460 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/get_cpu.html b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/get_cpu.html new file mode 100644 index 000000000..b03eb2630 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/get_cpu.html @@ -0,0 +1,1572 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/put_alloc.html b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/put_alloc.html new file mode 100644 index 000000000..cb5a24fb9 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/put_alloc.html @@ -0,0 +1,481 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/put_cpu.html b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/put_cpu.html new file mode 100644 index 000000000..a5d049aec --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4/put_cpu.html @@ -0,0 +1,2300 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/stage4.md b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/stage4.md new file mode 100644 index 000000000..d6acf9f08 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/stage4.md @@ -0,0 +1,91 @@ +## Нагрузочное тестирование +Тестирование проводил на одной машине в одной среде. Сервер и wrk/async запускались рядом. +JVM была прогрета запросами. Тестировал на трех нодах. Запросы шли с ack=2, from=3. +Процессор - i7-10510U. + +В рамках данного этапа было реализовано реплицирование, проанализирован код. + +## Наполнение базы данных +Было сгенерировано с помощью WRK ~500MB данных. +1. Первая нода - 24 SSTables +2. Вторая нода - 24 SSTables +3. Третья нода - 24 SSTables + + +### PUT 8000 RPS +WRK отчет. Тестировал одну минуту, в одном потоке и 64 подключения. Точка разладки ~8000 RPS. С добавлением времени теста, +сервер не деградирует. Но если поставить ~12000 RPS, время ответа становится сильно больше. +``` +Running 3m test @ http://localhost:8080/v0/entity + 1 threads and 64 connections + Thread calibration: mean lat.: 31.861ms, rate sampling interval: 226ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.43ms 4.50ms 111.81ms 97.88% + Req/Sec 8.02k 194.31 10.71k 96.53% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.72ms + 75.000% 2.49ms + 90.000% 3.29ms + 99.000% 21.68ms + 99.900% 67.07ms + 99.990% 94.46ms + 99.999% 104.19ms +100.000% 111.87ms +``` + +### async-profiler +#### CPU +Производительность PUT запросов упала ~20%. И видно, что 20% процессорного времени занимает чтение сокета (ожидание ответов от других нод). +Ещё, из-за множества HTTP запросов начал активнее работать GC, он начал есть целых 5.5% процессорного времени (что в два раза больше, чем в предыдущем этапе). +Теперь у нас практически всё процессорное время забирают сетевые операции: чтение/формирование заголовков, ожидание ответа от сокета и так далее. + +#### ALLOC +Аллокации практически не изменились. 22% аллокаций занимает локальная вставка, всё остальное - сетевые операции с другими хостами. А именно, формирование и чтение +заголовков, тела ответа и так далее. Также, при каждом новом запросе мы генерируем новый URL к ноде, ведь мы меняем параметры запроса. + +### GET 8000 RPS +WRK отчет. Тестировал одну минуту, в одном потоке и 64 подключения. Точка разладки ~8000 RPS. С добавлением времени теста, +сервер не деградирует. Но точка разладки очень чувствительна, даже шаг в 1000 RPS поднимает в разы время ответа. +``` +Running 1m test @ http://localhost:8080/v0/entity + 1 threads and 64 connections + Thread calibration: mean lat.: 17.085ms, rate sampling interval: 123ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 3.55ms 13.11ms 226.18ms 97.34% + Req/Sec 8.02k 641.34 15.64k 95.66% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.59ms + 75.000% 2.28ms + 90.000% 2.76ms + 99.000% 71.42ms + 99.900% 174.34ms + 99.990% 201.34ms + 99.999% 216.57ms +100.000% 226.30ms + +``` +### async-profiler +#### CPU +Чтение также просело ~20% по производительности. И снова видно, что в разы выросло использование сетевых операций, что в целом, не странно. +По графику видно, что ~70% сэмплов мы находимся на sys call'ах сетевых операций. Локальная же операция теперь занимает около ~1-2% от занятого +процессорного времени. Также стало видно, что one-nio подсчитывает количество запросов, которые сервер обработал (2.4% времени ушло на инкремент счетчика +выполненных запросов - Server::incRequestsProcessed) +Остальное выглядит примерно так же, как и в прошлом этапе. + +#### ALLOC +~90% аллокаций приходится на обработку сетевых операций, и только 5% на локальный get из MemTable/SSTable. В целом, мало что поменялось. +График выглядит очень похоже, почти все аллокации уходят на формирование/чтение заголовков, тела запроса и так далее. + +### Выводы +Реализация репликации привела к существенному увеличению времени, затрачиваемого на сетевые операции. В частности, ожидание ответов от других узлов заняло +около 20% процессорного времени при выполнении PUT и GET запросов. Это указывает на то, что сетевое взаимодействие между нодами является существенной нагрузкой +и может быть узким местом в архитектуре. Увеличение числа HTTP запросов также привело к более активной работе GC, что заняло до 5.5% процессорного времени. +Это в два раза больше, чем на предыдущем этапе, что может указывать на увеличение объема кратковременных объектов в памяти, требующих сборки. +С помощью реплицирования мы повысили надежность системы, но ухудшили производительность. + +Кажется, что для существенного улучшения производительности можно сжимать данные, так как основная проблема - долгие сетевые операции. Следовательно, +чем меньше данных передаем - тем быстрее выполняем запросы. + +##### Результаты +Посмотреть результаты async-profiler: src/main/java/ru/vk/itmo/test/asvistukhin/reports/async_profiler/lab4 +Скрипты для генерации wrk запросов: src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script/get.lua b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script/get.lua index c624d9c10..950101e95 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script/get.lua +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script/get.lua @@ -1,7 +1,7 @@ counter = 0 function request() - path = "/v0/entity?id=" .. counter + path = "/v0/entity?id=" .. counter .. "&ack=2&from=3" wrk.method = "GET" counter = counter + 1 return wrk.format(nil, path) diff --git a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script/put.lua b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script/put.lua index e170247af..f5df3d017 100644 --- a/src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script/put.lua +++ b/src/main/java/ru/vk/itmo/test/asvistukhin/reports/wrk_script/put.lua @@ -1,7 +1,7 @@ counter = 0 function request() - path = "/v0/entity?id=" .. counter + path = "/v0/entity?id=" .. counter .. "&ack=2&from=3" wrk.method = "PUT" wrk.body = randomString() counter = counter + 1 diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/RequestProcessingState.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/RequestProcessingState.java new file mode 100644 index 000000000..13a98baf3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/RequestProcessingState.java @@ -0,0 +1,31 @@ +package ru.vk.itmo.test.bandurinvladislav; + +import one.nio.http.Response; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +public class RequestProcessingState { + final List responses; + final AtomicBoolean responseSent = new AtomicBoolean(false); + final AtomicLong failedResponseCount = new AtomicLong(0); + + public RequestProcessingState(int capacity) { + this.responses = Collections.synchronizedList(new ArrayList<>(capacity)); + } + + public List getResponses() { + return responses; + } + + public AtomicBoolean isResponseSent() { + return responseSent; + } + + public AtomicLong getFailedResponseCount() { + return failedResponseCount; + } +} diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/Server.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/Server.java index 9a587d5bd..751d12307 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/Server.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/Server.java @@ -1,44 +1,70 @@ package ru.vk.itmo.test.bandurinvladislav; +import one.nio.http.HttpClient; import one.nio.http.HttpServer; -import one.nio.http.HttpServerConfig; import one.nio.http.HttpSession; import one.nio.http.Param; import one.nio.http.Request; import one.nio.http.Response; +import one.nio.net.ConnectionString; +import one.nio.util.Hash; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ru.vk.itmo.dao.BaseEntry; -import ru.vk.itmo.dao.Config; -import ru.vk.itmo.dao.Entry; +import ru.vk.itmo.test.bandurinvladislav.config.DhtServerConfig; +import ru.vk.itmo.test.bandurinvladislav.dao.BaseEntry; +import ru.vk.itmo.test.bandurinvladislav.dao.Config; +import ru.vk.itmo.test.bandurinvladislav.dao.Entry; import ru.vk.itmo.test.bandurinvladislav.dao.ReferenceDao; import ru.vk.itmo.test.bandurinvladislav.util.Constants; import ru.vk.itmo.test.bandurinvladislav.util.MemSegUtil; -import ru.vk.itmo.test.bandurinvladislav.util.StringUtil; +import ru.vk.itmo.test.bandurinvladislav.util.NetworkUtil; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; public class Server extends HttpServer { private static final Logger logger = LoggerFactory.getLogger(Server.class); + private final Map clients; + private final int clusterSize; private final DaoWorkerPool workerPool; + private final DhtServerConfig serverConfig; + private final ReferenceDao dao; - public Server(HttpServerConfig serverConfig, java.nio.file.Path workingDir) throws IOException { - super(serverConfig); + public Server(DhtServerConfig config, java.nio.file.Path workingDir) throws IOException { + super(config); workerPool = new DaoWorkerPool( - Constants.THREADS, - Constants.THREADS, - 60, + Constants.ACCEPTOR_THREADS, + Constants.ACCEPTOR_THREADS, + Constants.THREAD_KEEP_ALIVE_TIME, TimeUnit.SECONDS ); + + serverConfig = config; workerPool.prestartAllCoreThreads(); Config daoConfig = new Config(workingDir, Constants.FLUSH_THRESHOLD_BYTES); dao = new ReferenceDao(daoConfig); + clients = new HashMap<>(); + clusterSize = serverConfig.clusterUrls.size(); + config.clusterUrls.stream() + .filter(url -> !url.equals(config.selfUrl)) + .forEach(u -> { + HttpClient client = new HttpClient(new ConnectionString(u)); + client.setTimeout(Constants.CLIENT_RESPONSE_TIMEOUT_MILLIS); + clients.put(u, client); + }); logger.info("Server started"); } @@ -47,14 +73,22 @@ public Response getEntity(@Param(value = "id", required = true) String id) { if (result == null) { return new Response(Response.NOT_FOUND, Response.EMPTY); } - return Response.ok(result.value().toArray(ValueLayout.JAVA_BYTE)); + Response response; + if (result.value() == null) { + response = new Response(Response.NOT_FOUND, Response.EMPTY); + } else { + response = Response.ok(result.value().toArray(ValueLayout.JAVA_BYTE)); + } + response.addHeader(Constants.HEADER_TIMESTAMP + result.timestamp()); + return response; } public Response putEntity(@Param(value = "id", required = true) String id, Request request) { dao.upsert( new BaseEntry<>( MemSegUtil.fromString(id), - MemorySegment.ofArray(request.getBody()) + MemorySegment.ofArray(request.getBody()), + System.currentTimeMillis() ) ); return new Response(Response.CREATED, Response.EMPTY); @@ -64,7 +98,8 @@ public Response deleteEntity(@Param(value = "id", required = true) String id) { dao.upsert( new BaseEntry<>( MemSegUtil.fromString(id), - null + null, + System.currentTimeMillis() ) ); return new Response(Response.ACCEPTED, Response.EMPTY); @@ -96,6 +131,7 @@ public void handleRequest(Request request, HttpSession session) throws IOExcepti } } + @SuppressWarnings("unused") private void handleDaoCall(Request request, HttpSession session) throws IOException { String path = request.getPath(); if (!path.equals(Constants.ENDPOINT)) { @@ -103,12 +139,55 @@ private void handleDaoCall(Request request, HttpSession session) throws IOExcept return; } + if (!NetworkUtil.isMethodAllowed(request)) { + session.sendResponse(new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY)); + return; + } + + int ack = NetworkUtil.getParameterAsInt(request.getParameter(Constants.PARAMETER_ACK), + clusterSize / 2 + clusterSize % 2); + int from = NetworkUtil.getParameterAsInt( + request.getParameter(Constants.PARAMETER_FROM), clusterSize); String key = request.getParameter(Constants.PARAMETER_ID); - if (StringUtil.isEmpty(key)) { - session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); + + Response validationResponse = NetworkUtil.validateParams(key, ack, from, clusterSize); + if (validationResponse != null) { + session.sendResponse(validationResponse); + return; + } + + String sender = request.getHeader(Constants.HEADER_SENDER); + if (sender != null) { + session.sendResponse(invokeLocal(request, key)); return; + } else { + request.addHeader(Constants.HEADER_SENDER + serverConfig.clusterUrls); + } + + List sortedClients = getClientsByKey(key, from); + RequestProcessingState rs = new RequestProcessingState(from); + for (HttpClient client : sortedClients) { + if (client == null) { + Response response = invokeLocal(request, key); + NetworkUtil.handleResponse(session, rs, response, ack, from); + continue; + } + CompletableFuture remote = CompletableFuture.supplyAsync( + () -> invokeRemote(request, client), workerPool) + .completeOnTimeout( + new Response(Response.GATEWAY_TIMEOUT, Response.EMPTY), 100, TimeUnit.MILLISECONDS); + CompletableFuture responseAction = remote.thenAccept(r -> { + if (r.getStatus() == HttpURLConnection.HTTP_INTERNAL_ERROR + || r.getStatus() == HttpURLConnection.HTTP_GATEWAY_TIMEOUT) { + NetworkUtil.handleTimeout(session, rs, ack, from); + } else { + NetworkUtil.handleResponse(session, rs, r, ack, from); + } + }); } + } + private Response invokeLocal(Request request, String key) { Response response; try { response = switch (request.getMethod()) { @@ -122,7 +201,35 @@ private void handleDaoCall(Request request, HttpSession session) throws IOExcept response = new Response(Response.INTERNAL_ERROR, Response.EMPTY); } - session.sendResponse(response); + return response; + } + + private static Response invokeRemote(Request request, HttpClient client) { + Response response; + try { + response = client.invoke(request); + } catch (Exception e) { + logger.error("Internal error during request handling: " + e.getMessage()); + response = new Response(Response.INTERNAL_ERROR, Response.EMPTY); + } + return response; + } + + private List getClientsByKey(String key, int from) { + TreeMap hashUrlSorted = new TreeMap<>(); + + for (int i = 0; i < clusterSize; i++) { + String currentNodeUrl = serverConfig.clusterUrls.get(i); + int currentHash = Hash.murmur3(currentNodeUrl + key); + hashUrlSorted.put(currentHash, currentNodeUrl); + } + + List res = new ArrayList<>(); + Iterator iterator = hashUrlSorted.values().iterator(); + while (from-- > 0) { + res.add(clients.get(iterator.next())); + } + return res; } @Override diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/ServerLauncher.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/ServerLauncher.java index 8c2bed5ba..371855918 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/ServerLauncher.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/ServerLauncher.java @@ -1,25 +1,48 @@ package ru.vk.itmo.test.bandurinvladislav; -import one.nio.http.HttpServerConfig; import ru.vk.itmo.ServiceConfig; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; public final class ServerLauncher { private ServerLauncher() { } - public static void main(String[] args) throws IOException { - ServiceConfig serviceConfig = new ServiceConfig( - 8080, "http://localhost", List.of("http://localhost"), - Path.of("/home/vbandurin/github/tmp/db")); - HttpServerConfig serverConfig = ServiceImpl.createServerConfig(serviceConfig); - Server server = new Server(serverConfig, serviceConfig.workingDir()); - server.start(); + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException, TimeoutException { + int[] nodePorts = new int[]{8080, 8090, 8100}; + Map nodes = new HashMap<>(); - Runtime.getRuntime().addShutdownHook(new Thread(server::stop)); + for (int nodePort : nodePorts) { + nodes.put(nodePort, "http://localhost:" + nodePort); + } + + List clusterUrls = new ArrayList<>(nodes.values()); + List clusterConfs = new ArrayList<>(); + for (Map.Entry entry : nodes.entrySet()) { + int port = entry.getKey(); + String url = entry.getValue(); + Path path = Files.createTempDirectory("tmp-db-" + port); + ServiceConfig serviceConfig = new ServiceConfig(port, + url, + clusterUrls, + path); + clusterConfs.add(serviceConfig); + } + + for (ServiceConfig serviceConfig : clusterConfs) { + ServiceImpl instance = new ServiceImpl(serviceConfig); + instance.start().get(1, TimeUnit.SECONDS); + } } } diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/ServiceImpl.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/ServiceImpl.java index 05838672d..dc74a4c0d 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/ServiceImpl.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/ServiceImpl.java @@ -1,17 +1,17 @@ package ru.vk.itmo.test.bandurinvladislav; -import one.nio.http.HttpServerConfig; import one.nio.server.AcceptorConfig; import ru.vk.itmo.Service; import ru.vk.itmo.ServiceConfig; import ru.vk.itmo.test.ServiceFactory; +import ru.vk.itmo.test.bandurinvladislav.config.DhtServerConfig; import java.io.IOException; import java.io.UncheckedIOException; import java.util.concurrent.CompletableFuture; public class ServiceImpl implements Service { - private final HttpServerConfig serverConfig; + private final DhtServerConfig serverConfig; private final ServiceConfig config; private Server server; @@ -37,18 +37,20 @@ public CompletableFuture stop() { return CompletableFuture.completedFuture(null); } - public static HttpServerConfig createServerConfig(ServiceConfig serviceConfig) { - HttpServerConfig serverConfig = new HttpServerConfig(); + public static DhtServerConfig createServerConfig(ServiceConfig serviceConfig) { + DhtServerConfig serverConfig = new DhtServerConfig(); AcceptorConfig acceptorConfig = new AcceptorConfig(); acceptorConfig.port = serviceConfig.selfPort(); acceptorConfig.reusePort = true; serverConfig.acceptors = new AcceptorConfig[]{acceptorConfig}; serverConfig.closeSessions = true; + serverConfig.clusterUrls = serviceConfig.clusterUrls(); + serverConfig.selfUrl = serviceConfig.selfUrl(); return serverConfig; } - @ServiceFactory(stage = 1) + @ServiceFactory(stage = 5) public static class Factory implements ServiceFactory.Factory { @Override public Service create(ServiceConfig config) { diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/config/DhtServerConfig.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/config/DhtServerConfig.java new file mode 100644 index 000000000..09529eb5c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/config/DhtServerConfig.java @@ -0,0 +1,10 @@ +package ru.vk.itmo.test.bandurinvladislav.config; + +import one.nio.http.HttpServerConfig; + +import java.util.List; + +public class DhtServerConfig extends HttpServerConfig { + public List clusterUrls; + public String selfUrl; +} diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/BaseEntry.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/BaseEntry.java new file mode 100644 index 000000000..1f0bbcc7f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/BaseEntry.java @@ -0,0 +1,8 @@ +package ru.vk.itmo.test.bandurinvladislav.dao; + +public record BaseEntry(D key, D value, long timestamp) implements Entry { + @Override + public String toString() { + return "{" + key + ":" + value + "}"; + } +} diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Config.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Config.java new file mode 100644 index 000000000..7029ff5eb --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Config.java @@ -0,0 +1,8 @@ +package ru.vk.itmo.test.bandurinvladislav.dao; + +import java.nio.file.Path; + +public record Config( + Path basePath, + long flushThresholdBytes) { +} diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Dao.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Dao.java new file mode 100644 index 000000000..6573c58c9 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Dao.java @@ -0,0 +1,87 @@ +package ru.vk.itmo.test.bandurinvladislav.dao; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; + +public interface Dao> extends Closeable { + /** + * Returns ordered iterator of entries with keys between from (inclusive) and to (exclusive). + * @param from lower bound of range (inclusive) + * @param to upper bound of range (exclusive) + * @return entries [from;to) + */ + Iterator get(D from, D to); + + /** + * Returns entry by key. Note: default implementation is far from optimal. + * @param key entry`s key + * @return entry + */ + default E get(D key) { + Iterator iterator = get(key, null); + if (!iterator.hasNext()) { + return null; + } + + E next = iterator.next(); + if (next.key().equals(key)) { + return next; + } + return null; + } + + /** + * Returns ordered iterator of all entries with keys from (inclusive). + * @param from lower bound of range (inclusive) + * @return entries with key >= from + */ + default Iterator allFrom(D from) { + return get(from, null); + } + + /** + * Returns ordered iterator of all entries with keys < to. + * @param to upper bound of range (exclusive) + * @return entries with key < to + */ + default Iterator allTo(D to) { + return get(null, to); + } + + /** + * Returns ordered iterator of all entries. + * @return all entries + */ + default Iterator all() { + return get(null, null); + } + + /** + * Inserts of replaces entry. + * @param entry element to upsert + */ + void upsert(E entry); + + /** + * Persists data (no-op by default). + */ + default void flush() throws IOException { + // Do nothing + } + + /** + * Compacts data (no-op by default). + */ + default void compact() throws IOException { + // Do nothing + } + + /* + * Releases Dao (calls flush by default). + */ + @Override + default void close() throws IOException { + flush(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Entry.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Entry.java new file mode 100644 index 000000000..f920f8923 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/Entry.java @@ -0,0 +1,9 @@ +package ru.vk.itmo.test.bandurinvladislav.dao; + +public interface Entry { + D key(); + + D value(); + + long timestamp(); +} diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/LiveFilteringIterator.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/LiveFilteringIterator.java index 6d240efbf..fcd912663 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/LiveFilteringIterator.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/LiveFilteringIterator.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.bandurinvladislav.dao; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.util.Iterator; import java.util.NoSuchElementException; diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/MemTable.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/MemTable.java index 4f857b3b7..51bb2b926 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/MemTable.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/MemTable.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.bandurinvladislav.dao; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.util.Iterator; import java.util.NavigableMap; diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/MergingEntryIterator.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/MergingEntryIterator.java index 6b18b0bd0..3977863d2 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/MergingEntryIterator.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/MergingEntryIterator.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.bandurinvladislav.dao; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.util.Iterator; import java.util.List; diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/ReferenceDao.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/ReferenceDao.java index de1bbca4b..59543b61e 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/ReferenceDao.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/ReferenceDao.java @@ -1,9 +1,5 @@ package ru.vk.itmo.test.bandurinvladislav.dao; -import ru.vk.itmo.dao.Config; -import ru.vk.itmo.dao.Dao; -import ru.vk.itmo.dao.Entry; - import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -109,10 +105,10 @@ private static long sizeOf(final Entry entry) { } if (entry.value() == null) { - return entry.key().byteSize(); + return entry.key().byteSize() + Long.BYTES; } - return entry.key().byteSize() + entry.value().byteSize(); + return entry.key().byteSize() + entry.value().byteSize() + Long.BYTES; } private void initiateFlush(final boolean auto) { diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/SSTable.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/SSTable.java index cebfa71b4..03232ae7d 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/SSTable.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/SSTable.java @@ -1,8 +1,5 @@ package ru.vk.itmo.test.bandurinvladislav.dao; -import ru.vk.itmo.dao.BaseEntry; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.util.Collections; @@ -54,7 +51,7 @@ private long entryBinarySearch(final MemorySegment key) { while (low <= high) { final long mid = (low + high) >>> 1; final long midEntryOffset = entryOffset(mid); - final long midKeyLength = getLength(midEntryOffset); + final long midKeyLength = getLong(midEntryOffset); final int compare = MemorySegmentComparator.compare( data, @@ -82,7 +79,7 @@ private long entryOffset(final long entry) { entry * Long.BYTES); } - private long getLength(final long offset) { + private long getLong(final long offset) { return data.get( ValueLayout.OfLong.JAVA_LONG_UNALIGNED, offset); @@ -144,15 +141,19 @@ Entry get(final MemorySegment key) { long offset = entryOffset(entry); offset += Long.BYTES + key.byteSize(); // Extract value length - final long valueLength = getLength(offset); + final long valueLength = getLong(offset); if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { // Tombstone encountered - return new BaseEntry<>(key, null); + offset += Long.BYTES; + long timestamp = getLong(offset); + return new BaseEntry<>(key, null, timestamp); } else { // Get value offset += Long.BYTES; final MemorySegment value = data.asSlice(offset, valueLength); - return new BaseEntry<>(key, value); + offset += valueLength; + long timestamp = getLong(offset); + return new BaseEntry<>(key, value, timestamp); } } @@ -179,7 +180,7 @@ public Entry next() { } // Read key length - final long keyLength = getLength(offset); + final long keyLength = getLong(offset); offset += Long.BYTES; // Read key @@ -187,17 +188,19 @@ public Entry next() { offset += keyLength; // Read value length - final long valueLength = getLength(offset); + final long valueLength = getLong(offset); offset += Long.BYTES; // Read value if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { // Tombstone encountered - return new BaseEntry<>(key, null); + long timestamp = getLong(offset); + return new BaseEntry<>(key, null, timestamp); } else { final MemorySegment value = data.asSlice(offset, valueLength); offset += valueLength; - return new BaseEntry<>(key, value); + long timestamp = getLong(offset); + return new BaseEntry<>(key, value, timestamp); } } } diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/SSTableWriter.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/SSTableWriter.java index a76c6e680..c0392d972 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/SSTableWriter.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/SSTableWriter.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.bandurinvladislav.dao; -import ru.vk.itmo.dao.Entry; - import java.io.BufferedOutputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -136,6 +134,7 @@ private long writeEntry( final OutputStream os) throws IOException { final MemorySegment key = entry.key(); final MemorySegment value = entry.value(); + final long timestamp = entry.timestamp(); long result = 0L; // Key size @@ -161,6 +160,9 @@ private long writeEntry( result += value.byteSize(); } + writeLong(timestamp, os); + result += Long.BYTES; + return result; } } diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/TableSet.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/TableSet.java index 451348c78..e43b3a56a 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/TableSet.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/TableSet.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.bandurinvladislav.dao; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.util.ArrayList; import java.util.Collections; @@ -153,7 +151,7 @@ Entry get(final MemorySegment key) { Entry result = memTable.get(key); if (result != null) { // Transform tombstone - return swallowTombstone(result); + return result; } // Then check flushing @@ -161,7 +159,7 @@ Entry get(final MemorySegment key) { result = flushingTable.get(key); if (result != null) { // Transform tombstone - return swallowTombstone(result); + return result; } } @@ -170,7 +168,7 @@ Entry get(final MemorySegment key) { result = ssTable.get(key); if (result != null) { // Transform tombstone - return swallowTombstone(result); + return result; } } @@ -178,10 +176,6 @@ Entry get(final MemorySegment key) { return null; } - private static Entry swallowTombstone(final Entry entry) { - return entry.value() == null ? null : entry; - } - Entry upsert(final Entry entry) { return memTable.upsert(entry); } diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/WeightedPeekingEntryIterator.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/WeightedPeekingEntryIterator.java index 360d88257..82aa49680 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/WeightedPeekingEntryIterator.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/dao/WeightedPeekingEntryIterator.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.bandurinvladislav.dao; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.util.Iterator; import java.util.NoSuchElementException; diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getAlloc.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getAlloc.html new file mode 100644 index 000000000..636db5e5a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getAlloc.html @@ -0,0 +1,809 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getCpu.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getCpu.html new file mode 100644 index 000000000..b2071219b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getCpu.html @@ -0,0 +1,3099 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getLock.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getLock.html new file mode 100644 index 000000000..7d9ff3b44 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/getLock.html @@ -0,0 +1,465 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putAlloc.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putAlloc.html new file mode 100644 index 000000000..384441cb1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putAlloc.html @@ -0,0 +1,829 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putCpu.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putCpu.html new file mode 100644 index 000000000..2b91b4590 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putCpu.html @@ -0,0 +1,3442 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putLock.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putLock.html new file mode 100644 index 000000000..061234733 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_html/putLock.html @@ -0,0 +1,491 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/get_cpu.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/get_cpu.png new file mode 100644 index 000000000..62c16f028 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/get_cpu.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/get_lock.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/get_lock.png new file mode 100644 index 000000000..e9da4388d Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/get_lock.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/threads.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/threads.png new file mode 100644 index 000000000..122c74a88 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_png/threads.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_wrk/get_35k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_wrk/get_35k new file mode 100644 index 000000000..d4f6e45c2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_wrk/get_35k @@ -0,0 +1,122 @@ + Latency 0.88ms 430.34us 9.33ms 65.35% + Req/Sec 4.38k 9.79 4.42k 75.66% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 0.86ms + 75.000% 1.20ms + 90.000% 1.41ms + 99.000% 1.79ms + 99.900% 2.93ms + 99.990% 7.49ms + 99.999% 8.68ms +100.000% 9.34ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.068 0.000000 1 1.00 + 0.346 0.100000 70219 1.11 + 0.472 0.200000 140126 1.25 + 0.594 0.300000 209723 1.43 + 0.722 0.400000 279868 1.67 + 0.855 0.500000 349329 2.00 + 0.925 0.550000 384599 2.22 + 0.996 0.600000 419596 2.50 + 1.065 0.650000 454215 2.86 + 1.132 0.700000 489151 3.33 + 1.198 0.750000 524168 4.00 + 1.231 0.775000 541793 4.44 + 1.264 0.800000 559132 5.00 + 1.298 0.825000 576773 5.71 + 1.333 0.850000 594102 6.67 + 1.371 0.875000 611637 8.00 + 1.391 0.887500 620270 8.89 + 1.413 0.900000 629080 10.00 + 1.436 0.912500 637493 11.43 + 1.463 0.925000 646405 13.33 + 1.493 0.937500 654966 16.00 + 1.510 0.943750 659306 17.78 + 1.529 0.950000 663699 20.00 + 1.551 0.956250 668128 22.86 + 1.575 0.962500 672460 26.67 + 1.604 0.968750 676841 32.00 + 1.620 0.971875 678961 35.56 + 1.639 0.975000 681135 40.00 + 1.661 0.978125 683363 45.71 + 1.686 0.981250 685551 53.33 + 1.715 0.984375 687668 64.00 + 1.733 0.985938 688753 71.11 + 1.753 0.987500 689885 80.00 + 1.776 0.989062 690957 91.43 + 1.804 0.990625 692069 106.67 + 1.840 0.992188 693137 128.00 + 1.860 0.992969 693675 142.22 + 1.884 0.993750 694218 160.00 + 1.912 0.994531 694772 182.86 + 1.945 0.995313 695308 213.33 + 1.986 0.996094 695847 256.00 + 2.013 0.996484 696119 284.44 + 2.041 0.996875 696398 320.00 + 2.077 0.997266 696665 365.71 + 2.119 0.997656 696942 426.67 + 2.177 0.998047 697213 512.00 + 2.217 0.998242 697348 568.89 + 2.277 0.998437 697486 640.00 + 2.367 0.998633 697617 731.43 + 2.557 0.998828 697755 853.33 + 2.987 0.999023 697890 1024.00 + 3.309 0.999121 697959 1137.78 + 3.681 0.999219 698029 1280.00 + 3.971 0.999316 698095 1462.86 + 4.419 0.999414 698163 1706.67 + 4.927 0.999512 698231 2048.00 + 5.227 0.999561 698266 2275.56 + 5.515 0.999609 698300 2560.00 + 5.815 0.999658 698334 2925.71 + 6.091 0.999707 698369 3413.33 + 6.403 0.999756 698402 4096.00 + 6.579 0.999780 698419 4551.11 + 6.775 0.999805 698436 5120.00 + 6.955 0.999829 698453 5851.43 + 7.143 0.999854 698470 6826.67 + 7.267 0.999878 698488 8192.00 + 7.387 0.999890 698496 9102.22 + 7.515 0.999902 698504 10240.00 + 7.643 0.999915 698513 11702.86 + 7.711 0.999927 698521 13653.33 + 7.867 0.999939 698530 16384.00 + 7.903 0.999945 698534 18204.44 + 7.939 0.999951 698538 20480.00 + 8.059 0.999957 698543 23405.71 + 8.139 0.999963 698547 27306.67 + 8.279 0.999969 698551 32768.00 + 8.351 0.999973 698553 36408.89 + 8.383 0.999976 698555 40960.00 + 8.511 0.999979 698559 46811.43 + 8.519 0.999982 698561 54613.33 + 8.527 0.999985 698562 65536.00 + 8.615 0.999986 698563 72817.78 + 8.631 0.999988 698564 81920.00 + 8.679 0.999989 698565 93622.86 + 8.727 0.999991 698566 109226.67 + 8.735 0.999992 698567 131072.00 + 8.775 0.999993 698568 145635.56 + 8.775 0.999994 698568 163840.00 + 8.895 0.999995 698569 187245.71 + 8.895 0.999995 698569 218453.33 + 8.951 0.999996 698570 262144.00 + 8.951 0.999997 698570 291271.11 + 8.951 0.999997 698570 327680.00 + 8.959 0.999997 698571 374491.43 + 8.959 0.999998 698571 436906.67 + 8.959 0.999998 698571 524288.00 + 8.959 0.999998 698571 582542.22 + 8.959 0.999998 698571 655360.00 + 9.335 0.999999 698572 748982.86 + 9.335 1.000000 698572 inf +#[Mean = 0.878, StdDeviation = 0.430] +#[Max = 9.328, Total count = 698572] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1049337 requests in 30.00s, 137.37MB read +Requests/sec: 34978.44 +Transfer/sec: 4.58MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_wrk/put_45k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_wrk/put_45k new file mode 100644 index 000000000..b9b4fe849 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/profile_wrk/put_45k @@ -0,0 +1,125 @@ + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.45ms 1.08ms 29.54ms 90.59% + Req/Sec 5.93k 654.89 9.89k 85.12% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.27ms + 75.000% 1.74ms + 90.000% 2.23ms + 99.000% 6.08ms + 99.900% 12.18ms + 99.990% 19.02ms + 99.999% 26.86ms +100.000% 29.55ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.056 0.000000 1 1.00 + 0.567 0.100000 89920 1.11 + 0.776 0.200000 179947 1.25 + 0.948 0.300000 269868 1.43 + 1.108 0.400000 359623 1.67 + 1.267 0.500000 449183 2.00 + 1.351 0.550000 494095 2.22 + 1.439 0.600000 539029 2.50 + 1.533 0.650000 583926 2.86 + 1.634 0.700000 629119 3.33 + 1.744 0.750000 673921 4.00 + 1.804 0.775000 696292 4.44 + 1.869 0.800000 718821 5.00 + 1.941 0.825000 741198 5.71 + 2.022 0.850000 763633 6.67 + 2.115 0.875000 785957 8.00 + 2.171 0.887500 797525 8.89 + 2.231 0.900000 808481 10.00 + 2.303 0.912500 819732 11.43 + 2.391 0.925000 831023 13.33 + 2.507 0.937500 842122 16.00 + 2.585 0.943750 847697 17.78 + 2.695 0.950000 853329 20.00 + 2.859 0.956250 858931 22.86 + 3.199 0.962500 864532 26.67 + 3.811 0.968750 870141 32.00 + 4.139 0.971875 872946 35.56 + 4.463 0.975000 875763 40.00 + 4.791 0.978125 878587 45.71 + 5.111 0.981250 881373 53.33 + 5.439 0.984375 884173 64.00 + 5.611 0.985938 885606 71.11 + 5.779 0.987500 887001 80.00 + 5.963 0.989062 888387 91.43 + 6.163 0.990625 889788 106.67 + 6.379 0.992188 891202 128.00 + 6.499 0.992969 891900 142.22 + 6.631 0.993750 892602 160.00 + 6.783 0.994531 893300 182.86 + 6.979 0.995313 893995 213.33 + 7.231 0.996094 894697 256.00 + 7.407 0.996484 895046 284.44 + 7.667 0.996875 895392 320.00 + 8.263 0.997266 895742 365.71 + 9.791 0.997656 896092 426.67 + 10.791 0.998047 896445 512.00 + 11.111 0.998242 896620 568.89 + 11.447 0.998437 896795 640.00 + 11.679 0.998633 896974 731.43 + 11.919 0.998828 897145 853.33 + 12.199 0.999023 897320 1024.00 + 12.367 0.999121 897415 1137.78 + 12.503 0.999219 897496 1280.00 + 12.663 0.999316 897585 1462.86 + 12.839 0.999414 897673 1706.67 + 13.039 0.999512 897763 2048.00 + 13.183 0.999561 897805 2275.56 + 13.415 0.999609 897847 2560.00 + 13.727 0.999658 897892 2925.71 + 14.095 0.999707 897935 3413.33 + 15.015 0.999756 897979 4096.00 + 15.367 0.999780 898000 4551.11 + 16.127 0.999805 898022 5120.00 + 16.751 0.999829 898045 5851.43 + 17.423 0.999854 898066 6826.67 + 18.303 0.999878 898088 8192.00 + 18.703 0.999890 898099 9102.22 + 19.103 0.999902 898110 10240.00 + 19.807 0.999915 898121 11702.86 + 20.591 0.999927 898132 13653.33 + 22.191 0.999939 898143 16384.00 + 23.055 0.999945 898148 18204.44 + 23.823 0.999951 898154 20480.00 + 24.335 0.999957 898159 23405.71 + 24.847 0.999963 898165 27306.67 + 25.167 0.999969 898170 32768.00 + 25.247 0.999973 898173 36408.89 + 25.615 0.999976 898176 40960.00 + 25.919 0.999979 898178 46811.43 + 26.271 0.999982 898181 54613.33 + 26.447 0.999985 898184 65536.00 + 26.655 0.999986 898185 72817.78 + 26.831 0.999988 898187 81920.00 + 26.863 0.999989 898188 93622.86 + 26.927 0.999991 898189 109226.67 + 27.247 0.999992 898191 131072.00 + 27.247 0.999993 898191 145635.56 + 27.695 0.999994 898192 163840.00 + 28.207 0.999995 898193 187245.71 + 28.207 0.999995 898193 218453.33 + 28.255 0.999996 898194 262144.00 + 28.255 0.999997 898194 291271.11 + 28.639 0.999997 898195 327680.00 + 28.639 0.999997 898195 374491.43 + 28.639 0.999998 898195 436906.67 + 28.911 0.999998 898196 524288.00 + 28.911 0.999998 898196 582542.22 + 28.911 0.999998 898196 655360.00 + 28.911 0.999999 898196 748982.86 + 28.911 0.999999 898196 873813.33 + 29.551 0.999999 898197 1048576.00 + 29.551 1.000000 898197 inf +#[Mean = 1.447, StdDeviation = 1.085] +#[Max = 29.536, Total count = 898197] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1349179 requests in 30.00s, 117.08MB read +Requests/sec: 44973.05 +Transfer/sec: 3.90MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/report.md b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/report.md new file mode 100644 index 000000000..edb421c79 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage-5/report.md @@ -0,0 +1,44 @@ +# Stage-5 + +Задача этого этапа - сделать взаимодействие между нодами асинхронным. + +### Пропусная способность + +После нагрузки сервера с помощью _wrk_, я получил следующие результаты: + +- [get_35k](profile_wrk%2Fget_35k) +- [put_45k](profile_wrk%2Fput_45k) + +И у Get, и у Put `RPS` уменьшился на `5000` по сравнению с прошлым этапом. Нагрузку с большим количеством запросов +в секунду сервер не выдерживал. \ +Сначала мне показалось это немного странным, поэтому я открыл _visualvm_ и посмотрел, чем заняты потоки во время нагрузки. \ +![threads.png](profile_png%2Fthreads.png) \ +Видно, что 2/3 потоков, которые приходятся на одно ядро процессора очень много простаивают, пока один берёт работу на +себя. Думаю, что в реальной жизни, когда ноды находятся на разных физических машинах, переход на асинхронное взаимодействие +должен быть быстрее. + +## Что же изменилось на профиле? + +### Аллокации +Get и Put - Появились новые аллокации, связанные с добавление `CompletableFuture`, суммарно они занимают около 3% +от всех аллокаций. Помимо этого есть улучшения в методах _successResponse_ и _getClientsByKey_, т.к. я убрал `SteamAPI` в этих методах. + +### CPU +У обоих методов достаточно много накладных расходов из-за `CompletableFuture`, внутри которых они обрабатывают и +отправляют пришедные ответы. +Кроме того, у Get сильно возросла доля, занимаемая методом `invokeLocal` (20% -> 40%) \ +![get_cpu.png](profile_png%2Fget_cpu.png) \ +Думаю, это связано с тем, что при асинхронном заимодействии, когда мы дублируем выполнение запроса на несколько нод, +получается ситуация, когда они одновременно вызывают `invokeLocal`, от чего он и начал есть больше ресурсов. + +### Lock +Блокировок тоже стало сильно больше: +1) Увеличились количество ожиданий в `ThreadPoolExecutor.getTask`. Кажется, это как раз связано с тем, что ядер на +моём ноутбуке не хватает на все потоки и происходит сильная конкуренция за ресурсы. +2) Добавились новые локи, связанные с работой с `CompletableFuture`. \ +![get_lock.png](profile_png%2Fget_lock.png) + +## Вывод +Переход на асинхронное взаимодействие между нодами увеличил накладные расходы по всем параметрам и показал меньшую +производительность по сравнению с прошлым этапом. Однако для того, чтобы оценить его реальную работоспособность, нужно +проводить тестирование на нескольких физических машинах. diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage1/report.md b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage1/report.md index c9d7b9ddc..4e861fa5e 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage1/report.md +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage1/report.md @@ -12,7 +12,8 @@ Команды для запуска нагрузки: [GET](../../scripts/GetRequest.lua): - - wrk2: `wrk -d 60 -t 1 -c 1 -R 17000 -L -s ./scripts/[GetRequest.lua](./scripts/GetRequest.lua) http://localhost:8080` + - wrk2: `wrk -d 60 -t 1 -c 1 -R 45000 -L -s ./scripts/[GetRequest.lua](./scripts/GetRequest.lua) http://localhost:8080` + [PUT](../../scripts/PutRequest.lua): - wrk2: `wrk -d 60 -t 1 -c 1 -R 45000 -L -s ./scripts/[GetRequest.lua](./scripts/PutRequest.lua) http://localhost:8080` diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getAlloc.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getAlloc.html new file mode 100644 index 000000000..c7e3d7032 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getAlloc.html @@ -0,0 +1,543 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getCpu.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getCpu.html new file mode 100644 index 000000000..ff932b4be --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getCpu.html @@ -0,0 +1,3227 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getLock.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getLock.html new file mode 100644 index 000000000..4c57c0d85 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/getLock.html @@ -0,0 +1,351 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putAlloc.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putAlloc.html new file mode 100644 index 000000000..51e460b0e --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putAlloc.html @@ -0,0 +1,602 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putCpu.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putCpu.html new file mode 100644 index 000000000..0685ff2a2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putCpu.html @@ -0,0 +1,3785 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putLock.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putLock.html new file mode 100644 index 000000000..2daa6ee43 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_html/putLock.html @@ -0,0 +1,364 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/MD5.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/MD5.png new file mode 100644 index 000000000..07189d40e Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/MD5.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/Sha-1.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/Sha-1.png new file mode 100644 index 000000000..c52593dfc Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/Sha-1.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/alloc.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/alloc.png new file mode 100644 index 000000000..a9023599f Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/alloc.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/cpu.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/cpu.png new file mode 100644 index 000000000..070da98a3 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/cpu.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/get_hist.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/get_hist.png new file mode 100644 index 000000000..e6b6b8d5c Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/get_hist.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/heapdump.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/heapdump.png new file mode 100644 index 000000000..9f4c6dda3 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/heapdump.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/murmur3.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/murmur3.png new file mode 100644 index 000000000..89cbdc84d Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/murmur3.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/put_hist.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/put_hist.png new file mode 100644 index 000000000..621ba0ee6 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_png/put_hist.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/get_110k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/get_110k new file mode 100644 index 000000000..4f7e06710 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/get_110k @@ -0,0 +1,131 @@ + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.25ms 0.89ms 19.28ms 83.76% + Req/Sec 14.51k 1.22k 31.78k 72.14% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.15ms + 75.000% 1.62ms + 90.000% 2.07ms + 99.000% 3.22ms + 99.900% 11.20ms + 99.990% 14.65ms + 99.999% 17.33ms +100.000% 19.30ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.023 0.000000 1 1.00 + 0.418 0.100000 220124 1.11 + 0.636 0.200000 439994 1.25 + 0.827 0.300000 659330 1.43 + 0.995 0.400000 879203 1.67 + 1.154 0.500000 1098136 2.00 + 1.236 0.550000 1208677 2.22 + 1.321 0.600000 1317891 2.50 + 1.412 0.650000 1427681 2.86 + 1.512 0.700000 1537424 3.33 + 1.622 0.750000 1646649 4.00 + 1.683 0.775000 1701955 4.44 + 1.747 0.800000 1756454 5.00 + 1.816 0.825000 1811294 5.71 + 1.891 0.850000 1866213 6.67 + 1.975 0.875000 1921242 8.00 + 2.021 0.887500 1948710 8.89 + 2.071 0.900000 1976148 10.00 + 2.127 0.912500 2003733 11.43 + 2.191 0.925000 2031634 13.33 + 2.263 0.937500 2058391 16.00 + 2.305 0.943750 2072347 17.78 + 2.351 0.950000 2086009 20.00 + 2.403 0.956250 2099517 22.86 + 2.463 0.962500 2113177 26.67 + 2.537 0.968750 2126946 32.00 + 2.583 0.971875 2134016 35.56 + 2.631 0.975000 2140658 40.00 + 2.691 0.978125 2147510 45.71 + 2.765 0.981250 2154381 53.33 + 2.861 0.984375 2161216 64.00 + 2.927 0.985938 2164722 71.11 + 3.009 0.987500 2168090 80.00 + 3.123 0.989062 2171533 91.43 + 3.311 0.990625 2174924 106.67 + 3.819 0.992188 2178349 128.00 + 4.379 0.992969 2180071 142.22 + 5.131 0.993750 2181783 160.00 + 6.043 0.994531 2183499 182.86 + 6.927 0.995313 2185212 213.33 + 7.771 0.996094 2186928 256.00 + 8.135 0.996484 2187794 284.44 + 8.503 0.996875 2188646 320.00 + 8.903 0.997266 2189508 365.71 + 9.303 0.997656 2190356 426.67 + 9.775 0.998047 2191216 512.00 + 10.023 0.998242 2191644 568.89 + 10.327 0.998437 2192076 640.00 + 10.607 0.998633 2192508 731.43 + 10.895 0.998828 2192929 853.33 + 11.247 0.999023 2193357 1024.00 + 11.399 0.999121 2193574 1137.78 + 11.591 0.999219 2193787 1280.00 + 11.783 0.999316 2194002 1462.86 + 12.023 0.999414 2194217 1706.67 + 12.303 0.999512 2194434 2048.00 + 12.455 0.999561 2194540 2275.56 + 12.615 0.999609 2194644 2560.00 + 12.887 0.999658 2194751 2925.71 + 13.103 0.999707 2194858 3413.33 + 13.391 0.999756 2194967 4096.00 + 13.527 0.999780 2195019 4551.11 + 13.711 0.999805 2195073 5120.00 + 13.927 0.999829 2195130 5851.43 + 14.135 0.999854 2195180 6826.67 + 14.439 0.999878 2195233 8192.00 + 14.567 0.999890 2195261 9102.22 + 14.671 0.999902 2195287 10240.00 + 14.823 0.999915 2195315 11702.86 + 14.991 0.999927 2195341 13653.33 + 15.311 0.999939 2195367 16384.00 + 15.439 0.999945 2195381 18204.44 + 15.631 0.999951 2195394 20480.00 + 15.839 0.999957 2195409 23405.71 + 15.991 0.999963 2195421 27306.67 + 16.167 0.999969 2195435 32768.00 + 16.287 0.999973 2195441 36408.89 + 16.543 0.999976 2195448 40960.00 + 16.687 0.999979 2195455 46811.43 + 16.799 0.999982 2195462 54613.33 + 17.007 0.999985 2195468 65536.00 + 17.071 0.999986 2195471 72817.78 + 17.215 0.999988 2195476 81920.00 + 17.279 0.999989 2195478 93622.86 + 17.407 0.999991 2195481 109226.67 + 17.583 0.999992 2195485 131072.00 + 17.791 0.999993 2195486 145635.56 + 18.031 0.999994 2195488 163840.00 + 18.111 0.999995 2195491 187245.71 + 18.111 0.999995 2195491 218453.33 + 18.319 0.999996 2195493 262144.00 + 18.399 0.999997 2195494 291271.11 + 18.431 0.999997 2195495 327680.00 + 18.479 0.999997 2195496 374491.43 + 18.479 0.999998 2195496 436906.67 + 18.511 0.999998 2195497 524288.00 + 18.607 0.999998 2195498 582542.22 + 18.607 0.999998 2195498 655360.00 + 18.959 0.999999 2195499 748982.86 + 18.959 0.999999 2195499 873813.33 + 18.959 0.999999 2195499 1048576.00 + 19.151 0.999999 2195500 1165084.44 + 19.151 0.999999 2195500 1310720.00 + 19.151 0.999999 2195500 1497965.71 + 19.151 0.999999 2195500 1747626.67 + 19.151 1.000000 2195500 2097152.00 + 19.295 1.000000 2195501 2330168.89 + 19.295 1.000000 2195501 inf +#[Mean = 1.254, StdDeviation = 0.886] +#[Max = 19.280, Total count = 2195501] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 3297887 requests in 30.00s, 333.35MB read +Requests/sec: 109931.00 +Transfer/sec: 11.11MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/get_110k_stage2 b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/get_110k_stage2 new file mode 100644 index 000000000..5a73268b7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/get_110k_stage2 @@ -0,0 +1,132 @@ + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.81ms 1.46ms 27.18ms 84.54% + Req/Sec 14.53k 2.49k 33.67k 76.75% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.45ms + 75.000% 2.20ms + 90.000% 3.42ms + 99.000% 7.38ms + 99.900% 12.72ms + 99.990% 19.30ms + 99.999% 24.37ms +100.000% 27.20ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.022 0.000000 1 1.00 + 0.531 0.100000 220246 1.11 + 0.800 0.200000 439756 1.25 + 1.023 0.300000 658937 1.43 + 1.231 0.400000 878467 1.67 + 1.452 0.500000 1097958 2.00 + 1.573 0.550000 1207579 2.22 + 1.705 0.600000 1317728 2.50 + 1.849 0.650000 1427418 2.86 + 2.011 0.700000 1537076 3.33 + 2.205 0.750000 1646941 4.00 + 2.323 0.775000 1702097 4.44 + 2.459 0.800000 1756710 5.00 + 2.623 0.825000 1811946 5.71 + 2.823 0.850000 1866298 6.67 + 3.079 0.875000 1921178 8.00 + 3.235 0.887500 1948595 8.89 + 3.417 0.900000 1976092 10.00 + 3.633 0.912500 2003549 11.43 + 3.891 0.925000 2031023 13.33 + 4.203 0.937500 2058508 16.00 + 4.383 0.943750 2072118 17.78 + 4.587 0.950000 2085873 20.00 + 4.819 0.956250 2099609 22.86 + 5.083 0.962500 2113358 26.67 + 5.391 0.968750 2126957 32.00 + 5.567 0.971875 2133846 35.56 + 5.763 0.975000 2140714 40.00 + 5.979 0.978125 2147545 45.71 + 6.243 0.981250 2154459 53.33 + 6.555 0.984375 2161259 64.00 + 6.751 0.985938 2164737 71.11 + 6.963 0.987500 2168123 80.00 + 7.219 0.989062 2171576 91.43 + 7.511 0.990625 2174981 106.67 + 7.879 0.992188 2178414 128.00 + 8.087 0.992969 2180127 142.22 + 8.327 0.993750 2181840 160.00 + 8.599 0.994531 2183583 182.86 + 8.903 0.995313 2185265 213.33 + 9.295 0.996094 2186990 256.00 + 9.519 0.996484 2187851 284.44 + 9.767 0.996875 2188694 320.00 + 10.071 0.997266 2189562 365.71 + 10.423 0.997656 2190421 426.67 + 10.871 0.998047 2191274 512.00 + 11.127 0.998242 2191698 568.89 + 11.455 0.998437 2192131 640.00 + 11.799 0.998633 2192562 731.43 + 12.247 0.998828 2192983 853.33 + 12.767 0.999023 2193415 1024.00 + 13.079 0.999121 2193632 1137.78 + 13.423 0.999219 2193843 1280.00 + 13.839 0.999316 2194057 1462.86 + 14.375 0.999414 2194269 1706.67 + 14.951 0.999512 2194484 2048.00 + 15.327 0.999561 2194594 2275.56 + 15.687 0.999609 2194698 2560.00 + 16.135 0.999658 2194805 2925.71 + 16.687 0.999707 2194913 3413.33 + 17.247 0.999756 2195019 4096.00 + 17.583 0.999780 2195073 4551.11 + 17.935 0.999805 2195129 5120.00 + 18.239 0.999829 2195182 5851.43 + 18.575 0.999854 2195234 6826.67 + 18.991 0.999878 2195287 8192.00 + 19.167 0.999890 2195314 9102.22 + 19.391 0.999902 2195341 10240.00 + 19.599 0.999915 2195368 11702.86 + 19.871 0.999927 2195397 13653.33 + 20.207 0.999939 2195421 16384.00 + 20.431 0.999945 2195436 18204.44 + 20.623 0.999951 2195448 20480.00 + 20.991 0.999957 2195462 23405.71 + 21.407 0.999963 2195475 27306.67 + 21.871 0.999969 2195488 32768.00 + 22.319 0.999973 2195495 36408.89 + 22.607 0.999976 2195502 40960.00 + 22.975 0.999979 2195509 46811.43 + 23.087 0.999982 2195515 54613.33 + 23.487 0.999985 2195522 65536.00 + 23.871 0.999986 2195525 72817.78 + 24.111 0.999988 2195529 81920.00 + 24.351 0.999989 2195532 93622.86 + 24.559 0.999991 2195535 109226.67 + 24.751 0.999992 2195539 131072.00 + 24.815 0.999993 2195540 145635.56 + 24.879 0.999994 2195542 163840.00 + 24.959 0.999995 2195544 187245.71 + 25.023 0.999995 2195545 218453.33 + 25.215 0.999996 2195547 262144.00 + 25.487 0.999997 2195548 291271.11 + 25.663 0.999997 2195549 327680.00 + 25.855 0.999997 2195550 374491.43 + 25.855 0.999998 2195550 436906.67 + 26.239 0.999998 2195551 524288.00 + 26.607 0.999998 2195552 582542.22 + 26.607 0.999998 2195552 655360.00 + 26.943 0.999999 2195553 748982.86 + 26.943 0.999999 2195553 873813.33 + 26.943 0.999999 2195553 1048576.00 + 27.039 0.999999 2195554 1165084.44 + 27.039 0.999999 2195554 1310720.00 + 27.039 0.999999 2195554 1497965.71 + 27.039 0.999999 2195554 1747626.67 + 27.039 1.000000 2195554 2097152.00 + 27.199 1.000000 2195555 2330168.89 + 27.199 1.000000 2195555 inf +#[Mean = 1.807, StdDeviation = 1.462] +#[Max = 27.184, Total count = 2195555] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 3297594 requests in 30.00s, 246.71MB read + Non-2xx or 3xx responses: 1813686 +Requests/sec: 109922.32 +Transfer/sec: 8.22MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/put_75k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/put_75k new file mode 100644 index 000000000..b13735d8a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/put_75k @@ -0,0 +1,128 @@ + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.28ms 1.07ms 24.08ms 91.73% + Req/Sec 9.88k 836.57 16.56k 78.18% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.14ms + 75.000% 1.59ms + 90.000% 2.04ms + 99.000% 5.28ms + 99.900% 13.94ms + 99.990% 19.68ms + 99.999% 22.85ms +100.000% 24.09ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.022 0.000000 1 1.00 + 0.432 0.100000 150142 1.11 + 0.650 0.200000 299782 1.25 + 0.825 0.300000 449933 1.43 + 0.983 0.400000 598935 1.67 + 1.136 0.500000 749141 2.00 + 1.213 0.550000 823968 2.22 + 1.296 0.600000 898460 2.50 + 1.386 0.650000 973523 2.86 + 1.483 0.700000 1048103 3.33 + 1.590 0.750000 1123210 4.00 + 1.647 0.775000 1160467 4.44 + 1.708 0.800000 1197787 5.00 + 1.775 0.825000 1235132 5.71 + 1.851 0.850000 1272748 6.67 + 1.938 0.875000 1310192 8.00 + 1.986 0.887500 1328597 8.89 + 2.040 0.900000 1347350 10.00 + 2.101 0.912500 1366006 11.43 + 2.171 0.925000 1384678 13.33 + 2.259 0.937500 1403610 16.00 + 2.311 0.943750 1412713 17.78 + 2.375 0.950000 1422227 20.00 + 2.453 0.956250 1431602 22.86 + 2.551 0.962500 1440800 26.67 + 2.693 0.968750 1450191 32.00 + 2.789 0.971875 1454834 35.56 + 2.921 0.975000 1459491 40.00 + 3.117 0.978125 1464186 45.71 + 3.427 0.981250 1468850 53.33 + 3.909 0.984375 1473526 64.00 + 4.203 0.985938 1475869 71.11 + 4.555 0.987500 1478203 80.00 + 4.975 0.989062 1480548 91.43 + 5.507 0.990625 1482882 106.67 + 6.163 0.992188 1485216 128.00 + 6.583 0.992969 1486390 142.22 + 7.079 0.993750 1487557 160.00 + 7.643 0.994531 1488725 182.86 + 8.327 0.995313 1489907 213.33 + 9.031 0.996094 1491073 256.00 + 9.455 0.996484 1491656 284.44 + 9.919 0.996875 1492242 320.00 + 10.391 0.997266 1492824 365.71 + 10.887 0.997656 1493402 426.67 + 11.479 0.998047 1493987 512.00 + 11.839 0.998242 1494281 568.89 + 12.223 0.998437 1494572 640.00 + 12.719 0.998633 1494866 731.43 + 13.343 0.998828 1495157 853.33 + 14.023 0.999023 1495450 1024.00 + 14.335 0.999121 1495595 1137.78 + 14.799 0.999219 1495744 1280.00 + 15.159 0.999316 1495888 1462.86 + 15.551 0.999414 1496036 1706.67 + 15.927 0.999512 1496181 2048.00 + 16.183 0.999561 1496254 2275.56 + 16.399 0.999609 1496329 2560.00 + 16.703 0.999658 1496400 2925.71 + 17.167 0.999707 1496472 3413.33 + 17.695 0.999756 1496545 4096.00 + 17.983 0.999780 1496582 4551.11 + 18.255 0.999805 1496619 5120.00 + 18.591 0.999829 1496658 5851.43 + 18.959 0.999854 1496692 6826.67 + 19.375 0.999878 1496728 8192.00 + 19.551 0.999890 1496746 9102.22 + 19.759 0.999902 1496766 10240.00 + 19.935 0.999915 1496783 11702.86 + 20.351 0.999927 1496802 13653.33 + 20.831 0.999939 1496819 16384.00 + 21.135 0.999945 1496828 18204.44 + 21.615 0.999951 1496837 20480.00 + 21.839 0.999957 1496848 23405.71 + 21.919 0.999963 1496856 27306.67 + 22.063 0.999969 1496867 32768.00 + 22.127 0.999973 1496869 36408.89 + 22.255 0.999976 1496874 40960.00 + 22.335 0.999979 1496880 46811.43 + 22.415 0.999982 1496883 54613.33 + 22.511 0.999985 1496888 65536.00 + 22.639 0.999986 1496891 72817.78 + 22.735 0.999988 1496892 81920.00 + 22.847 0.999989 1496895 93622.86 + 22.863 0.999991 1496897 109226.67 + 23.151 0.999992 1496899 131072.00 + 23.247 0.999993 1496901 145635.56 + 23.247 0.999994 1496901 163840.00 + 23.455 0.999995 1496903 187245.71 + 23.471 0.999995 1496904 218453.33 + 23.663 0.999996 1496905 262144.00 + 23.663 0.999997 1496905 291271.11 + 23.727 0.999997 1496906 327680.00 + 23.823 0.999997 1496907 374491.43 + 23.823 0.999998 1496907 436906.67 + 23.887 0.999998 1496908 524288.00 + 23.887 0.999998 1496908 582542.22 + 23.887 0.999998 1496908 655360.00 + 23.999 0.999999 1496909 748982.86 + 23.999 0.999999 1496909 873813.33 + 23.999 0.999999 1496909 1048576.00 + 23.999 0.999999 1496909 1165084.44 + 23.999 0.999999 1496909 1310720.00 + 24.095 0.999999 1496910 1497965.71 + 24.095 1.000000 1496910 inf +#[Mean = 1.277, StdDeviation = 1.069] +#[Max = 24.080, Total count = 1496910] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 2248551 requests in 30.00s, 177.97MB read +Requests/sec: 74954.31 +Transfer/sec: 5.93MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/put_R150k_stage2 b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/put_R150k_stage2 new file mode 100644 index 000000000..aa1720ab9 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/profile_wrk/put_R150k_stage2 @@ -0,0 +1,133 @@ + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.21ms 837.89us 21.70ms 79.85% + Req/Sec 19.78k 1.88k 48.70k 73.10% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.12ms + 75.000% 1.60ms + 90.000% 2.01ms + 99.000% 3.78ms + 99.900% 9.37ms + 99.990% 16.90ms + 99.999% 20.85ms +100.000% 21.71ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.017 0.000000 2 1.00 + 0.340 0.100000 299409 1.11 + 0.576 0.200000 598856 1.25 + 0.799 0.300000 898466 1.43 + 0.972 0.400000 1198011 1.67 + 1.124 0.500000 1497208 2.00 + 1.201 0.550000 1646696 2.22 + 1.288 0.600000 1797844 2.50 + 1.383 0.650000 1946411 2.86 + 1.488 0.700000 2096594 3.33 + 1.604 0.750000 2246031 4.00 + 1.664 0.775000 2321387 4.44 + 1.724 0.800000 2396283 5.00 + 1.787 0.825000 2470938 5.71 + 1.854 0.850000 2544990 6.67 + 1.927 0.875000 2620152 8.00 + 1.966 0.887500 2657502 8.89 + 2.008 0.900000 2694844 10.00 + 2.057 0.912500 2733171 11.43 + 2.113 0.925000 2769981 13.33 + 2.183 0.937500 2807344 16.00 + 2.225 0.943750 2826127 17.78 + 2.273 0.950000 2844829 20.00 + 2.329 0.956250 2863259 22.86 + 2.399 0.962500 2882040 26.67 + 2.489 0.968750 2900515 32.00 + 2.547 0.971875 2909920 35.56 + 2.617 0.975000 2919134 40.00 + 2.711 0.978125 2928533 45.71 + 2.839 0.981250 2937831 53.33 + 3.037 0.984375 2947143 64.00 + 3.183 0.985938 2951841 71.11 + 3.367 0.987500 2956522 80.00 + 3.603 0.989062 2961216 91.43 + 3.909 0.990625 2965879 106.67 + 4.303 0.992188 2970570 128.00 + 4.535 0.992969 2972885 142.22 + 4.799 0.993750 2975211 160.00 + 5.107 0.994531 2977569 182.86 + 5.459 0.995313 2979900 213.33 + 5.883 0.996094 2982237 256.00 + 6.127 0.996484 2983402 284.44 + 6.415 0.996875 2984573 320.00 + 6.739 0.997266 2985736 365.71 + 7.115 0.997656 2986909 426.67 + 7.603 0.998047 2988080 512.00 + 7.875 0.998242 2988668 568.89 + 8.167 0.998437 2989248 640.00 + 8.503 0.998633 2989835 731.43 + 8.903 0.998828 2990417 853.33 + 9.439 0.999023 2991003 1024.00 + 9.759 0.999121 2991291 1137.78 + 10.119 0.999219 2991584 1280.00 + 10.551 0.999316 2991879 1462.86 + 11.015 0.999414 2992168 1706.67 + 11.551 0.999512 2992466 2048.00 + 11.839 0.999561 2992611 2275.56 + 12.183 0.999609 2992759 2560.00 + 12.575 0.999658 2992902 2925.71 + 13.047 0.999707 2993046 3413.33 + 13.599 0.999756 2993192 4096.00 + 13.911 0.999780 2993266 4551.11 + 14.287 0.999805 2993338 5120.00 + 14.743 0.999829 2993412 5851.43 + 15.383 0.999854 2993487 6826.67 + 16.135 0.999878 2993557 8192.00 + 16.575 0.999890 2993595 9102.22 + 16.975 0.999902 2993630 10240.00 + 17.343 0.999915 2993667 11702.86 + 17.695 0.999927 2993703 13653.33 + 18.143 0.999939 2993740 16384.00 + 18.383 0.999945 2993758 18204.44 + 18.623 0.999951 2993776 20480.00 + 18.959 0.999957 2993795 23405.71 + 19.279 0.999963 2993813 27306.67 + 19.647 0.999969 2993831 32768.00 + 19.839 0.999973 2993840 36408.89 + 19.983 0.999976 2993850 40960.00 + 20.207 0.999979 2993860 46811.43 + 20.399 0.999982 2993868 54613.33 + 20.559 0.999985 2993877 65536.00 + 20.671 0.999986 2993881 72817.78 + 20.719 0.999988 2993887 81920.00 + 20.815 0.999989 2993891 93622.86 + 20.959 0.999991 2993895 109226.67 + 21.023 0.999992 2993900 131072.00 + 21.071 0.999993 2993902 145635.56 + 21.103 0.999994 2993904 163840.00 + 21.167 0.999995 2993907 187245.71 + 21.215 0.999995 2993909 218453.33 + 21.231 0.999996 2993911 262144.00 + 21.279 0.999997 2993912 291271.11 + 21.295 0.999997 2993913 327680.00 + 21.327 0.999997 2993915 374491.43 + 21.343 0.999998 2993916 436906.67 + 21.423 0.999998 2993917 524288.00 + 21.423 0.999998 2993917 582542.22 + 21.455 0.999998 2993918 655360.00 + 21.471 0.999999 2993919 748982.86 + 21.471 0.999999 2993919 873813.33 + 21.519 0.999999 2993920 1048576.00 + 21.519 0.999999 2993920 1165084.44 + 21.519 0.999999 2993920 1310720.00 + 21.551 0.999999 2993921 1497965.71 + 21.551 0.999999 2993921 1747626.67 + 21.551 1.000000 2993921 2097152.00 + 21.551 1.000000 2993921 2330168.89 + 21.551 1.000000 2993921 2621440.00 + 21.711 1.000000 2993922 2995931.43 + 21.711 1.000000 2993922 inf +#[Mean = 1.207, StdDeviation = 0.838] +#[Max = 21.696, Total count = 2993922] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 4497062 requests in 30.00s, 287.35MB read +Requests/sec: 149909.28 +Transfer/sec: 9.58MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/report.md b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/report.md new file mode 100644 index 000000000..71d692b08 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage3/report.md @@ -0,0 +1,61 @@ +# Stage 3 + +После реализации партиционирования, для тестов нужно было заполнить базу. При запуске [скрипта](..%2F..%2Fscripts%2FPutRequest.lua) +командой `wrk -d 600 -t 8 -c 64 -R 30000 -L -s ./scripts/PutRequest.lua http://localhost:8080`, через минуту начал вылетать OutOfMemoryError. +Для вычисления проблемы я воспользовался _visualVM_. + +При мониторинге я выяснил, что очень много памяти занимают объекты _BaseEntry_, которые хранятся in-memory, пока их не сбросят на диск. +Так как теперь у нас не один кластер, а несколько, каждый из них ждёт, пока _ConcurrentSkipListMap_ не заполнится => heap раздувается и вылетает ошибка. +![heapdump.png](profile_png%2Fheapdump.png) +Чтобы решить эту проблему, пришлось уменьшить `FLUSH_THRESHOLD_BYTES` до 3 МБ. + +Чтобы объективно оценить изменения нового этапа с новым объёмом БД (~ 300 МБ), перемерю stage-2 и выявлю новую нагрузку, с которой сервер в состоянии справиться. +1) [put_R150k_stage2](profile_wrk%2Fput_R150k_stage2) +2) [get_110k_stage2](profile_wrk%2Fget_110k_stage2) + +### Тестирование с шардированием + +1) [get](profile_wrk%2Fget_110k) - сервер выдерживает ту же нагрузку, что и раньше. +![get_hist.png](profile_png%2Fget_hist.png) + + +2) [put](profile_wrk%2Fput_75k) - производительность упала в 2 раза, т.к. больше, чем `RPS=75_000` сервер не выдерживает. +![put_hist.png](profile_png%2Fput_hist.png) + +#### Аллокации +[getAlloc.html](profile_html%2FgetAlloc.html) \ +[putAlloc.html](profile_html%2FputAlloc.html) \ +В сравнении с прошлыми этапами, большую долю аллокаций теперь составляет выделение _byte[]_ при создании _ResponseReader_. +![alloc.png](profile_png%2Falloc.png) + +#### CPU + +[getCpu.html](profile_html%2FgetCpu.html) \ +[putCpu.html](profile_html%2FputCpu.html) \ +Теперь помимо непосредственного поиска/вставки записи в базу данных, часть цпу отнимает делегирование запросов по сети на другие кластеры +![cpu.png](profile_png%2Fcpu.png) + +#### lock + +[getLock.html](profile_html%2FgetLock.html) - ситуация особо не изменилась относительно прошлого этапа. \ +[putLock.html](profile_html%2FputLock.html) - увеличилась доля блокировок на метод _Session.process_ из-за хождений по сети. + + +### Тестирование равномерности распределения с разными хэш-функциями + +Для сравнения распределений я выбрал murmur3, SHA-1 и MD5, результаты можно увидеть на картинках: + +1) murmur3 \ +![murmur3.png](profile_png%2Fmurmur3.png) + +2) SHA-1 \ +![Sha-1.png](profile_png%2FSha-1.png) + +3) MD5 \ +![MD5.png](profile_png%2FMD5.png) + +Видно, что у этих функций распределение равномерное + +#### Вывод +При добавлении новых кластеров, появилась дополнительная нагрузка в виде хождений по сети, из-за чего общая производительность +нашей системы упала. \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getAlloc.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getAlloc.html new file mode 100644 index 000000000..d821131e9 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getAlloc.html @@ -0,0 +1,772 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getCpu.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getCpu.html new file mode 100644 index 000000000..f22dd7218 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getCpu.html @@ -0,0 +1,3714 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getLock.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getLock.html new file mode 100644 index 000000000..4a7232ff8 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/getLock.html @@ -0,0 +1,347 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putAlloc.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putAlloc.html new file mode 100644 index 000000000..21b212f43 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putAlloc.html @@ -0,0 +1,759 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putCpu.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putCpu.html new file mode 100644 index 000000000..f16a7c561 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putCpu.html @@ -0,0 +1,4055 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putLock.html b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putLock.html new file mode 100644 index 000000000..43fce0d06 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_html/putLock.html @@ -0,0 +1,397 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/alloc_getClients.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/alloc_getClients.png new file mode 100644 index 000000000..03bd302cf Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/alloc_getClients.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/alloc_responses.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/alloc_responses.png new file mode 100644 index 000000000..942d41c45 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/alloc_responses.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/cpu_put.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/cpu_put.png new file mode 100644 index 000000000..9fd11596b Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/cpu_put.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/durability_s3.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/durability_s3.png new file mode 100644 index 000000000..a01a8ac4d Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/durability_s3.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/durability_s4.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/durability_s4.png new file mode 100644 index 000000000..a0cd4a28a Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/durability_s4.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/get_40k.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/get_40k.png new file mode 100644 index 000000000..1768ea77e Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/get_40k.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/put_50k.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/put_50k.png new file mode 100644 index 000000000..1930de98f Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/put_50k.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/put_lock.png b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/put_lock.png new file mode 100644 index 000000000..872204181 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_png/put_lock.png differ diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_40k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_40k new file mode 100644 index 000000000..71e76e037 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_40k @@ -0,0 +1,144 @@ + Thread calibration: mean lat.: 1.471ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.456ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.471ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.463ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.476ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.480ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.454ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.443ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.44ms 1.15ms 30.94ms 91.51% + Req/Sec 5.27k 605.18 11.00k 82.79% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.22ms + 75.000% 1.68ms + 90.000% 2.26ms + 99.000% 6.49ms + 99.900% 11.18ms + 99.990% 19.28ms + 99.999% 26.61ms +100.000% 30.96ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.085 0.000000 1 1.00 + 0.544 0.100000 441656 1.11 + 0.734 0.200000 880238 1.25 + 0.900 0.300000 1321766 1.43 + 1.058 0.400000 1760712 1.67 + 1.218 0.500000 2200297 2.00 + 1.301 0.550000 2421379 2.22 + 1.386 0.600000 2639488 2.50 + 1.477 0.650000 2861107 2.86 + 1.575 0.700000 3080760 3.33 + 1.684 0.750000 3299086 4.00 + 1.746 0.775000 3409465 4.44 + 1.815 0.800000 3519648 5.00 + 1.893 0.825000 3628740 5.71 + 1.985 0.850000 3738868 6.67 + 2.099 0.875000 3849172 8.00 + 2.169 0.887500 3904184 8.89 + 2.255 0.900000 3959531 10.00 + 2.363 0.912500 4013894 11.43 + 2.515 0.925000 4068582 13.33 + 2.779 0.937500 4123713 16.00 + 2.997 0.943750 4151160 17.78 + 3.293 0.950000 4178587 20.00 + 3.629 0.956250 4206038 22.86 + 3.999 0.962500 4233542 26.67 + 4.403 0.968750 4260933 32.00 + 4.623 0.971875 4274758 35.56 + 4.867 0.975000 4288574 40.00 + 5.127 0.978125 4302210 45.71 + 5.419 0.981250 4316033 53.33 + 5.747 0.984375 4329662 64.00 + 5.931 0.985938 4336652 71.11 + 6.123 0.987500 4343405 80.00 + 6.343 0.989062 4350335 91.43 + 6.583 0.990625 4357145 106.67 + 6.863 0.992188 4364035 128.00 + 7.019 0.992969 4367488 142.22 + 7.199 0.993750 4370954 160.00 + 7.399 0.994531 4374345 182.86 + 7.627 0.995313 4377778 213.33 + 7.919 0.996094 4381194 256.00 + 8.091 0.996484 4382903 284.44 + 8.295 0.996875 4384649 320.00 + 8.543 0.997266 4386367 365.71 + 8.855 0.997656 4388099 426.67 + 9.271 0.998047 4389804 512.00 + 9.535 0.998242 4390655 568.89 + 9.839 0.998437 4391500 640.00 + 10.223 0.998633 4392357 731.43 + 10.711 0.998828 4393212 853.33 + 11.263 0.999023 4394071 1024.00 + 11.583 0.999121 4394509 1137.78 + 11.927 0.999219 4394940 1280.00 + 12.319 0.999316 4395359 1462.86 + 12.839 0.999414 4395794 1706.67 + 13.415 0.999512 4396219 2048.00 + 13.807 0.999561 4396438 2275.56 + 14.207 0.999609 4396653 2560.00 + 14.623 0.999658 4396866 2925.71 + 15.135 0.999707 4397078 3413.33 + 15.687 0.999756 4397293 4096.00 + 16.047 0.999780 4397399 4551.11 + 16.479 0.999805 4397507 5120.00 + 17.007 0.999829 4397619 5851.43 + 17.535 0.999854 4397724 6826.67 + 18.415 0.999878 4397831 8192.00 + 18.815 0.999890 4397882 9102.22 + 19.391 0.999902 4397936 10240.00 + 19.823 0.999915 4397990 11702.86 + 20.351 0.999927 4398043 13653.33 + 20.719 0.999939 4398097 16384.00 + 20.991 0.999945 4398126 18204.44 + 21.231 0.999951 4398151 20480.00 + 21.855 0.999957 4398179 23405.71 + 22.495 0.999963 4398205 27306.67 + 23.311 0.999969 4398232 32768.00 + 23.711 0.999973 4398245 36408.89 + 23.967 0.999976 4398259 40960.00 + 24.639 0.999979 4398272 46811.43 + 25.055 0.999982 4398285 54613.33 + 25.519 0.999985 4398298 65536.00 + 25.839 0.999986 4398305 72817.78 + 26.287 0.999988 4398312 81920.00 + 26.511 0.999989 4398319 93622.86 + 26.735 0.999991 4398326 109226.67 + 27.263 0.999992 4398332 131072.00 + 27.375 0.999993 4398335 145635.56 + 27.583 0.999994 4398339 163840.00 + 27.695 0.999995 4398342 187245.71 + 28.079 0.999995 4398345 218453.33 + 28.383 0.999996 4398349 262144.00 + 28.559 0.999997 4398351 291271.11 + 28.575 0.999997 4398352 327680.00 + 28.751 0.999997 4398354 374491.43 + 28.783 0.999998 4398355 436906.67 + 29.215 0.999998 4398357 524288.00 + 29.311 0.999998 4398359 582542.22 + 29.311 0.999998 4398359 655360.00 + 29.567 0.999999 4398361 748982.86 + 29.567 0.999999 4398361 873813.33 + 29.567 0.999999 4398361 1048576.00 + 29.663 0.999999 4398362 1165084.44 + 29.663 0.999999 4398362 1310720.00 + 29.887 0.999999 4398363 1497965.71 + 29.887 0.999999 4398363 1747626.67 + 29.887 1.000000 4398363 2097152.00 + 30.431 1.000000 4398364 2330168.89 + 30.431 1.000000 4398364 2621440.00 + 30.431 1.000000 4398364 2995931.43 + 30.431 1.000000 4398364 3495253.33 + 30.431 1.000000 4398364 4194304.00 + 30.959 1.000000 4398365 4660337.78 + 30.959 1.000000 4398365 inf +#[Mean = 1.443, StdDeviation = 1.154] +#[Max = 30.944, Total count = 4398365] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 4799241 requests in 2.00m, 615.21MB read +Requests/sec: 39994.01 +Transfer/sec: 5.13MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_45k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_45k new file mode 100644 index 000000000..94611a12d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_45k @@ -0,0 +1,145 @@ + Thread calibration: mean lat.: 1.796ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.872ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.795ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.732ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.777ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.804ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.872ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.850ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.84ms 1.93ms 51.74ms 90.52% + Req/Sec 5.93k 729.00 11.67k 80.06% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.38ms + 75.000% 1.99ms + 90.000% 3.63ms + 99.000% 7.98ms + 99.900% 23.93ms + 99.990% 41.85ms + 99.999% 50.27ms +100.000% 51.78ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.090 0.000000 1 1.00 + 0.594 0.100000 496008 1.11 + 0.820 0.200000 990449 1.25 + 1.018 0.300000 1486509 1.43 + 1.202 0.400000 1980392 1.67 + 1.385 0.500000 2474320 2.00 + 1.482 0.550000 2723564 2.22 + 1.584 0.600000 2969266 2.50 + 1.697 0.650000 3217350 2.86 + 1.827 0.700000 3464883 3.33 + 1.987 0.750000 3711881 4.00 + 2.089 0.775000 3836163 4.44 + 2.215 0.800000 3958682 5.00 + 2.387 0.825000 4082839 5.71 + 2.641 0.850000 4206030 6.67 + 3.055 0.875000 4329710 8.00 + 3.331 0.887500 4391461 8.89 + 3.635 0.900000 4453396 10.00 + 3.955 0.912500 4515168 11.43 + 4.303 0.925000 4577501 13.33 + 4.679 0.937500 4639066 16.00 + 4.883 0.943750 4669857 17.78 + 5.103 0.950000 4700689 20.00 + 5.347 0.956250 4731633 22.86 + 5.619 0.962500 4762598 26.67 + 5.931 0.968750 4793491 32.00 + 6.111 0.971875 4809133 35.56 + 6.303 0.975000 4824356 40.00 + 6.523 0.978125 4839741 45.71 + 6.787 0.981250 4855363 53.33 + 7.103 0.984375 4870660 64.00 + 7.295 0.985938 4878372 71.11 + 7.519 0.987500 4886134 80.00 + 7.787 0.989062 4893883 91.43 + 8.131 0.990625 4901637 106.67 + 8.607 0.992188 4909366 128.00 + 8.935 0.992969 4913219 142.22 + 9.367 0.993750 4917053 160.00 + 9.927 0.994531 4920908 182.86 + 10.615 0.995313 4924766 213.33 + 11.495 0.996094 4928628 256.00 + 12.031 0.996484 4930563 284.44 + 12.639 0.996875 4932484 320.00 + 13.383 0.997266 4934431 365.71 + 14.327 0.997656 4936356 426.67 + 15.543 0.998047 4938288 512.00 + 16.335 0.998242 4939258 568.89 + 17.455 0.998437 4940218 640.00 + 19.231 0.998633 4941189 731.43 + 21.375 0.998828 4942156 853.33 + 24.383 0.999023 4943119 1024.00 + 26.383 0.999121 4943601 1137.78 + 28.783 0.999219 4944082 1280.00 + 31.535 0.999316 4944564 1462.86 + 34.079 0.999414 4945050 1706.67 + 36.575 0.999512 4945534 2048.00 + 37.439 0.999561 4945777 2275.56 + 38.175 0.999609 4946024 2560.00 + 38.783 0.999658 4946257 2925.71 + 39.327 0.999707 4946505 3413.33 + 39.775 0.999756 4946745 4096.00 + 40.063 0.999780 4946871 4551.11 + 40.319 0.999805 4946988 5120.00 + 40.607 0.999829 4947101 5851.43 + 40.959 0.999854 4947232 6826.67 + 41.375 0.999878 4947347 8192.00 + 41.631 0.999890 4947408 9102.22 + 41.951 0.999902 4947464 10240.00 + 42.527 0.999915 4947527 11702.86 + 43.263 0.999927 4947587 13653.33 + 44.575 0.999939 4947647 16384.00 + 45.727 0.999945 4947676 18204.44 + 46.719 0.999951 4947705 20480.00 + 47.551 0.999957 4947735 23405.71 + 48.159 0.999963 4947765 27306.67 + 48.767 0.999969 4947796 32768.00 + 49.023 0.999973 4947812 36408.89 + 49.247 0.999976 4947828 40960.00 + 49.535 0.999979 4947841 46811.43 + 49.791 0.999982 4947859 54613.33 + 49.919 0.999985 4947872 65536.00 + 50.015 0.999986 4947880 72817.78 + 50.143 0.999988 4947887 81920.00 + 50.239 0.999989 4947896 93622.86 + 50.335 0.999991 4947902 109226.67 + 50.463 0.999992 4947911 131072.00 + 50.527 0.999993 4947914 145635.56 + 50.559 0.999994 4947918 163840.00 + 50.623 0.999995 4947922 187245.71 + 50.655 0.999995 4947924 218453.33 + 50.751 0.999996 4947929 262144.00 + 50.783 0.999997 4947930 291271.11 + 50.847 0.999997 4947932 327680.00 + 50.879 0.999997 4947933 374491.43 + 50.975 0.999998 4947936 436906.67 + 51.007 0.999998 4947937 524288.00 + 51.103 0.999998 4947938 582542.22 + 51.167 0.999998 4947939 655360.00 + 51.199 0.999999 4947940 748982.86 + 51.295 0.999999 4947941 873813.33 + 51.455 0.999999 4947942 1048576.00 + 51.455 0.999999 4947942 1165084.44 + 51.487 0.999999 4947943 1310720.00 + 51.487 0.999999 4947943 1497965.71 + 51.583 0.999999 4947944 1747626.67 + 51.583 1.000000 4947944 2097152.00 + 51.583 1.000000 4947944 2330168.89 + 51.679 1.000000 4947945 2621440.00 + 51.679 1.000000 4947945 2995931.43 + 51.679 1.000000 4947945 3495253.33 + 51.679 1.000000 4947945 4194304.00 + 51.679 1.000000 4947945 4660337.78 + 51.775 1.000000 4947946 5242880.00 + 51.775 1.000000 4947946 inf +#[Mean = 1.836, StdDeviation = 1.928] +#[Max = 51.744, Total count = 4947946] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 5398942 requests in 2.00m, 692.10MB read +Requests/sec: 44991.36 +Transfer/sec: 5.77MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_55k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_55k new file mode 100644 index 000000000..669b18a0b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/get_55k @@ -0,0 +1,114 @@ + Thread calibration: mean lat.: 136.972ms, rate sampling interval: 620ms + Thread calibration: mean lat.: 137.138ms, rate sampling interval: 628ms + Thread calibration: mean lat.: 137.705ms, rate sampling interval: 631ms + Thread calibration: mean lat.: 138.997ms, rate sampling interval: 629ms + Thread calibration: mean lat.: 144.400ms, rate sampling interval: 666ms + Thread calibration: mean lat.: 136.597ms, rate sampling interval: 622ms + Thread calibration: mean lat.: 136.388ms, rate sampling interval: 627ms + Thread calibration: mean lat.: 138.213ms, rate sampling interval: 637ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 920.40ms 374.13ms 1.76s 57.87% + Req/Sec 6.45k 93.22 6.71k 70.56% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 891.90ms + 75.000% 1.23s + 90.000% 1.47s + 99.000% 1.63s + 99.900% 1.71s + 99.990% 1.76s + 99.999% 1.76s +100.000% 1.76s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 311.039 0.000000 1 1.00 + 439.295 0.100000 102883 1.11 + 530.431 0.200000 205736 1.25 + 636.927 0.300000 308986 1.43 + 760.831 0.400000 411750 1.67 + 891.903 0.500000 514699 2.00 + 958.975 0.550000 566095 2.22 + 1025.023 0.600000 617252 2.50 + 1092.607 0.650000 669206 2.86 + 1160.191 0.700000 720241 3.33 + 1231.871 0.750000 772201 4.00 + 1267.711 0.775000 797800 4.44 + 1304.575 0.800000 823001 5.00 + 1345.535 0.825000 848864 5.71 + 1385.471 0.850000 874794 6.67 + 1427.455 0.875000 900526 8.00 + 1446.911 0.887500 913389 8.89 + 1466.367 0.900000 926242 10.00 + 1485.823 0.912500 939254 11.43 + 1505.279 0.925000 952048 13.33 + 1525.759 0.937500 964556 16.00 + 1537.023 0.943750 971142 17.78 + 1548.287 0.950000 977627 20.00 + 1558.527 0.956250 983871 22.86 + 1568.767 0.962500 990371 26.67 + 1581.055 0.968750 996933 32.00 + 1587.199 0.971875 1000154 35.56 + 1593.343 0.975000 1003070 40.00 + 1600.511 0.978125 1006440 45.71 + 1607.679 0.981250 1009624 53.33 + 1615.871 0.984375 1012776 64.00 + 1619.967 0.985938 1014242 71.11 + 1626.111 0.987500 1016043 80.00 + 1631.231 0.989062 1017402 91.43 + 1637.375 0.990625 1019165 106.67 + 1644.543 0.992188 1020677 128.00 + 1647.615 0.992969 1021448 142.22 + 1652.735 0.993750 1022354 160.00 + 1655.807 0.994531 1023016 182.86 + 1661.951 0.995313 1023964 213.33 + 1668.095 0.996094 1024707 256.00 + 1671.167 0.996484 1025091 284.44 + 1676.287 0.996875 1025456 320.00 + 1683.455 0.997266 1025872 365.71 + 1688.575 0.997656 1026287 426.67 + 1692.671 0.998047 1026682 512.00 + 1695.743 0.998242 1026843 568.89 + 1699.839 0.998437 1027047 640.00 + 1702.911 0.998633 1027257 731.43 + 1704.959 0.998828 1027421 853.33 + 1709.055 0.999023 1027688 1024.00 + 1710.079 0.999121 1027732 1137.78 + 1712.127 0.999219 1027856 1280.00 + 1713.151 0.999316 1027943 1462.86 + 1715.199 0.999414 1028074 1706.67 + 1717.247 0.999512 1028138 2048.00 + 1718.271 0.999561 1028196 2275.56 + 1719.295 0.999609 1028242 2560.00 + 1720.319 0.999658 1028286 2925.71 + 1721.343 0.999707 1028341 3413.33 + 1722.367 0.999756 1028377 4096.00 + 1746.943 0.999780 1028407 4551.11 + 1748.991 0.999805 1028429 5120.00 + 1750.015 0.999829 1028449 5851.43 + 1755.135 0.999854 1028477 6826.67 + 1758.207 0.999878 1028514 8192.00 + 1758.207 0.999890 1028514 9102.22 + 1759.231 0.999902 1028533 10240.00 + 1760.255 0.999915 1028567 11702.86 + 1760.255 0.999927 1028567 13653.33 + 1760.255 0.999939 1028567 16384.00 + 1760.255 0.999945 1028567 18204.44 + 1761.279 0.999951 1028578 20480.00 + 1762.303 0.999957 1028583 23405.71 + 1763.327 0.999963 1028608 27306.67 + 1763.327 0.999969 1028608 32768.00 + 1763.327 0.999973 1028608 36408.89 + 1763.327 0.999976 1028608 40960.00 + 1763.327 0.999979 1028608 46811.43 + 1763.327 0.999982 1028608 54613.33 + 1763.327 0.999985 1028608 65536.00 + 1764.351 0.999986 1028623 72817.78 + 1764.351 1.000000 1028623 inf +#[Mean = 920.397, StdDeviation = 374.129] +#[Max = 1763.328, Total count = 1028623] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1560611 requests in 30.00s, 199.97MB read +Requests/sec: 52023.44 +Transfer/sec: 6.67MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/put_50k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/put_50k new file mode 100644 index 000000000..f21217433 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/put_50k @@ -0,0 +1,133 @@ +Thread calibration: mean lat.: 2.244ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.244ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.241ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.258ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.243ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.178ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.325ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.228ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.58ms 1.42ms 27.78ms 93.30% + Req/Sec 6.59k 771.33 12.00k 82.53% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.29ms + 75.000% 1.81ms + 90.000% 2.51ms + 99.000% 8.40ms + 99.900% 13.99ms + 99.990% 20.53ms + 99.999% 25.89ms +100.000% 27.79ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.079 0.000000 1 1.00 + 0.547 0.100000 99949 1.11 + 0.756 0.200000 200077 1.25 + 0.937 0.300000 299644 1.43 + 1.114 0.400000 399334 1.67 + 1.292 0.500000 499473 2.00 + 1.382 0.550000 548904 2.22 + 1.476 0.600000 599045 2.50 + 1.576 0.650000 649080 2.86 + 1.685 0.700000 698829 3.33 + 1.810 0.750000 748554 4.00 + 1.882 0.775000 773414 4.44 + 1.964 0.800000 798598 5.00 + 2.059 0.825000 823526 5.71 + 2.173 0.850000 848301 6.67 + 2.315 0.875000 873329 8.00 + 2.403 0.887500 885712 8.89 + 2.507 0.900000 898298 10.00 + 2.633 0.912500 910753 11.43 + 2.803 0.925000 923130 13.33 + 3.061 0.937500 935595 16.00 + 3.279 0.943750 941824 17.78 + 3.569 0.950000 948071 20.00 + 3.941 0.956250 954304 22.86 + 4.415 0.962500 960543 26.67 + 4.995 0.968750 966779 32.00 + 5.343 0.971875 969925 35.56 + 5.703 0.975000 973002 40.00 + 6.107 0.978125 976145 45.71 + 6.551 0.981250 979254 53.33 + 7.067 0.984375 982385 64.00 + 7.363 0.985938 983923 71.11 + 7.715 0.987500 985481 80.00 + 8.115 0.989062 987039 91.43 + 8.575 0.990625 988609 106.67 + 9.047 0.992188 990179 128.00 + 9.287 0.992969 990957 142.22 + 9.535 0.993750 991719 160.00 + 9.831 0.994531 992499 182.86 + 10.175 0.995313 993289 213.33 + 10.607 0.996094 994055 256.00 + 10.887 0.996484 994441 284.44 + 11.207 0.996875 994832 320.00 + 11.551 0.997266 995227 365.71 + 11.967 0.997656 995616 426.67 + 12.423 0.998047 996003 512.00 + 12.679 0.998242 996199 568.89 + 12.975 0.998437 996392 640.00 + 13.271 0.998633 996587 731.43 + 13.631 0.998828 996780 853.33 + 14.047 0.999023 996975 1024.00 + 14.271 0.999121 997073 1137.78 + 14.511 0.999219 997171 1280.00 + 14.775 0.999316 997268 1462.86 + 15.095 0.999414 997367 1706.67 + 15.471 0.999512 997462 2048.00 + 15.799 0.999561 997511 2275.56 + 16.159 0.999609 997560 2560.00 + 16.703 0.999658 997610 2925.71 + 17.295 0.999707 997657 3413.33 + 18.127 0.999756 997706 4096.00 + 18.607 0.999780 997732 4551.11 + 18.959 0.999805 997755 5120.00 + 19.375 0.999829 997779 5851.43 + 19.663 0.999854 997803 6826.67 + 20.095 0.999878 997828 8192.00 + 20.367 0.999890 997840 9102.22 + 20.623 0.999902 997852 10240.00 + 20.847 0.999915 997864 11702.86 + 21.215 0.999927 997876 13653.33 + 21.663 0.999939 997890 16384.00 + 21.823 0.999945 997895 18204.44 + 22.079 0.999951 997901 20480.00 + 22.239 0.999957 997907 23405.71 + 22.511 0.999963 997913 27306.67 + 22.719 0.999969 997919 32768.00 + 23.087 0.999973 997922 36408.89 + 23.503 0.999976 997925 40960.00 + 23.999 0.999979 997928 46811.43 + 24.271 0.999982 997931 54613.33 + 24.719 0.999985 997934 65536.00 + 25.007 0.999986 997936 72817.78 + 25.119 0.999988 997937 81920.00 + 25.887 0.999989 997939 93622.86 + 26.223 0.999991 997940 109226.67 + 26.703 0.999992 997942 131072.00 + 27.087 0.999993 997943 145635.56 + 27.087 0.999994 997943 163840.00 + 27.263 0.999995 997944 187245.71 + 27.471 0.999995 997945 218453.33 + 27.487 0.999996 997946 262144.00 + 27.487 0.999997 997946 291271.11 + 27.487 0.999997 997946 327680.00 + 27.711 0.999997 997947 374491.43 + 27.711 0.999998 997947 436906.67 + 27.743 0.999998 997948 524288.00 + 27.743 0.999998 997948 582542.22 + 27.743 0.999998 997948 655360.00 + 27.743 0.999999 997948 748982.86 + 27.743 0.999999 997948 873813.33 + 27.791 0.999999 997949 1048576.00 + 27.791 1.000000 997949 inf +#[Mean = 1.576, StdDeviation = 1.425] +#[Max = 27.776, Total count = 997949] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1499029 requests in 30.00s, 118.65MB read +Requests/sec: 49968.19 +Transfer/sec: 3.96MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/put_75k b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/put_75k new file mode 100644 index 000000000..fa615d5d2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/profile_wrk/put_75k @@ -0,0 +1,108 @@ + Thread calibration: mean lat.: 451.221ms, rate sampling interval: 1886ms + Thread calibration: mean lat.: 555.343ms, rate sampling interval: 2160ms + Thread calibration: mean lat.: 559.233ms, rate sampling interval: 2181ms + Thread calibration: mean lat.: 505.114ms, rate sampling interval: 1975ms + Thread calibration: mean lat.: 483.417ms, rate sampling interval: 2003ms + Thread calibration: mean lat.: 508.803ms, rate sampling interval: 1984ms + Thread calibration: mean lat.: 522.275ms, rate sampling interval: 2049ms + Thread calibration: mean lat.: 490.098ms, rate sampling interval: 1955ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.44s 787.32ms 3.94s 56.58% + Req/Sec 8.15k 409.18 8.79k 65.79% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 2.48s + 75.000% 3.04s + 90.000% 3.64s + 99.000% 3.82s + 99.900% 3.90s + 99.990% 3.94s + 99.999% 3.94s +100.000% 3.94s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 1123.327 0.000000 4 1.00 + 1402.879 0.100000 130794 1.11 + 1570.815 0.200000 261392 1.25 + 1838.079 0.300000 391935 1.43 + 2170.879 0.400000 522353 1.67 + 2476.031 0.500000 654200 2.00 + 2555.903 0.550000 718950 2.22 + 2650.111 0.600000 783557 2.50 + 2768.895 0.650000 849056 2.86 + 2906.111 0.700000 914410 3.33 + 3043.327 0.750000 980046 4.00 + 3123.199 0.775000 1012091 4.44 + 3223.551 0.800000 1045010 5.00 + 3342.335 0.825000 1077317 5.71 + 3487.743 0.850000 1109906 6.67 + 3583.999 0.875000 1142554 8.00 + 3614.719 0.887500 1159870 8.89 + 3637.247 0.900000 1175442 10.00 + 3657.727 0.912500 1192440 11.43 + 3678.207 0.925000 1209758 13.33 + 3694.591 0.937500 1224597 16.00 + 3704.831 0.943750 1232578 17.78 + 3719.167 0.950000 1241375 20.00 + 3731.455 0.956250 1248902 22.86 + 3745.791 0.962500 1257803 26.67 + 3760.127 0.968750 1265595 32.00 + 3766.271 0.971875 1269135 35.56 + 3774.463 0.975000 1273213 40.00 + 3782.655 0.978125 1277564 45.71 + 3790.847 0.981250 1281894 53.33 + 3799.039 0.984375 1285401 64.00 + 3805.183 0.985938 1287557 71.11 + 3811.327 0.987500 1289459 80.00 + 3817.471 0.989062 1291584 91.43 + 3825.663 0.990625 1293821 106.67 + 3835.903 0.992188 1295915 128.00 + 3839.999 0.992969 1296955 142.22 + 3846.143 0.993750 1297875 160.00 + 3852.287 0.994531 1298930 182.86 + 3856.383 0.995313 1299659 213.33 + 3864.575 0.996094 1300855 256.00 + 3868.671 0.996484 1301327 284.44 + 3870.719 0.996875 1301820 320.00 + 3874.815 0.997266 1302212 365.71 + 3878.911 0.997656 1302751 426.67 + 3883.007 0.998047 1303189 512.00 + 3887.103 0.998242 1303520 568.89 + 3889.151 0.998437 1303713 640.00 + 3895.295 0.998633 1304053 731.43 + 3899.391 0.998828 1304343 853.33 + 3901.439 0.999023 1304500 1024.00 + 3903.487 0.999121 1304744 1137.78 + 3903.487 0.999219 1304744 1280.00 + 3907.583 0.999316 1304948 1462.86 + 3909.631 0.999414 1304992 1706.67 + 3915.775 0.999512 1305171 2048.00 + 3915.775 0.999561 1305171 2275.56 + 3917.823 0.999609 1305267 2560.00 + 3919.871 0.999658 1305304 2925.71 + 3921.919 0.999707 1305386 3413.33 + 3923.967 0.999756 1305432 4096.00 + 3926.015 0.999780 1305462 4551.11 + 3928.063 0.999805 1305516 5120.00 + 3928.063 0.999829 1305516 5851.43 + 3934.207 0.999854 1305547 6826.67 + 3936.255 0.999878 1305639 8192.00 + 3936.255 0.999890 1305639 9102.22 + 3936.255 0.999902 1305639 10240.00 + 3936.255 0.999915 1305639 11702.86 + 3938.303 0.999927 1305697 13653.33 + 3938.303 0.999939 1305697 16384.00 + 3938.303 0.999945 1305697 18204.44 + 3938.303 0.999951 1305697 20480.00 + 3938.303 0.999957 1305697 23405.71 + 3938.303 0.999963 1305697 27306.67 + 3940.351 0.999969 1305737 32768.00 + 3940.351 1.000000 1305737 inf +#[Mean = 2441.787, StdDeviation = 787.318] +#[Max = 3938.304, Total count = 1305737] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1966261 requests in 30.00s, 155.63MB read +Requests/sec: 65545.40 +Transfer/sec: 5.19MB diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/report.md b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/report.md new file mode 100644 index 000000000..f9df5b65b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/results/stage4/report.md @@ -0,0 +1,55 @@ +# Stage-4 + +В данном этапе нужно было реализовать реплицирование данных на несколько нод. Проверим, как изменилась производительность +сервера. + +#### PUT +[put_75k](profile_wrk%2Fput_75k) - нагрузку прошлого этапа для put с `RPS=75_000` сервер не выдержал.\ +[put_50k](profile_wrk%2Fput_50k) - сервер выдержал нагрузку с `RPS=50_000`. +![put_50k.png](profile_png%2Fput_50k.png) + +#### GET + +[get_55k](profile_wrk%2Fget_55k) - сервер не справляется даже с половиной `RPS` от прошлого этапа.\ +[get_40k](profile_wrk%2Fget_40k) - но хорошо себя показывает даже при долгом запуске с `RPS=40_000`.\ +[get_45k](profile_wrk%2Fget_45k) - с такой же нагрузкой сервер тоже справляется, но появляются медленные запросы. +![get_40k.png](profile_png%2Fget_40k.png) + +### Аллокации +Get и Put - появились новые аллокации, связанные с обработкой запроса. В частности, для работы с хедерами и выявления самого свежего ответа из множества. +1) аллокации при выборе клиентов +![alloc_getClients.png](profile_png%2Falloc_getClients.png) +2) аллокации при выборе самого свежего ответа +![alloc_responses.png](profile_png%2Falloc_responses.png) + +### CPU +По сравнению с предыдущим этапом, заметно возросла доля, занимаемая методом _invokeRemote_, +так как сейчас для одного и того же ключа мы перенаправляем запрос одновременно на несколько нод. +![cpu_put.png](profile_png%2Fcpu_put.png) + +### Lock +Количество блокировок также значительно увеличилось, поскольку на данном этапе мы отправляем запросы на несколько нод, +которые, в свою очередь, выполняют одну и ту же работу, требующую захвата блокировки. Возможно, +использование асинхронной обработки запросов помогло бы сократить количество блокировок, но мы рассмотрим это в следующем этапе ;) +![put_lock.png](profile_png%2Fput_lock.png) + +### Надёжность +После заполнения данными с `from=2` и `ack=2` (т.е. реплицировании их на 2/3 нод), +я выключил третью ноду и сравнил надёжность системы с прошлым этапом. Получились следующие результаты: +1) stage-3 \ +![durability_s3.png](profile_png%2Fdurability_s3.png) +2) stage-4 \ +![durability_s4.png](profile_png%2Fdurability_s4.png) + +В четвёртом этапе я запускал Get с параметрами `from=2` и `ack=1`. Видно, что сервер смог ответить на все запросы, несмотря на то, что одна +нода была недоступна. С другой стороны, если собирать `ack` со всех доступных нод (при условии, что одна не работает), +то больше, чем половине случаев выдавало `404 NOT_FOUND`. Однако в реальной жизни мы вряд ли хотим получить подтверждение +от всех, поэтому в данном случае нас это не интересует. + + +#### Вывод +В отличие от предыдущего этапа, где достаточно было получить ответ от одной ноды, теперь мы практически всегда +(в зависимости от параметров _ack_ и _from_) вынуждены обращаться сразу к нескольким серверам для получения ответа по +одному и тому же ключу, что существенно снижает производительность нашей системы. С другой же стороны, живучесть +системы сильно возросла и способна отправлять ответы, даже если какая-то её часть недоступна. Что касается оправданности такого trade-off, если данные в нашей базе критичны или конфиденциальны, то чем надежнее система, тем лучше. Если же нам важнее скорость, как, например, в системах реального времени или, возможно, в каком-то интерактивном приложении, то ускорение обработки запросов и повышение скорости ответов может иметь приоритет перед абсолютной надежностью данных, при условии, что риск потери данных или ошибок приемлем. + diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/scripts/Get2.lua b/src/main/java/ru/vk/itmo/test/bandurinvladislav/scripts/Get2.lua index dd091c0d5..565e9e548 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/scripts/Get2.lua +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/scripts/Get2.lua @@ -1,8 +1,8 @@ math.randomseed(os.time()) function request() - counter = math.random(100000100, 100000000 + 34000000) + counter = math.random(100000100, 100000000 + 1562001) headers = {} headers["Host"] = "localhost:8080" - return wrk.format("GET", "/v0/entity?id=" .. tostring(counter), headers) + return wrk.format("GET", "/v0/entity?id=" .. tostring(counter) .. "&from=2&ack=1", headers) end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/scripts/PutRequest.lua b/src/main/java/ru/vk/itmo/test/bandurinvladislav/scripts/PutRequest.lua index a0025c53d..15895efb1 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/scripts/PutRequest.lua +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/scripts/PutRequest.lua @@ -7,5 +7,5 @@ function request() headers["Content-Type"] = "text/plain" headers["Content-Length"] = #{string.byte(body, 1, -1)} headers["Host"] = "localhost:8080" - return wrk.format("PUT", "/v0/entity?id=" .. tostring(counter), headers, body) + return wrk.format("PUT", "/v0/entity?id=" .. tostring(counter) .. "&from=2&ack=2", headers, body) end diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/util/Constants.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/util/Constants.java index 0810dfd7c..8cd5f8036 100644 --- a/src/main/java/ru/vk/itmo/test/bandurinvladislav/util/Constants.java +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/util/Constants.java @@ -2,12 +2,19 @@ public final class Constants { public static final String ENDPOINT = "/v0/entity"; + public static final String HEADER_TIMESTAMP = "X-Timestamp: "; + public static final String HEADER_SENDER = "X-Sender: "; public static final String PARAMETER_ID = "id="; + public static final String PARAMETER_ACK = "ack="; + public static final String PARAMETER_FROM = "from="; public static final String TOO_MANY_REQUESTS = "429 too many requests"; + public static final String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; - public static final int THREADS = Runtime.getRuntime().availableProcessors(); + public static final int CLIENT_RESPONSE_TIMEOUT_MILLIS = 100; + public static final int ACCEPTOR_THREADS = Runtime.getRuntime().availableProcessors(); + public static final int THREAD_KEEP_ALIVE_TIME = 60; public static final long TASK_DEADLINE_MILLIS = 500; - public static final int FLUSH_THRESHOLD_BYTES = 10 * 1024 * 1024; + public static final int FLUSH_THRESHOLD_BYTES = 3 * 1024 * 1024; public static final int QUEUE_SIZE = 128; private Constants() { diff --git a/src/main/java/ru/vk/itmo/test/bandurinvladislav/util/NetworkUtil.java b/src/main/java/ru/vk/itmo/test/bandurinvladislav/util/NetworkUtil.java new file mode 100644 index 000000000..9c3785057 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/bandurinvladislav/util/NetworkUtil.java @@ -0,0 +1,121 @@ +package ru.vk.itmo.test.bandurinvladislav.util; + +import one.nio.http.HttpSession; +import one.nio.http.Request; +import one.nio.http.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ru.vk.itmo.test.bandurinvladislav.RequestProcessingState; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; +import java.util.List; + +public class NetworkUtil { + + private static final Logger logger = LoggerFactory.getLogger(NetworkUtil.class); + + private NetworkUtil() { + } + + public static int getParameterAsInt(String parameter, int defaultValue) { + return parameter == null ? defaultValue : Integer.parseInt(parameter); + } + + public static boolean isMethodAllowed(Request request) { + return switch (request.getMethod()) { + case Request.METHOD_GET, Request.METHOD_PUT, + Request.METHOD_DELETE -> true; + default -> false; + }; + } + + public static Response successResponse(List responses) { + if (responses == null || responses.isEmpty()) { + return new Response(Response.INTERNAL_ERROR, + "Unable to create a successful response from the received data." + .getBytes(StandardCharsets.UTF_8)); + } + Response response = responses.getFirst(); + long maxTimestamp = getTimestampHeader(response); + for (int i = 1; i < responses.size(); i++) { + long timestamp = getTimestampHeader(responses.get(i)); + if (Math.max(maxTimestamp, timestamp) == timestamp) { + response = responses.get(i); + maxTimestamp = timestamp; + } + } + return response; + } + + public static Long getTimestampHeader(Response response) { + String header = response.getHeader(Constants.HEADER_TIMESTAMP); + return header == null ? -1 : Long.parseLong(header); + } + + public static Response validateParams(String key, int ack, int from, int clusterSize) { + if (StringUtil.isEmpty(key)) { + return new Response(Response.BAD_REQUEST, "Id can't be empty".getBytes(StandardCharsets.UTF_8)); + } + + if (ack <= 0 || from <= 0) { + return new Response(Response.BAD_REQUEST, + "ack and from can't be negative".getBytes(StandardCharsets.UTF_8)); + } + + if (from < ack) { + return new Response(Response.BAD_REQUEST, + "from can't be less than ack".getBytes(StandardCharsets.UTF_8)); + } + + if (from > clusterSize) { + return new Response(Response.BAD_REQUEST, + "from can't be greater than nodes count".getBytes(StandardCharsets.UTF_8)); + } + + return null; + } + + public static boolean shouldHandle(Response r) { + return r.getStatus() < HttpURLConnection.HTTP_MULT_CHOICE + || r.getStatus() == HttpURLConnection.HTTP_NOT_FOUND; + } + + public static void handleResponse(HttpSession session, RequestProcessingState rs, Response r, int ack, int from) { + if (NetworkUtil.shouldHandle(r)) { + rs.getResponses().add(r); + if (rs.getResponses().size() >= ack + && rs.isResponseSent().compareAndSet(false, true)) { + trySendResponse(session, successResponse(rs.getResponses())); + } else if (from - rs.getFailedResponseCount().get() < ack) { + trySendResponse(session, new Response(Constants.NOT_ENOUGH_REPLICAS, Response.EMPTY)); + } + } else { + rs.getFailedResponseCount().getAndIncrement(); + } + } + + public static void trySendResponse(HttpSession session, Response response) { + try { + session.sendResponse(response); + } catch (IOException e) { + logger.error("IO exception during response sending: " + e.getMessage()); + try { + session.sendError(Response.INTERNAL_ERROR, null); + } catch (IOException ex) { + logger.error("Exception while sending close connection:", e); + session.scheduleClose(); + } + } + } + + public static void handleTimeout(HttpSession session, RequestProcessingState rs, int ack, int from) { + rs.getFailedResponseCount().getAndIncrement(); + if (from - rs.getFailedResponseCount().get() < ack + && rs.isResponseSent().compareAndSet(false, true)) { + NetworkUtil.trySendResponse(session, + new Response(Response.GATEWAY_TIMEOUT, Response.EMPTY)); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DaoMediator.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DaoMediator.java new file mode 100644 index 000000000..d037d205b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DaoMediator.java @@ -0,0 +1,35 @@ +package ru.vk.itmo.test.emelyanovvitaliy; + +import one.nio.http.Request; +import ru.vk.itmo.dao.Entry; +import ru.vk.itmo.test.emelyanovvitaliy.dao.TimestampedEntry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; + +public abstract class DaoMediator { + public static final long NEVER_TIMESTAMP = Long.MIN_VALUE; + + protected static MemorySegment keyFor(String id) { + return MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)); + } + + protected static byte[] valueFor(Entry entry) { + return entry.value().toArray(ValueLayout.JAVA_BYTE); + } + + abstract void stop(); + + abstract boolean isStopped(); + + abstract boolean put(Request request); + + // returns null if error, + // entry with null value and timestamp if the answer is tombstone + // entry with null value and timestamp equals NEVER_TIMESTAMP if no answer + // entry if it exists + abstract TimestampedEntry get(Request request); + + abstract boolean delete(Request request); +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DhtServer.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DhtServer.java index 64fca3516..8cf073db3 100644 --- a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DhtServer.java +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DhtServer.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.emelyanovvitaliy; -import one.nio.http.HttpClient; -import one.nio.http.HttpException; import one.nio.http.HttpServer; import one.nio.http.HttpServerConfig; import one.nio.http.HttpSession; @@ -10,23 +8,15 @@ import one.nio.http.Request; import one.nio.http.RequestMethod; import one.nio.http.Response; -import one.nio.net.ConnectionString; -import one.nio.pool.PoolException; import one.nio.server.AcceptorConfig; -import one.nio.util.Hash; import ru.vk.itmo.ServiceConfig; -import ru.vk.itmo.dao.BaseEntry; -import ru.vk.itmo.dao.Config; import ru.vk.itmo.dao.Entry; -import ru.vk.itmo.test.reference.dao.ReferenceDao; +import ru.vk.itmo.test.emelyanovvitaliy.dao.TimestampedEntry; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -37,23 +27,22 @@ import static one.nio.http.Request.METHOD_PUT; public class DhtServer extends HttpServer { - protected static final byte[] EMPTY_BODY = new byte[0]; + public static final String NOT_ENOUGH_REPLICAS_STATUS = "504 Not Enough Replicas"; + public static final String TIMESTAMP_HEADER = "X-Timestamp: "; + public static final String ID_KEY = "id="; protected static final int THREADS_PER_PROCESSOR = 2; - protected static final int FLUSH_THRESHOLD_BYTES = 1 << 24; // 16 MiB + protected static final byte[] EMPTY_BODY = new byte[0]; protected static final long KEEP_ALIVE_TIME_MILLIS = 1000; protected static final int REQUEST_TIMEOUT_MILLIS = 1024; protected static final int THREAD_POOL_TERMINATION_TIMEOUT_SECONDS = 600; - protected static final int REMOTE_CONNECT_TIMEOUT_MILLIS = 100; protected static final int TASK_QUEUE_SIZE = Runtime.getRuntime().availableProcessors() * THREADS_PER_PROCESSOR; - protected final HttpClient[] httpClients; - protected final int[] clientsHashes; - + protected final MergeDaoMediator mergeDaoMediator; protected final AtomicInteger threadsInPool; protected final ThreadPoolExecutor threadPoolExecutor; - protected final ReferenceDao dao; public DhtServer(ServiceConfig config) throws IOException { super(createConfig(config)); + mergeDaoMediator = new MergeDaoMediator(config.workingDir(), config.selfUrl(), config.clusterUrls()); threadsInPool = new AtomicInteger(0); threadPoolExecutor = new ThreadPoolExecutor( Runtime.getRuntime().availableProcessors() * THREADS_PER_PROCESSOR, @@ -68,9 +57,6 @@ public DhtServer(ServiceConfig config) throws IOException { return t; } ); - httpClients = getHttpClients(config.clusterUrls(), config.selfUrl()); - clientsHashes = getClientHashes(config.clusterUrls()); - dao = new ReferenceDao(new Config(config.workingDir(), FLUSH_THRESHOLD_BYTES)); } @Override @@ -81,14 +67,7 @@ public synchronized void stop() { if (!threadPoolExecutor.awaitTermination(THREAD_POOL_TERMINATION_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { throw new UncheckedTimeoutException("Waited too lot to stop the thread pool"); } - for (HttpClient httpClient : httpClients) { - if (httpClient != null && !httpClient.isClosed()) { - httpClient.close(); - } - } - dao.close(); - } catch (IOException e) { - throw new UncheckedIOException(e); + mergeDaoMediator.stop(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new UncheckedInterruptedException(e); @@ -97,112 +76,101 @@ public synchronized void stop() { @RequestMethod(METHOD_GET) @Path("/v0/entity") - public void entity(@Param(value = "id") String id, HttpSession session, Request request) throws IOException { - if (isKeyIncorrect(id)) { - sendBadRequestResponse(session); - } else { - long startTimeInMillis = System.currentTimeMillis(); - threadPoolExecutor.execute( - () -> { - try { - if (System.currentTimeMillis() - startTimeInMillis > REQUEST_TIMEOUT_MILLIS) { - session.sendResponse(new Response(Response.PAYMENT_REQUIRED, EMPTY_BODY)); - } else if (!tryForward(session, request, id)) { - Entry entry = dao.get(keyFor(id)); - if (entry == null) { - session.sendResponse(new Response(Response.NOT_FOUND, EMPTY_BODY)); - } else { - session.sendResponse(new Response(Response.OK, valueFor(entry))); - } + public void entity(HttpSession session, Request request) throws IOException { + String id = request.getParameter(ID_KEY); + requestProccessing(id, session, + () -> { + try { + TimestampedEntry entry = mergeDaoMediator.get(request); + if (entry == null) { + sendNotEnoughReplicas(session); + } else if (entry.timestamp() == DaoMediator.NEVER_TIMESTAMP) { + session.sendResponse(new Response(Response.NOT_FOUND, EMPTY_BODY)); + } else { + Response response; + if (entry.value() == null) { + response = new Response(Response.NOT_FOUND, EMPTY_BODY); + } else { + response = new Response( + Response.OK, + ((Entry) entry).value().toArray(ValueLayout.JAVA_BYTE) + ); } - } catch (IOException e) { - throw new UncheckedIOException(e); + response.addHeader(TIMESTAMP_HEADER + entry.timestamp()); + session.sendResponse(response); } + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (IllegalArgumentException e) { + sendBadRequestResponseUnchecked(session); } - ); - } - } - - // choose the client by rendezvous hashing - private HttpClient getHttpClientByKey(String key) { - int maxHash = Integer.MIN_VALUE; - HttpClient choosen = httpClients[0]; - for (int i = 0; i < httpClients.length; i++) { - // cantor pairing function works nicely only with non-negatives - int keyHash = Math.abs(Hash.murmur3(key)); - int totalHash = cantorPairingFunction(clientsHashes[i], keyHash); - if (totalHash > maxHash) { - maxHash = totalHash; - choosen = httpClients[i]; - } - } - return choosen; - } - - private static int cantorPairingFunction(int a, int b) { - return (a + b) * (a + b + 1) / 2 + b; - } - - private boolean tryForward(HttpSession session, Request request, String key) throws IOException { - HttpClient httpClient = getHttpClientByKey(key); - if (httpClient == null) { - return false; - } - try { - session.sendResponse(httpClient.invoke(request, REMOTE_CONNECT_TIMEOUT_MILLIS)); - } catch (PoolException | HttpException e) { - session.sendResponse(new Response(Response.INTERNAL_ERROR, EMPTY_BODY)); - } catch (InterruptedException e) { - session.sendResponse(new Response(Response.INTERNAL_ERROR, EMPTY_BODY)); - Thread.currentThread().interrupt(); - } - return true; + } + ); } @RequestMethod(METHOD_PUT) @Path("/v0/entity") public void putEntity(@Param(value = "id") String id, HttpSession httpSession, Request request) throws IOException { - if (isKeyIncorrect(id) || request.getBody() == null) { - sendBadRequestResponse(httpSession); - } else { - long startTimeInMillis = System.currentTimeMillis(); - threadPoolExecutor.execute( - () -> { - try { - if (System.currentTimeMillis() - startTimeInMillis > REQUEST_TIMEOUT_MILLIS) { - httpSession.sendResponse(new Response(Response.PAYMENT_REQUIRED, EMPTY_BODY)); - return; - } - if (!tryForward(httpSession, request, id)) { - dao.upsert(new BaseEntry<>(keyFor(id), MemorySegment.ofArray(request.getBody()))); - httpSession.sendResponse(new Response(Response.CREATED, EMPTY_BODY)); - } - } catch (IOException e) { - throw new UncheckedIOException(e); + requestProccessing(id, httpSession, + () -> { + try { + if (mergeDaoMediator.put(request)) { + httpSession.sendResponse(new Response(Response.CREATED, EMPTY_BODY)); + } else { + sendNotEnoughReplicas(httpSession); } + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (IllegalArgumentException e) { + sendBadRequestResponseUnchecked(httpSession); } - ); - } + } + ); } @RequestMethod(METHOD_DELETE) @Path("/v0/entity") public void deleteEntity(@Param("id") String id, HttpSession httpSession, Request request) throws IOException { + requestProccessing(id, httpSession, + () -> { + try { + if (mergeDaoMediator.delete(request)) { + httpSession.sendResponse(new Response(Response.ACCEPTED, EMPTY_BODY)); + } else { + sendNotEnoughReplicas(httpSession); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (IllegalArgumentException e) { + sendBadRequestResponseUnchecked(httpSession); + } + } + ); + } + + @Override + public void handleDefault(Request request, HttpSession session) throws IOException { + int requestMethod = request.getMethod(); + if (requestMethod == METHOD_GET || requestMethod == METHOD_PUT || requestMethod == METHOD_DELETE) { + sendBadRequestResponse(session); + } else { + session.sendResponse(new Response(Response.METHOD_NOT_ALLOWED, EMPTY_BODY)); + } + } + + private void requestProccessing(String id, HttpSession session, Runnable runnable) throws IOException { if (isKeyIncorrect(id)) { - sendBadRequestResponse(httpSession); + sendBadRequestResponse(session); } else { long startTimeInMillis = System.currentTimeMillis(); threadPoolExecutor.execute( () -> { try { if (System.currentTimeMillis() - startTimeInMillis > REQUEST_TIMEOUT_MILLIS) { - httpSession.sendResponse(new Response(Response.PAYMENT_REQUIRED, EMPTY_BODY)); + session.sendResponse(new Response(Response.PAYMENT_REQUIRED, EMPTY_BODY)); return; } - if (!tryForward(httpSession, request, id)) { - dao.upsert(new BaseEntry<>(keyFor(id), null)); - httpSession.sendResponse(new Response(Response.ACCEPTED, EMPTY_BODY)); - } + runnable.run(); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -211,20 +179,22 @@ public void deleteEntity(@Param("id") String id, HttpSession httpSession, Reques } } - @Override - public void handleDefault(Request request, HttpSession session) throws IOException { - int requestMethod = request.getMethod(); - if (requestMethod == METHOD_GET || requestMethod == METHOD_PUT || requestMethod == METHOD_DELETE) { - sendBadRequestResponse(session); - } else { - session.sendResponse(new Response(Response.METHOD_NOT_ALLOWED, EMPTY_BODY)); - } + private static void sendNotEnoughReplicas(HttpSession session) throws IOException { + session.sendResponse(new Response(NOT_ENOUGH_REPLICAS_STATUS, EMPTY_BODY)); } private static void sendBadRequestResponse(HttpSession httpSession) throws IOException { httpSession.sendResponse(new Response(Response.BAD_REQUEST, EMPTY_BODY)); } + private void sendBadRequestResponseUnchecked(HttpSession session) { + try { + sendBadRequestResponse(session); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private static boolean isKeyIncorrect(String key) { return key == null || key.isEmpty(); } @@ -238,36 +208,4 @@ private static HttpServerConfig createConfig(ServiceConfig serviceConfig) { config.closeSessions = true; return config; } - - private static HttpClient[] getHttpClients(List urls, String thisUrl) { - HttpClient[] clients = new HttpClient[urls.size()]; - int cnt = 0; - List tmpList = new ArrayList<>(urls); - tmpList.sort(String::compareTo); - for (String url : tmpList) { - clients[cnt++] = url.equals(thisUrl) ? null : new HttpClient(new ConnectionString(url)); - } - return clients; - } - - private static int[] getClientHashes(List urls) { - int[] hashes = new int[urls.size()]; - int cnt = 0; - List tmpList = new ArrayList<>(urls); - tmpList.sort(String::compareTo); - for (String url : tmpList) { - // cantor pairing function works nicely only with non-negatives - hashes[cnt++] = Math.abs(Hash.murmur3(url)); - } - return hashes; - } - - private static MemorySegment keyFor(String id) { - return MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)); - } - - private static byte[] valueFor(Entry entry) { - return entry.value().toArray(ValueLayout.JAVA_BYTE); - } - } diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DhtService.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DhtService.java index 2bd0ed2c0..f9326bc7f 100644 --- a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DhtService.java +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/DhtService.java @@ -31,7 +31,7 @@ public CompletableFuture stop() throws IOException { return CompletableFuture.completedFuture(null); } - @ServiceFactory(stage = 3) + @ServiceFactory(stage = 4) public static class Factory implements ServiceFactory.Factory { @Override diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/LocalDaoMediator.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/LocalDaoMediator.java new file mode 100644 index 000000000..5d8edbcdc --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/LocalDaoMediator.java @@ -0,0 +1,60 @@ +package ru.vk.itmo.test.emelyanovvitaliy; + +import one.nio.http.Request; +import ru.vk.itmo.test.emelyanovvitaliy.dao.ReferenceDao; +import ru.vk.itmo.test.emelyanovvitaliy.dao.TimestampedEntry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.MemorySegment; +import java.util.concurrent.atomic.AtomicBoolean; + +public class LocalDaoMediator extends DaoMediator { + protected final ReferenceDao dao; + protected final AtomicBoolean isClosed = new AtomicBoolean(false); + + LocalDaoMediator(ReferenceDao dao) { + this.dao = dao; + } + + @Override + void stop() { + if (isClosed.getAndSet(true)) { + return; + } + try { + dao.close(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + boolean isStopped() { + return isClosed.get(); + } + + @Override + boolean put(Request request) { + String id = request.getParameter(DhtServer.ID_KEY); + dao.upsert(new TimestampedEntry<>(keyFor(id), MemorySegment.ofArray(request.getBody()))); + return true; + } + + @Override + TimestampedEntry get(Request request) { + MemorySegment id = keyFor(request.getParameter(DhtServer.ID_KEY)); + TimestampedEntry entry = dao.get(id); + if (entry == null) { + return new TimestampedEntry<>(id, null, NEVER_TIMESTAMP); + } + return entry; + } + + @Override + boolean delete(Request request) { + String id = request.getParameter(DhtServer.ID_KEY); + dao.upsert(new TimestampedEntry<>(keyFor(id), null)); + return true; + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/MergeDaoMediator.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/MergeDaoMediator.java new file mode 100644 index 000000000..ffa3d993d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/MergeDaoMediator.java @@ -0,0 +1,213 @@ +package ru.vk.itmo.test.emelyanovvitaliy; + +import one.nio.http.HttpClient; +import one.nio.http.Request; +import one.nio.net.ConnectionString; +import one.nio.util.Hash; +import ru.vk.itmo.dao.Config; +import ru.vk.itmo.test.emelyanovvitaliy.dao.ReferenceDao; +import ru.vk.itmo.test.emelyanovvitaliy.dao.TimestampedEntry; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; + +public class MergeDaoMediator extends DaoMediator { + public static final String FINAL_EXECUTION_HEADER = "X-Final-Execution: "; + public static final String ACK_KEY = "ack="; + public static final String FROM_KEY = "from="; + protected static final String TRUE_STRING = "true"; + protected static final int FLUSH_THRESHOLD_BYTES = 1 << 24; // 16 MiB + public static final String ID_KEY = "id="; + protected final AtomicBoolean isStopped = new AtomicBoolean(false); + protected final DaoMediator[] daoMediators; + protected final int[] mediatorsHashes; + private final LocalDaoMediator localDaoMediator; + + MergeDaoMediator(Path localDir, String thisUrl, List urls) throws IOException { + localDaoMediator = new LocalDaoMediator( + new ReferenceDao( + new Config( + localDir, + FLUSH_THRESHOLD_BYTES + ) + ) + ); + daoMediators = getDaoMediators(urls, thisUrl, localDaoMediator); + mediatorsHashes = getMediatorsHashes(urls); + } + + @Override + void stop() { + if (isStopped.getAndSet(true)) { + return; + } + for (DaoMediator daoMediator : daoMediators) { + daoMediator.stop(); + } + } + + @Override + boolean isStopped() { + return isStopped.get(); + } + + @Override + boolean put(Request request) throws IllegalArgumentException { + return simpleReplicate(request, false); + } + + @Override + TimestampedEntry get(Request request) throws IllegalArgumentException { + if (Objects.equals(request.getHeader(FINAL_EXECUTION_HEADER), TRUE_STRING)) { + return localDaoMediator.get(request); + } else { + int ack; + int from; + int answered = 0; + try { + ack = getAck(request); + from = getFrom(request); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + if (!isAckFromCorrect(ack, from)) { + throw new IllegalArgumentException("Wrong ack/from: " + ack + "/" + from); + } + String id = request.getParameter(ID_KEY); + request.addHeader(FINAL_EXECUTION_HEADER + TRUE_STRING); + int currentMediatorIndex = getFirstMediatorIndex(id); + TimestampedEntry lastEntry = null; + for (int i = 0; i < from; i++) { + TimestampedEntry entry = + daoMediators[currentMediatorIndex].get(request); + lastEntry = findLastEntry(lastEntry, entry); + if (entry != null) { + answered += 1; + } + currentMediatorIndex = (currentMediatorIndex + 1) % daoMediators.length; + } + return answered >= ack ? lastEntry : null; + } + } + + @Override + boolean delete(Request request) throws IllegalArgumentException { + return simpleReplicate(request, true); + } + + private static DaoMediator[] getDaoMediators(List urls, String thisUrl, LocalDaoMediator localDaoMediator) { + DaoMediator[] mediators = new DaoMediator[urls.size()]; + int cnt = 0; + List tmpList = new ArrayList<>(urls); + tmpList.sort(String::compareTo); + for (String url : tmpList) { + if (url.equals(thisUrl)) { + mediators[cnt] = localDaoMediator; + } else { + mediators[cnt] = new RemoteDaoMediator(new HttpClient(new ConnectionString(url))); + } + cnt++; + } + return mediators; + } + + private static int[] getMediatorsHashes(List urls) { + int[] hashes = new int[urls.size()]; + int cnt = 0; + List tmpList = new ArrayList<>(urls); + tmpList.sort(String::compareTo); + for (String url : tmpList) { + // cantor pairing function works nicely only with non-negatives + hashes[cnt++] = Math.abs(Hash.murmur3(url)); + } + return hashes; + } + + private static TimestampedEntry findLastEntry( + TimestampedEntry a, + TimestampedEntry b + ) { + if (a == null) { + return b; + } + if (b == null) { + return a; + } + if (a.timestamp() > b.timestamp()) { + return a; + } + return b; + } + + private boolean simpleReplicate(Request request, boolean delete) + throws IllegalArgumentException { + if (Objects.equals(request.getHeader(FINAL_EXECUTION_HEADER), TRUE_STRING)) { + if (delete) { + return localDaoMediator.delete(request); + } else { + return localDaoMediator.put(request); + } + } + int ack; + int from; + int answered = 0; + try { + ack = getAck(request); + from = getFrom(request); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(e); + } + if (!isAckFromCorrect(ack, from)) { + throw new IllegalArgumentException("Wrong ack/from: " + ack + "/" + from); + } + request.addHeader(FINAL_EXECUTION_HEADER + TRUE_STRING); + String id = request.getParameter(ID_KEY); + int currentMediatorIndex = getFirstMediatorIndex(id); + for (int i = 0; i < from; i++) { + boolean res; + if (delete) { + res = daoMediators[currentMediatorIndex].delete(request); + } else { + res = daoMediators[currentMediatorIndex].put(request); + } + answered += res ? 1 : 0; + currentMediatorIndex = (currentMediatorIndex + 1) % daoMediators.length; + } + return answered >= ack; + } + + private int getAck(Request request) throws NumberFormatException { + String rawAck = request.getParameter(ACK_KEY); + return rawAck == null ? (daoMediators.length / 2) + 1 : Integer.parseInt(rawAck); + } + + private int getFrom(Request request) throws NumberFormatException { + String rawFrom = request.getParameter(FROM_KEY); + return rawFrom == null ? daoMediators.length : Integer.parseInt(rawFrom); + } + + private boolean isAckFromCorrect(int ack, int from) { + return ack > 0 && from > 0 && ack <= from && from <= daoMediators.length; + } + + private int getFirstMediatorIndex(String key) { + int maxHash = Integer.MIN_VALUE; + int choosen = 0; + int keyHash = Math.abs(Hash.murmur3(key)); + for (int i = 0; i < mediatorsHashes.length; i++) { + // cantor pairing function works nicely only with non-negatives + int totalHash = (mediatorsHashes[i] + keyHash) * (mediatorsHashes[i] + keyHash + 1) / 2 + keyHash; + if (totalHash > maxHash) { + maxHash = totalHash; + choosen = i; + } + } + return choosen; + } + +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/RemoteDaoMediator.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/RemoteDaoMediator.java new file mode 100644 index 000000000..c2559433f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/RemoteDaoMediator.java @@ -0,0 +1,101 @@ +package ru.vk.itmo.test.emelyanovvitaliy; + +import one.nio.http.HttpClient; +import one.nio.http.HttpException; +import one.nio.http.Request; +import one.nio.http.Response; +import one.nio.pool.PoolException; +import ru.vk.itmo.test.emelyanovvitaliy.dao.TimestampedEntry; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; + +import static ru.vk.itmo.test.emelyanovvitaliy.DhtServer.TIMESTAMP_HEADER; + +public class RemoteDaoMediator extends DaoMediator { + protected static final int DEFAULT_TIMEOUT = 100; + protected final HttpClient client; + protected final int timeout; + + RemoteDaoMediator(HttpClient httpClient) { + this(httpClient, DEFAULT_TIMEOUT); + } + + RemoteDaoMediator(HttpClient httpClient, int timeout) { + this.client = httpClient; + this.timeout = timeout; + } + + protected boolean isStatusCodeCorrect(int statusCode) { + return statusCode >= 200 && statusCode < 300; + } + + @Override + void stop() { + if (!client.isClosed()) { + client.close(); + } + } + + @Override + boolean isStopped() { + return client.isClosed(); + } + + @Override + protected boolean put(Request request) { + return simpleForward(request); + } + + protected boolean simpleForward(Request request) { + try { + Response response = client.invoke(request); + return isStatusCodeCorrect(response.getStatus()); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UncheckedInterruptedException(e); + } catch (PoolException | IOException | HttpException e) { + return false; + } + } + + @Override + TimestampedEntry get(Request request) { + Response response = null; + try { + response = client.invoke(request, timeout); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new UncheckedInterruptedException(e); + } catch (PoolException | IOException | HttpException e) { + return null; + } + String timestampHeader = response.getHeader(TIMESTAMP_HEADER); + long timestamp; + if (timestampHeader == null) { + timestamp = NEVER_TIMESTAMP; + } else { + timestamp = Long.parseLong(timestampHeader); + } + if (isStatusCodeCorrect(response.getStatus()) && timestampHeader != null) { + return new TimestampedEntry<>( + keyFor(request.getParameter(DhtServer.ID_KEY)), + MemorySegment.ofArray(response.getBody()), + timestamp + ); + } else if (response.getStatus() == 404) { + return new TimestampedEntry<>( + keyFor(request.getParameter(DhtServer.ID_KEY)), + null, + timestamp + ); + } else { + return null; + } + } + + @Override + boolean delete(Request request) { + return simpleForward(request); + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/ByteArraySegment.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/ByteArraySegment.java new file mode 100644 index 000000000..340d39c81 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/ByteArraySegment.java @@ -0,0 +1,48 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.nio.ByteBuffer; + +/** + * Growable buffer with {@link ByteBuffer} and {@link MemorySegment} interface. + * + * @author incubos + */ +final class ByteArraySegment { + private byte[] array; + private MemorySegment segment; + + ByteArraySegment(final int capacity) { + this.array = new byte[capacity]; + this.segment = MemorySegment.ofArray(array); + } + + void withArray(final ArrayConsumer consumer) throws IOException { + consumer.process(array); + } + + MemorySegment segment() { + return segment; + } + + void ensureCapacity(final long size) { + if (size > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Too big!"); + } + + final int capacity = (int) size; + if (array.length >= capacity) { + return; + } + + // Grow to the nearest bigger power of 2 + final int newSize = Integer.highestOneBit(capacity) << 1; + array = new byte[newSize]; + segment = MemorySegment.ofArray(array); + } + + interface ArrayConsumer { + void process(byte[] array) throws IOException; + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/LiveFilteringIterator.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/LiveFilteringIterator.java new file mode 100644 index 000000000..3775a7919 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/LiveFilteringIterator.java @@ -0,0 +1,52 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Filters non tombstone {@link Entry}s. + * + * @author incubos + */ +final class LiveFilteringIterator implements Iterator> { + private final Iterator> delegate; + private TimestampedEntry next; + + LiveFilteringIterator(final Iterator> delegate) { + this.delegate = delegate; + skipTombstones(); + } + + private void skipTombstones() { + while (delegate.hasNext()) { + final TimestampedEntry entry = delegate.next(); + if (entry.value() != null) { + this.next = entry; + break; + } + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public TimestampedEntry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Consume + final TimestampedEntry result = next; + next = null; + + skipTombstones(); + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MemTable.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MemTable.java new file mode 100644 index 000000000..31f23e29b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MemTable.java @@ -0,0 +1,47 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Memory table. + * + * @author incubos + */ +final class MemTable { + private final NavigableMap> map = + new ConcurrentSkipListMap<>( + MemorySegmentComparator.INSTANCE); + + boolean isEmpty() { + return map.isEmpty(); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + if (from == null && to == null) { + // All + return map.values().iterator(); + } else if (from == null) { + // Head + return map.headMap(to).values().iterator(); + } else if (to == null) { + // Tail + return map.tailMap(from).values().iterator(); + } else { + // Slice + return map.subMap(from, to).values().iterator(); + } + } + + TimestampedEntry get(final MemorySegment key) { + return map.get(key); + } + + TimestampedEntry upsert(final TimestampedEntry entry) { + return map.put(entry.key(), entry); + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MemorySegmentComparator.java new file mode 100644 index 000000000..6ad2e6d2c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MemorySegmentComparator.java @@ -0,0 +1,89 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +/** + * Compares {@link MemorySegment}s. + * + * @author incubos + */ +final class MemorySegmentComparator implements Comparator { + static final Comparator INSTANCE = + new MemorySegmentComparator(); + + private MemorySegmentComparator() { + // Singleton + } + + @Override + public int compare( + final MemorySegment left, + final MemorySegment right) { + final long mismatch = left.mismatch(right); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == left.byteSize()) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == right.byteSize()) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + left.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch), + right.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch)); + } + + static int compare( + final MemorySegment srcSegment, + final long srcFromOffset, + final long srcLength, + final MemorySegment dstSegment, + final long dstFromOffset, + final long dstLength) { + final long mismatch = + MemorySegment.mismatch( + srcSegment, + srcFromOffset, + srcFromOffset + srcLength, + dstSegment, + dstFromOffset, + dstFromOffset + dstLength); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == srcLength) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == dstLength) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + srcSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + srcFromOffset + mismatch), + dstSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + dstFromOffset + mismatch)); + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MergingEntryIterator.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MergingEntryIterator.java new file mode 100644 index 000000000..cb6f13965 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/MergingEntryIterator.java @@ -0,0 +1,70 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +/** + * Merges entry {@link Iterator}s. + * + * @author incubos + */ +final class MergingEntryIterator implements Iterator> { + private final Queue iterators; + + MergingEntryIterator(final List iterators) { + assert iterators.stream().allMatch(WeightedPeekingEntryIterator::hasNext); + + this.iterators = new PriorityQueue<>(iterators); + } + + @Override + public boolean hasNext() { + return !iterators.isEmpty(); + } + + @Override + public TimestampedEntry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final WeightedPeekingEntryIterator top = iterators.remove(); + final TimestampedEntry result = top.next(); + + if (top.hasNext()) { + // Not exhausted + iterators.add(top); + } + + // Remove older versions of the key + while (true) { + final WeightedPeekingEntryIterator iterator = iterators.peek(); + if (iterator == null) { + // Nothing left + break; + } + + // Skip entries with the same key + final TimestampedEntry entry = iterator.peek(); + if (MemorySegmentComparator.INSTANCE.compare(result.key(), entry.key()) != 0) { + // Reached another key + break; + } + + // Drop + iterators.remove(); + // Skip + iterator.next(); + if (iterator.hasNext()) { + // Not exhausted + iterators.add(iterator); + } + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/ReferenceDao.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/ReferenceDao.java new file mode 100644 index 000000000..01a9cf6fd --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/ReferenceDao.java @@ -0,0 +1,292 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import ru.vk.itmo.dao.Config; +import ru.vk.itmo.dao.Dao; +import ru.vk.itmo.dao.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Reference implementation of {@link Dao}. + * + * @author incubos + */ +public class ReferenceDao implements Dao> { + private final Config config; + private final Arena arena; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + // Guarded by lock + private volatile TableSet tableSet; + + private final ExecutorService flusher = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("flusher"); + return result; + }); + private final ExecutorService compactor = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("compactor"); + return result; + }); + + private final AtomicBoolean closed = new AtomicBoolean(); + + public ReferenceDao(final Config config) throws IOException { + this.config = config; + this.arena = Arena.ofShared(); + + // First complete promotion of compacted SSTables + SSTables.promote( + config.basePath(), + 0, + 1); + + this.tableSet = + TableSet.from( + SSTables.discover( + arena, + config.basePath())); + } + + @Override + public Iterator> get( + final MemorySegment from, + final MemorySegment to) { + return new LiveFilteringIterator( + tableSet.get( + from, + to)); + } + + @Override + public TimestampedEntry get(final MemorySegment key) { + // Without lock, just snapshot of table set + return tableSet.get(key); + } + + @Override + public void upsert(final TimestampedEntry entry) { + final boolean autoFlush; + lock.readLock().lock(); + try { + if (tableSet.memTableSize.get() > config.flushThresholdBytes() + && tableSet.flushingTable != null) { + throw new IllegalStateException("Can't keep up with flushing!"); + } + + // Upsert + final Entry previous = tableSet.upsert(entry); + + // Update size estimate + final long size = tableSet.memTableSize.addAndGet(sizeOf(entry) - sizeOf(previous)); + autoFlush = size > config.flushThresholdBytes(); + } finally { + lock.readLock().unlock(); + } + + if (autoFlush) { + initiateFlush(true); + } + } + + private static long sizeOf(final Entry entry) { + if (entry == null) { + return 0L; + } + + if (entry.value() == null) { + return entry.key().byteSize(); + } + + return entry.key().byteSize() + entry.value().byteSize(); + } + + private void initiateFlush(final boolean auto) { + flusher.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + if (this.tableSet.memTable.isEmpty()) { + // Nothing to flush + return; + } + + if (auto && this.tableSet.memTableSize.get() < config.flushThresholdBytes()) { + // Not enough data to flush + return; + } + + // Switch memTable to flushing + currentTableSet = this.tableSet.flushing(); + this.tableSet = currentTableSet; + } finally { + lock.writeLock().unlock(); + } + + // Write + final int sequence = currentTableSet.nextSequence(); + try { + new SSTableWriter() + .write( + config.basePath(), + sequence, + currentTableSet.flushingTable.get(null, null)); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-1); + return; + } + + // Open + final SSTable flushed; + try { + flushed = SSTables.open( + arena, + config.basePath(), + sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-2); + return; + } + + // Switch + lock.writeLock().lock(); + try { + this.tableSet = this.tableSet.flushed(flushed); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void flush() throws IOException { + initiateFlush(false); + } + + @Override + public void compact() throws IOException { + compactor.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + currentTableSet = this.tableSet; + if (currentTableSet.ssTables.size() < 2) { + // Nothing to compact + return; + } + } finally { + lock.writeLock().unlock(); + } + + // Compact to 0 + try { + new SSTableWriter() + .write( + config.basePath(), + 0, + new LiveFilteringIterator( + currentTableSet.allSSTableEntries())); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-3); + } + + // Open 0 + final SSTable compacted; + try { + compacted = + SSTables.open( + arena, + config.basePath(), + 0); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-4); + return; + } + + // Replace old SSTables with compacted one to + // keep serving requests + final Set replaced = new HashSet<>(currentTableSet.ssTables); + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + replaced, + compacted); + } finally { + lock.writeLock().unlock(); + } + + // Remove compacted SSTables starting from the oldest ones. + // If we crash, 0 contains all the data, and + // it will be promoted on reopen. + for (final SSTable ssTable : currentTableSet.ssTables.reversed()) { + try { + SSTables.remove( + config.basePath(), + ssTable.sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-5); + } + } + + // Promote zero to one (possibly replacing) + try { + SSTables.promote( + config.basePath(), + 0, + 1); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-6); + } + + // Replace promoted SSTable + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + Collections.singleton(compacted), + compacted.withSequence(1)); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void close() throws IOException { + if (closed.getAndSet(true)) { + // Already closed + return; + } + + // Maybe flush + flush(); + + // Stop all the threads + flusher.close(); + compactor.close(); + + // Close arena + arena.close(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTable.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTable.java new file mode 100644 index 000000000..9102bf2d4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTable.java @@ -0,0 +1,218 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Persistent SSTable in data file and index file. + * + * @author incubos + * @see SSTables + */ +final class SSTable { + final int sequence; + + private final MemorySegment index; + private final MemorySegment time; + private final MemorySegment data; + private final long size; + + SSTable( + final int sequence, + final MemorySegment index, + final MemorySegment time, + final MemorySegment data) { + this.sequence = sequence; + this.index = index; + this.time = time; + this.data = data; + this.size = index.byteSize() / Long.BYTES; + } + + SSTable withSequence(final int sequence) { + return new SSTable( + sequence, + index, + time, + data); + } + + /** + * Returns index of the entry if found; otherwise, (-(insertion point) - 1). + * The insertion point is defined as the point at which the key would be inserted: + * the index of the first element greater than the key, + * or size if all keys are less than the specified key. + * Note that this guarantees that the return value will be >= 0 + * if and only if the key is found. + */ + private long entryBinarySearch(final MemorySegment key) { + long low = 0L; + long high = size - 1; + + while (low <= high) { + final long mid = (low + high) >>> 1; + final long midEntryOffset = entryOffset(mid); + final long midKeyLength = getLength(midEntryOffset); + final int compare = + MemorySegmentComparator.compare( + data, + midEntryOffset + Long.BYTES, // Position at key + midKeyLength, + key, + 0L, + key.byteSize()); + + if (compare < 0) { + low = mid + 1; + } else if (compare > 0) { + high = mid - 1; + } else { + return mid; + } + } + + return -(low + 1); + } + + private long entryOffset(final long entry) { + return index.get( + ValueLayout.OfLong.JAVA_LONG, + entry * Long.BYTES); + } + + private long getTime(final long entry) { + return time.get( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + entry * Long.BYTES + ); + } + + private long getLength(final long offset) { + return data.get( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + offset); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + assert from == null || to == null || MemorySegmentComparator.INSTANCE.compare(from, to) <= 0; + + // Slice of SSTable in absolute offsets + final long fromEntry; + final long fromOffset; + final long toOffset; + + // Left offset bound + if (from == null) { + // Start from the beginning + fromEntry = 0L; + fromOffset = 0L; + } else { + fromEntry = entryBinarySearch(from); + if (fromEntry >= 0L) { + fromOffset = entryOffset(fromEntry); + } else if (-fromEntry - 1 == size) { + // No relevant data + return Collections.emptyIterator(); + } else { + // Greater but existing key found + fromOffset = entryOffset(-fromEntry - 1); + } + } + + // Right offset bound + if (to == null) { + // Up to the end + toOffset = data.byteSize(); + } else { + final long toEntry = entryBinarySearch(to); + if (toEntry >= 0L) { + toOffset = entryOffset(toEntry); + } else if (-toEntry - 1 == size) { + // Up to the end + toOffset = data.byteSize(); + } else { + // Greater but existing key found + toOffset = entryOffset(-toEntry - 1); + } + } + + return new SliceIterator(fromEntry, fromOffset, toOffset); + } + + TimestampedEntry get(final MemorySegment key) { + final long entry = entryBinarySearch(key); + if (entry < 0) { + return null; + } + + // Skip key (will reuse the argument) + long offset = entryOffset(entry); + offset += Long.BYTES + key.byteSize(); + // Extract value length + final long valueLength = getLength(offset); + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new TimestampedEntry<>(key, null, getTime(entry)); + } else { + // Get value + offset += Long.BYTES; + final MemorySegment value = data.asSlice(offset, valueLength); + return new TimestampedEntry<>(key, value, getTime(entry)); + } + } + + private final class SliceIterator implements Iterator> { + private long entry; + private long offset; + private final long toOffset; + + private SliceIterator( + final long fromEntry, + final long offset, + final long toOffset) { + this.offset = offset; + this.entry = fromEntry; + this.toOffset = toOffset; + } + + @Override + public boolean hasNext() { + return offset < toOffset; + } + + @Override + public TimestampedEntry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Read key length + final long keyLength = getLength(offset); + offset += Long.BYTES; + + // Read key + final MemorySegment key = data.asSlice(offset, keyLength); + offset += keyLength; + + // Read value length + final long valueLength = getLength(offset); + offset += Long.BYTES; + + // Read value + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new TimestampedEntry<>(key, null); + } else { + final MemorySegment value = data.asSlice(offset, valueLength); + offset += valueLength; + entry += 1; + return new TimestampedEntry<>(key, value, getTime(entry)); + } + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTableWriter.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTableWriter.java new file mode 100644 index 000000000..972ec2013 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTableWriter.java @@ -0,0 +1,185 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import ru.vk.itmo.dao.Entry; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Iterator; + +/** + * Writes {@link Entry} {@link Iterator} to SSTable on disk. + * + *

Index file {@code .index} contains {@code long} offsets to entries in data file: + * {@code [offset0, offset1, ...]} + * + *

Data file {@code .data} contains serialized entries: + * {@code } + * + *

Tombstones are encoded as {@code valueLength} {@code -1} and no subsequent value. + * + * @author incubos + */ +final class SSTableWriter { + private static final int BUFFER_SIZE = 64 * 1024; + + // Reusable buffers to eliminate allocations. + // But excessive memory copying is still there :( + // Long cell + private final ByteArraySegment longBuffer = new ByteArraySegment(Long.BYTES); + // Growable blob cell + private final ByteArraySegment blobBuffer = new ByteArraySegment(512); + + void write( + final Path baseDir, + final int sequence, + final Iterator> entries) throws IOException { + // Write to temporary files + final Path tempIndexName = SSTables.tempIndexName(baseDir, sequence); + final Path tempDataName = SSTables.tempDataName(baseDir, sequence); + final Path tempTimeName = SSTables.tempTimeName(baseDir, sequence); + + // Delete temporary files to eliminate tails + Files.deleteIfExists(tempIndexName); + Files.deleteIfExists(tempDataName); + + // Iterate in a single pass! + // Will write through FileChannel despite extra memory copying and + // no buffering (which may be implemented later). + // Looking forward to MemorySegment facilities in FileChannel! + try (OutputStream index = + new BufferedOutputStream( + new FileOutputStream( + tempIndexName.toFile()), + BUFFER_SIZE); + OutputStream data = + new BufferedOutputStream( + new FileOutputStream( + tempDataName.toFile()), + BUFFER_SIZE); + OutputStream time = + new BufferedOutputStream( + new FileOutputStream( + tempTimeName.toFile()), + BUFFER_SIZE) + ) { + long entryOffset = 0L; + + // Iterate and serialize + while (entries.hasNext()) { + // First write offset to the entry + writeLong(entryOffset, index); + + // Then write the entry + final TimestampedEntry entry = entries.next(); + writeLong(entry.timestamp(), time); + entryOffset += writeEntry(entry, data); + } + } + + // Publish files atomically + // FIRST index, LAST data + final Path indexName = + SSTables.indexName( + baseDir, + sequence); + Files.move( + tempIndexName, + indexName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + + final Path timeName = + SSTables.timeName( + baseDir, + sequence); + Files.move( + tempTimeName, + timeName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + + final Path dataName = + SSTables.dataName( + baseDir, + sequence); + Files.move( + tempDataName, + dataName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + + private void writeLong( + final long value, + final OutputStream os) throws IOException { + longBuffer.segment().set( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + 0, + value); + longBuffer.withArray(os::write); + } + + private void writeSegment( + final MemorySegment value, + final OutputStream os) throws IOException { + final long size = value.byteSize(); + blobBuffer.ensureCapacity(size); + MemorySegment.copy( + value, + 0L, + blobBuffer.segment(), + 0L, + size); + blobBuffer.withArray(array -> + os.write( + array, + 0, + (int) size)); + } + + /** + * Writes {@link Entry} to {@link FileChannel}. + * + * @return written bytes + */ + private long writeEntry( + final Entry entry, + final OutputStream os) throws IOException { + final MemorySegment key = entry.key(); + final MemorySegment value = entry.value(); + long result = 0L; + + // Key size + writeLong(key.byteSize(), os); + result += Long.BYTES; + + // Key + writeSegment(key, os); + result += key.byteSize(); + + // Value size and possibly value + if (value == null) { + // Tombstone + writeLong(SSTables.TOMBSTONE_VALUE_LENGTH, os); + result += Long.BYTES; + } else { + // Value length + writeLong(value.byteSize(), os); + result += Long.BYTES; + + // Value + writeSegment(value, os); + result += value.byteSize(); + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTables.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTables.java new file mode 100644 index 000000000..1c3c5656b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/SSTables.java @@ -0,0 +1,180 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +/** + * Provides {@link SSTable} management facilities: dumping and discovery. + * + * @author incubos + */ +final class SSTables { + public static final String INDEX_SUFFIX = ".index"; + public static final String DATA_SUFFIX = ".data"; + public static final String TIME_SUFFIX = ".time"; + public static final long TOMBSTONE_VALUE_LENGTH = -1L; + + private static final String TEMP_SUFFIX = ".tmp"; + + /** + * Can't instantiate. + */ + private SSTables() { + // Only static methods + } + + static Path indexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX); + } + + static Path dataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX); + } + + static Path timeName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + TIME_SUFFIX); + } + + static Path tempIndexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX + TEMP_SUFFIX); + } + + static Path tempDataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX + TEMP_SUFFIX); + } + + public static Path tempTimeName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + TIME_SUFFIX + TEMP_SUFFIX); + } + + /** + * Returns {@link List} of {@link SSTable}s from freshest to oldest. + */ + static List discover( + final Arena arena, + final Path baseDir) throws IOException { + if (!Files.exists(baseDir)) { + return Collections.emptyList(); + } + + final List result = new ArrayList<>(); + try (Stream files = Files.list(baseDir)) { + files.forEach(file -> { + final String fileName = file.getFileName().toString(); + if (!fileName.endsWith(DATA_SUFFIX)) { + // Skip non data + return; + } + + final int sequence = + // .data -> N + Integer.parseInt( + fileName.substring( + 0, + fileName.length() - DATA_SUFFIX.length())); + + try { + result.add(open(arena, baseDir, sequence)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + // Sort from freshest to oldest + result.sort((o1, o2) -> Integer.compare(o2.sequence, o1.sequence)); + + return Collections.unmodifiableList(result); + } + + static SSTable open( + final Arena arena, + final Path baseDir, + final int sequence) throws IOException { + final MemorySegment index = + mapReadOnly( + arena, + indexName(baseDir, sequence)); + final MemorySegment time = + mapReadOnly( + arena, + timeName(baseDir, sequence)); + final MemorySegment data = + mapReadOnly( + arena, + dataName(baseDir, sequence)); + + return new SSTable( + sequence, + index, + time, + data); + } + + private static MemorySegment mapReadOnly( + final Arena arena, + final Path file) throws IOException { + try (FileChannel channel = + FileChannel.open( + file, + StandardOpenOption.READ)) { + return channel.map( + FileChannel.MapMode.READ_ONLY, + 0L, + Files.size(file), + arena); + } + } + + static void remove( + final Path baseDir, + final int sequence) throws IOException { + // First delete data file to make SSTable invisible + Files.delete(dataName(baseDir, sequence)); + Files.delete(indexName(baseDir, sequence)); + } + + static void promote( + final Path baseDir, + final int from, + final int to) throws IOException { + // Build to progress to the same outcome + if (Files.exists(indexName(baseDir, from))) { + Files.move( + indexName(baseDir, from), + indexName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + if (Files.exists(dataName(baseDir, from))) { + Files.move( + dataName(baseDir, from), + dataName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/TableSet.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/TableSet.java new file mode 100644 index 000000000..2fc4ac1c4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/TableSet.java @@ -0,0 +1,197 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Data set in various tables. + * + * @author incubos + */ +final class TableSet { + final MemTable memTable; + final AtomicLong memTableSize; + // null or read-only + final MemTable flushingTable; + // From freshest to oldest + final List ssTables; + + private TableSet( + final MemTable memTable, + final AtomicLong memTableSize, + final MemTable flushingTable, + final List ssTables) { + this.memTable = memTable; + this.memTableSize = memTableSize; + this.flushingTable = flushingTable; + this.ssTables = ssTables; + } + + static TableSet from(final List ssTables) { + return new TableSet( + new MemTable(), + new AtomicLong(), + null, + ssTables); + } + + int nextSequence() { + return ssTables.stream() + .mapToInt(t -> t.sequence) + .max() + .orElse(0) + 1; + } + + TableSet flushing() { + if (memTable.isEmpty()) { + throw new IllegalStateException("Nothing to flush"); + } + + if (flushingTable != null) { + throw new IllegalStateException("Already flushing"); + } + + return new TableSet( + new MemTable(), + new AtomicLong(), + memTable, + ssTables); + } + + TableSet flushed(final SSTable flushed) { + final List newSSTables = new ArrayList<>(ssTables.size() + 1); + newSSTables.add(flushed); + newSSTables.addAll(ssTables); + return new TableSet( + memTable, + memTableSize, + null, + newSSTables); + } + + TableSet compacted( + final Set replaced, + final SSTable with) { + final List newSsTables = new ArrayList<>(this.ssTables.size() + 1); + + // Keep not replaced SSTables + for (final SSTable ssTable : this.ssTables) { + if (!replaced.contains(ssTable)) { + newSsTables.add(ssTable); + } + } + + // Logically the oldest one + newSsTables.add(with); + + return new TableSet( + memTable, + memTableSize, + flushingTable, + newSsTables); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + final List iterators = + new ArrayList<>(2 + ssTables.size()); + + // MemTable goes first + final Iterator> memTableIterator = + memTable.get(from, to); + if (memTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE, + memTableIterator)); + } + + // Then goes flushing + if (flushingTable != null) { + final Iterator> flushingIterator = + flushingTable.get(from, to); + if (flushingIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE + 1, + flushingIterator)); + } + } + + // Then go all the SSTables + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(from, to); + if (ssTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + } + + return switch (iterators.size()) { + case 0 -> Collections.emptyIterator(); + case 1 -> iterators.getFirst(); + default -> new MergingEntryIterator(iterators); + }; + } + + TimestampedEntry get(final MemorySegment key) { + // Slightly optimized version not to pollute the heap + + // First check MemTable + TimestampedEntry result = memTable.get(key); + if (result != null) { + return result; + } + + // Then check flushing + if (flushingTable != null) { + result = flushingTable.get(key); + if (result != null) { + return result; + } + } + + // At last check SSTables from freshest to oldest + for (final SSTable ssTable : ssTables) { + result = ssTable.get(key); + if (result != null) { + // Transform tombstone + return result; + } + } + + // Nothing found + return null; + } + + TimestampedEntry upsert(final TimestampedEntry entry) { + return memTable.upsert(entry); + } + + Iterator> allSSTableEntries() { + final List iterators = + new ArrayList<>(ssTables.size()); + + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(null, null); + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + + return new MergingEntryIterator(iterators); + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/TimestampedEntry.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/TimestampedEntry.java new file mode 100644 index 000000000..d8aee78f3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/TimestampedEntry.java @@ -0,0 +1,37 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import ru.vk.itmo.dao.Entry; + +public class TimestampedEntry implements Entry { + private final D key; + private final D value; + private final long timestamp; + + public TimestampedEntry(Entry entry) { + this(entry.key(), entry.value()); + } + + public TimestampedEntry(D key, D value) { + this(key, value, System.currentTimeMillis()); + } + + public TimestampedEntry(D key, D value, long timestamp) { + this.key = key; + this.value = value; + this.timestamp = timestamp; + } + + public long timestamp() { + return timestamp; + } + + @Override + public D key() { + return key; + } + + @Override + public D value() { + return value; + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/WeightedPeekingEntryIterator.java b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/WeightedPeekingEntryIterator.java new file mode 100644 index 000000000..c90861f66 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/dao/WeightedPeekingEntryIterator.java @@ -0,0 +1,65 @@ +package ru.vk.itmo.test.emelyanovvitaliy.dao; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Peeking {@link Iterator} wrapper. + * + * @author incubos + */ +final class WeightedPeekingEntryIterator + implements Iterator>, + Comparable { + private final int weight; + private final Iterator> delegate; + private TimestampedEntry next; + + WeightedPeekingEntryIterator( + final int weight, + final Iterator> delegate) { + this.weight = weight; + this.delegate = delegate; + this.next = delegate.hasNext() ? delegate.next() : null; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public TimestampedEntry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final TimestampedEntry result = next; + next = delegate.hasNext() ? delegate.next() : null; + return result; + } + + TimestampedEntry peek() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + return next; + } + + @Override + public int compareTo(final WeightedPeekingEntryIterator other) { + // First compare keys + int result = + MemorySegmentComparator.INSTANCE.compare( + peek().key(), + other.peek().key()); + if (result != 0) { + return result; + } + + // Then compare weights if keys are equal + return Integer.compare(weight, other.weight); + } +} diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/4.md b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/4.md new file mode 100644 index 000000000..a96805561 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/4.md @@ -0,0 +1,60 @@ +# 4 этап, отчет, Емельянов Виталий +## Запись +Профилирование проводилось под нагрузкой +20'000 RPS (точка разладки ~22'000 RPS) +скриптом, аналогичным использовавшемуся ранее +(т.е. необходимо собрать кворум 2/3). + +[Логи wrk2](wrk2_put.txt) + +![](wrk2_put.png) +Виден качественный переход после ~95.6%, выяснить его природу не удалось. + +### ЦПУ +[Профиль](put_cpu.html) + +По ЦПУ наиболее ощутим резкий рост работы, связанной с HttpClient, ожидаемый, +так как теперь требуется 1-2 Http запроса на одну запись вместо старых 0-1. +В процессе анализа профилей было замечено, что хэш ключа вычисляется для каждой ноды, +хотя это и не необходимо. Это было исправлено, но это составляет очень малую долю работы, +из-за чего сильного падения замечено не было. + +### Аллокации +[Профиль](put_alloc.html) +Видимых изменений нет + +### Локи +[Профиль](put_lock.html) +Видимых изменений нет + + +## Чтение + +Профилирование на чтение производилось под нагрузкой 50'000 RPS +(точка разладки ~55'000 RPS) скриптом, аналогичным использовавшемуся ранее +(т.е. необходимо собрать кворум 2/3). + +[Логи wrk2](wrk2_get.txt) +![](wrk2_get.png) + +### ЦПУ +[Профиль](get_cpu.html) + +Судить о росте работы HttpClient, поскольку она составляет очень малую +долю по сравнению с работой Dao (в каждой ноде SSTable'ов на 1.4 ГиБ) + +### Аллокации +[Профиль](get_alloc.html) + +Заметен рост аллокаций на TimestampedEntry (ранее - BaseEntry) +с 0.13% сэмплов до ~1%. + +### Локи +[Профиль](get_lock.html) + +При том, что сами блокировки полностью идентичны, +распределение времени между ними теперь качественно другое. +Если ранее блокировки на selector потоках составляли лишь 30% сэмплов, то +теперь - все 95%. Вероятнее всего, это связано с тем, что в этот +раз профилирование проходит ближе к точке разладки, из-за чего +ThreadPoolExecutor'ы теперь блокируют работу куда заметнее \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_alloc.html b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_alloc.html new file mode 100644 index 000000000..e6657db0b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_alloc.html @@ -0,0 +1,3158 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_cpu.html b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_cpu.html new file mode 100644 index 000000000..2ebc7ff74 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_cpu.html @@ -0,0 +1,15279 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_lock.html b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_lock.html new file mode 100644 index 000000000..47dc7c139 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/get_lock.html @@ -0,0 +1,2115 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_alloc.html b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_alloc.html new file mode 100644 index 000000000..bfee3ce2d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_alloc.html @@ -0,0 +1,6018 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_cpu.html b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_cpu.html new file mode 100644 index 000000000..0926a98a2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_cpu.html @@ -0,0 +1,15250 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_lock.html b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_lock.html new file mode 100644 index 000000000..5416c43b6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/put_lock.html @@ -0,0 +1,3268 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_get.png b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_get.png new file mode 100644 index 000000000..18c91289b Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_get.png differ diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_get.txt b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_get.txt new file mode 100644 index 000000000..c6e5d9355 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_get.txt @@ -0,0 +1,107 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 1024 connections + Thread calibration: mean lat.: 18.161ms, rate sampling interval: 84ms + Thread calibration: mean lat.: 21.714ms, rate sampling interval: 94ms + Thread calibration: mean lat.: 21.111ms, rate sampling interval: 81ms + Thread calibration: mean lat.: 17.584ms, rate sampling interval: 64ms + Thread calibration: mean lat.: 18.997ms, rate sampling interval: 76ms + Thread calibration: mean lat.: 25.712ms, rate sampling interval: 123ms + Thread calibration: mean lat.: 240.876ms, rate sampling interval: 1851ms + Thread calibration: mean lat.: 160.935ms, rate sampling interval: 1470ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 9.34ms 3.95ms 36.54ms 68.72% + Req/Sec 82.41 164.00 1.21k 88.25% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 9.22ms + 75.000% 11.81ms + 90.000% 14.11ms + 99.000% 20.56ms + 99.900% 30.50ms + 99.990% 34.24ms + 99.999% 36.58ms +100.000% 36.58ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.900 0.000000 1 1.00 + 4.383 0.100000 1806 1.11 + 5.691 0.200000 3603 1.25 + 6.947 0.300000 5401 1.43 + 8.147 0.400000 7200 1.67 + 9.215 0.500000 9003 2.00 + 9.711 0.550000 9902 2.22 + 10.215 0.600000 10800 2.50 + 10.711 0.650000 11700 2.86 + 11.247 0.700000 12599 3.33 + 11.807 0.750000 13508 4.00 + 12.087 0.775000 13956 4.44 + 12.391 0.800000 14411 5.00 + 12.727 0.825000 14860 5.71 + 13.111 0.850000 15304 6.67 + 13.559 0.875000 15750 8.00 + 13.823 0.887500 15975 8.89 + 14.111 0.900000 16202 10.00 + 14.455 0.912500 16428 11.43 + 14.815 0.925000 16652 13.33 + 15.303 0.937500 16874 16.00 + 15.591 0.943750 16988 17.78 + 15.871 0.950000 17104 20.00 + 16.231 0.956250 17211 22.86 + 16.639 0.962500 17326 26.67 + 17.103 0.968750 17436 32.00 + 17.471 0.971875 17499 35.56 + 17.775 0.975000 17551 40.00 + 18.047 0.978125 17605 45.71 + 18.575 0.981250 17661 53.33 + 19.151 0.984375 17717 64.00 + 19.423 0.985938 17747 71.11 + 19.647 0.987500 17774 80.00 + 20.159 0.989062 17803 91.43 + 20.767 0.990625 17830 106.67 + 21.391 0.992188 17858 128.00 + 21.743 0.992969 17872 142.22 + 22.271 0.993750 17886 160.00 + 23.103 0.994531 17901 182.86 + 23.743 0.995313 17914 213.33 + 24.351 0.996094 17928 256.00 + 25.087 0.996484 17935 284.44 + 25.631 0.996875 17942 320.00 + 26.287 0.997266 17949 365.71 + 26.879 0.997656 17956 426.67 + 28.111 0.998047 17963 512.00 + 28.543 0.998242 17967 568.89 + 28.991 0.998437 17970 640.00 + 29.551 0.998633 17974 731.43 + 30.031 0.998828 17977 853.33 + 30.607 0.999023 17981 1024.00 + 30.975 0.999121 17983 1137.78 + 30.991 0.999219 17984 1280.00 + 31.935 0.999316 17986 1462.86 + 32.319 0.999414 17988 1706.67 + 32.735 0.999512 17990 2048.00 + 32.991 0.999561 17991 2275.56 + 32.991 0.999609 17991 2560.00 + 33.023 0.999658 17992 2925.71 + 33.855 0.999707 17993 3413.33 + 34.207 0.999756 17995 4096.00 + 34.207 0.999780 17995 4551.11 + 34.207 0.999805 17995 5120.00 + 34.207 0.999829 17995 5851.43 + 34.239 0.999854 17996 6826.67 + 34.239 0.999878 17996 8192.00 + 36.287 0.999890 17997 9102.22 + 36.287 0.999902 17997 10240.00 + 36.287 0.999915 17997 11702.86 + 36.287 0.999927 17997 13653.33 + 36.287 0.999939 17997 16384.00 + 36.575 0.999945 17998 18204.44 + 36.575 1.000000 17998 inf +#[Mean = 9.342, StdDeviation = 3.954] +#[Max = 36.544, Total count = 17998] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 42854 requests in 20.64s, 3.99MB read + Socket errors: connect 11, read 0, write 0, timeout 8697 +Requests/sec: 2076.28 +Transfer/sec: 198.01KB diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_put.png b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_put.png new file mode 100644 index 000000000..d753e4477 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_put.png differ diff --git a/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_put.txt b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_put.txt new file mode 100644 index 000000000..b62bb81a4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/emelyanovvitaliy/reports/4/wrk2_put.txt @@ -0,0 +1,112 @@ +Running 20s test @ http://localhost:8082 + 1 threads and 64 connections + Thread calibration: mean lat.: 89.529ms, rate sampling interval: 571ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.64ms 1.93ms 22.56ms 96.05% + Req/Sec 9.07k 45.61 9.19k 87.50% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.29ms + 75.000% 1.59ms + 90.000% 1.91ms + 99.000% 13.08ms + 99.900% 17.68ms + 99.990% 20.98ms + 99.999% 22.35ms +100.000% 22.58ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.223 0.000000 1 1.00 + 0.747 0.100000 8798 1.11 + 0.930 0.200000 17580 1.25 + 1.053 0.300000 26319 1.43 + 1.170 0.400000 35164 1.67 + 1.294 0.500000 43900 2.00 + 1.360 0.550000 48279 2.22 + 1.420 0.600000 52672 2.50 + 1.473 0.650000 57029 2.86 + 1.527 0.700000 61429 3.33 + 1.588 0.750000 65849 4.00 + 1.624 0.775000 67993 4.44 + 1.665 0.800000 70206 5.00 + 1.713 0.825000 72406 5.71 + 1.762 0.850000 74584 6.67 + 1.827 0.875000 76769 8.00 + 1.865 0.887500 77872 8.89 + 1.908 0.900000 78972 10.00 + 1.958 0.912500 80066 11.43 + 2.081 0.925000 81156 13.33 + 2.385 0.937500 82244 16.00 + 2.487 0.943750 82792 17.78 + 2.595 0.950000 83342 20.00 + 2.817 0.956250 83889 22.86 + 4.215 0.962500 84436 26.67 + 6.191 0.968750 84983 32.00 + 7.227 0.971875 85257 35.56 + 8.311 0.975000 85531 40.00 + 9.391 0.978125 85806 45.71 + 10.431 0.981250 86081 53.33 + 11.455 0.984375 86357 64.00 + 11.855 0.985938 86491 71.11 + 12.311 0.987500 86628 80.00 + 12.823 0.989062 86769 91.43 + 13.231 0.990625 86902 106.67 + 13.735 0.992188 87042 128.00 + 13.983 0.992969 87109 142.22 + 14.255 0.993750 87178 160.00 + 14.591 0.994531 87247 182.86 + 14.895 0.995313 87313 213.33 + 15.263 0.996094 87382 256.00 + 15.463 0.996484 87416 284.44 + 15.623 0.996875 87450 320.00 + 15.895 0.997266 87485 365.71 + 16.151 0.997656 87521 426.67 + 16.447 0.998047 87555 512.00 + 16.655 0.998242 87570 568.89 + 16.831 0.998437 87588 640.00 + 17.087 0.998633 87605 731.43 + 17.455 0.998828 87622 853.33 + 17.727 0.999023 87644 1024.00 + 17.775 0.999121 87647 1137.78 + 17.935 0.999219 87656 1280.00 + 18.143 0.999316 87666 1462.86 + 18.319 0.999414 87673 1706.67 + 18.783 0.999512 87682 2048.00 + 18.863 0.999561 87686 2275.56 + 19.055 0.999609 87690 2560.00 + 19.183 0.999658 87695 2925.71 + 19.455 0.999707 87699 3413.33 + 19.887 0.999756 87703 4096.00 + 20.079 0.999780 87705 4551.11 + 20.255 0.999805 87707 5120.00 + 20.703 0.999829 87710 5851.43 + 20.831 0.999854 87713 6826.67 + 20.847 0.999878 87714 8192.00 + 20.975 0.999890 87715 9102.22 + 20.991 0.999902 87716 10240.00 + 21.391 0.999915 87717 11702.86 + 21.599 0.999927 87718 13653.33 + 21.695 0.999939 87719 16384.00 + 22.031 0.999945 87720 18204.44 + 22.031 0.999951 87720 20480.00 + 22.143 0.999957 87721 23405.71 + 22.143 0.999963 87721 27306.67 + 22.239 0.999969 87722 32768.00 + 22.239 0.999973 87722 36408.89 + 22.239 0.999976 87722 40960.00 + 22.351 0.999979 87723 46811.43 + 22.351 0.999982 87723 54613.33 + 22.351 0.999985 87723 65536.00 + 22.351 0.999986 87723 72817.78 + 22.351 0.999988 87723 81920.00 + 22.575 0.999989 87724 93622.86 + 22.575 1.000000 87724 inf +#[Mean = 1.637, StdDeviation = 1.932] +#[Max = 22.560, Total count = 87724] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 198020 requests in 20.00s, 12.65MB read + Socket errors: connect 0, read 0, write 0, timeout 264 +Requests/sec: 9901.05 +Transfer/sec: 647.82KB diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/Histogram.png b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/Histogram.png new file mode 100644 index 000000000..502fa1ba3 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/Histogram.png differ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/get.txt b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/get.txt new file mode 100644 index 000000000..e3538c10f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/get.txt @@ -0,0 +1,126 @@ +./wrk -t4 -c64 -d3m -R2000 -s get.lua -L http://localhost:8080 +Running 3m test @ http://localhost:8080 + 4 threads and 64 connections + Thread calibration: mean lat.: 1.885ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.975ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.896ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.940ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.84ms 1.46ms 112.38ms 99.04% + Req/Sec 527.10 105.56 2.00k 64.16% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.76ms + 75.000% 2.06ms + 90.000% 2.37ms + 99.000% 3.21ms + 99.900% 21.01ms + 99.990% 66.56ms + 99.999% 101.63ms +100.000% 112.45ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.212 0.000000 1 1.00 + 1.200 0.100000 34098 1.11 + 1.369 0.200000 68126 1.25 + 1.510 0.300000 102137 1.43 + 1.637 0.400000 135973 1.67 + 1.759 0.500000 170088 2.00 + 1.817 0.550000 186978 2.22 + 1.876 0.600000 204143 2.50 + 1.934 0.650000 221032 2.86 + 1.995 0.700000 238066 3.33 + 2.063 0.750000 255111 4.00 + 2.099 0.775000 263523 4.44 + 2.139 0.800000 271886 5.00 + 2.185 0.825000 280460 5.71 + 2.237 0.850000 289010 6.67 + 2.297 0.875000 297420 8.00 + 2.333 0.887500 301744 8.89 + 2.371 0.900000 305913 10.00 + 2.417 0.912500 310206 11.43 + 2.469 0.925000 314400 13.33 + 2.531 0.937500 318725 16.00 + 2.565 0.943750 320751 17.78 + 2.605 0.950000 322925 20.00 + 2.649 0.956250 325015 22.86 + 2.699 0.962500 327126 26.67 + 2.757 0.968750 329242 32.00 + 2.791 0.971875 330322 35.56 + 2.825 0.975000 331344 40.00 + 2.871 0.978125 332441 45.71 + 2.919 0.981250 333496 53.33 + 2.979 0.984375 334535 64.00 + 3.021 0.985938 335065 71.11 + 3.075 0.987500 335595 80.00 + 3.149 0.989062 336122 91.43 + 3.275 0.990625 336652 106.67 + 3.531 0.992188 337182 128.00 + 3.781 0.992969 337446 142.22 + 4.063 0.993750 337712 160.00 + 4.415 0.994531 337978 182.86 + 4.827 0.995313 338244 213.33 + 5.583 0.996094 338508 256.00 + 6.139 0.996484 338641 284.44 + 7.063 0.996875 338774 320.00 + 8.095 0.997266 338906 365.71 + 9.351 0.997656 339039 426.67 + 11.543 0.998047 339172 512.00 + 13.143 0.998242 339238 568.89 + 14.631 0.998437 339305 640.00 + 16.399 0.998633 339371 731.43 + 18.559 0.998828 339437 853.33 + 21.391 0.999023 339504 1024.00 + 23.647 0.999121 339537 1137.78 + 25.567 0.999219 339570 1280.00 + 27.599 0.999316 339603 1462.86 + 29.951 0.999414 339636 1706.67 + 33.535 0.999512 339670 2048.00 + 34.911 0.999561 339686 2275.56 + 36.383 0.999609 339704 2560.00 + 38.271 0.999658 339720 2925.71 + 40.479 0.999707 339736 3413.33 + 42.719 0.999756 339753 4096.00 + 45.439 0.999780 339761 4551.11 + 47.199 0.999805 339769 5120.00 + 48.255 0.999829 339777 5851.43 + 51.711 0.999854 339786 6826.67 + 61.567 0.999878 339794 8192.00 + 64.927 0.999890 339798 9102.22 + 67.135 0.999902 339802 10240.00 + 68.671 0.999915 339806 11702.86 + 71.423 0.999927 339811 13653.33 + 73.535 0.999939 339815 16384.00 + 74.303 0.999945 339817 18204.44 + 76.543 0.999951 339819 20480.00 + 78.399 0.999957 339821 23405.71 + 82.239 0.999963 339823 27306.67 + 87.231 0.999969 339825 32768.00 + 91.583 0.999973 339826 36408.89 + 91.647 0.999976 339827 40960.00 + 92.223 0.999979 339828 46811.43 + 92.351 0.999982 339829 54613.33 + 94.207 0.999985 339830 65536.00 + 100.351 0.999986 339831 72817.78 + 100.351 0.999988 339831 81920.00 + 101.631 0.999989 339832 93622.86 + 101.631 0.999991 339832 109226.67 + 103.999 0.999992 339834 131072.00 + 103.999 0.999993 339834 145635.56 + 103.999 0.999994 339834 163840.00 + 103.999 0.999995 339834 187245.71 + 103.999 0.999995 339834 218453.33 + 103.999 0.999996 339834 262144.00 + 103.999 0.999997 339834 291271.11 + 103.999 0.999997 339834 327680.00 + 112.447 0.999997 339835 374491.43 + 112.447 1.000000 339835 inf +#[Mean = 1.840, StdDeviation = 1.460] +#[Max = 112.384, Total count = 339835] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 359951 requests in 3.00m, 64.80MB read + Non-2xx or 3xx responses: 9462 +Requests/sec: 1999.73 +Transfer/sec: 368.64KB diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getCpu.png b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getCpu.png new file mode 100644 index 000000000..2601c1b7a Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getCpu.png differ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getLock.html b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getLock.html new file mode 100644 index 000000000..9b7bee005 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getLock.html @@ -0,0 +1,803 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

Matched:

+

 

+ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getMemory.png b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getMemory.png new file mode 100644 index 000000000..b2d2d5b6c Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getMemory.png differ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getalloc.html b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getalloc.html new file mode 100644 index 000000000..050f4e5ba --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getalloc.html @@ -0,0 +1,5689 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

Matched:

+

 

+ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getcpu.html b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getcpu.html new file mode 100644 index 000000000..2b42cc751 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/getcpu.html @@ -0,0 +1,12793 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

Matched:

+

 

+ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/put.txt b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/put.txt new file mode 100644 index 000000000..60c420525 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/put.txt @@ -0,0 +1,126 @@ +./wrk -t4 -c64 -d3m -R2000 -s put.lua -L http://localhost:8080 +Running 3m test @ http://localhost:8080 + 4 threads and 64 connections + Thread calibration: mean lat.: 2.258ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.386ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.579ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.244ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.87ms 1.99ms 147.33ms 98.59% + Req/Sec 526.75 110.67 1.78k 62.94% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.73ms + 75.000% 2.05ms + 90.000% 2.38ms + 99.000% 5.57ms + 99.900% 26.42ms + 99.990% 82.56ms + 99.999% 142.21ms +100.000% 147.46ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.212 0.000000 1 1.00 + 1.168 0.100000 34048 1.11 + 1.334 0.200000 68025 1.25 + 1.478 0.300000 102022 1.43 + 1.611 0.400000 135975 1.67 + 1.733 0.500000 170000 2.00 + 1.794 0.550000 187223 2.22 + 1.853 0.600000 204054 2.50 + 1.913 0.650000 220930 2.86 + 1.977 0.700000 237936 3.33 + 2.046 0.750000 255102 4.00 + 2.083 0.775000 263374 4.44 + 2.125 0.800000 271941 5.00 + 2.173 0.825000 280607 5.71 + 2.229 0.850000 289100 6.67 + 2.297 0.875000 297515 8.00 + 2.337 0.887500 301795 8.89 + 2.383 0.900000 306013 10.00 + 2.431 0.912500 310143 11.43 + 2.491 0.925000 314438 13.33 + 2.559 0.937500 318659 16.00 + 2.599 0.943750 320794 17.78 + 2.641 0.950000 322865 20.00 + 2.691 0.956250 324993 22.86 + 2.749 0.962500 327119 26.67 + 2.825 0.968750 329267 32.00 + 2.871 0.971875 330289 35.56 + 2.931 0.975000 331366 40.00 + 3.009 0.978125 332410 45.71 + 3.135 0.981250 333471 53.33 + 3.475 0.984375 334529 64.00 + 3.879 0.985938 335059 71.11 + 4.479 0.987500 335590 80.00 + 5.139 0.989062 336121 91.43 + 5.863 0.990625 336651 106.67 + 6.595 0.992188 337183 128.00 + 6.967 0.992969 337448 142.22 + 7.347 0.993750 337714 160.00 + 7.827 0.994531 337980 182.86 + 8.455 0.995313 338245 213.33 + 9.263 0.996094 338509 256.00 + 9.911 0.996484 338643 284.44 + 10.767 0.996875 338777 320.00 + 12.023 0.997266 338907 365.71 + 13.591 0.997656 339040 426.67 + 15.991 0.998047 339173 512.00 + 17.055 0.998242 339239 568.89 + 18.719 0.998437 339307 640.00 + 20.767 0.998633 339372 731.43 + 23.631 0.998828 339438 853.33 + 27.007 0.999023 339505 1024.00 + 29.775 0.999121 339538 1137.78 + 33.887 0.999219 339571 1280.00 + 38.559 0.999316 339604 1462.86 + 43.039 0.999414 339637 1706.67 + 47.679 0.999512 339671 2048.00 + 50.303 0.999561 339687 2275.56 + 52.927 0.999609 339704 2560.00 + 55.103 0.999658 339720 2925.71 + 57.823 0.999707 339737 3413.33 + 62.847 0.999756 339754 4096.00 + 65.279 0.999780 339762 4551.11 + 67.583 0.999805 339770 5120.00 + 70.911 0.999829 339778 5851.43 + 76.543 0.999854 339787 6826.67 + 80.895 0.999878 339795 8192.00 + 82.047 0.999890 339799 9102.22 + 83.199 0.999902 339803 10240.00 + 83.711 0.999915 339807 11702.86 + 84.223 0.999927 339812 13653.33 + 85.503 0.999939 339816 16384.00 + 85.759 0.999945 339818 18204.44 + 86.463 0.999951 339820 20480.00 + 87.103 0.999957 339822 23405.71 + 89.535 0.999963 339824 27306.67 + 98.367 0.999969 339826 32768.00 + 109.631 0.999973 339827 36408.89 + 111.423 0.999976 339828 40960.00 + 114.815 0.999979 339829 46811.43 + 115.263 0.999982 339830 54613.33 + 116.607 0.999985 339831 65536.00 + 141.183 0.999986 339832 72817.78 + 141.183 0.999988 339832 81920.00 + 142.207 0.999989 339833 93622.86 + 142.207 0.999991 339833 109226.67 + 142.463 0.999992 339834 131072.00 + 142.463 0.999993 339834 145635.56 + 142.463 0.999994 339834 163840.00 + 142.847 0.999995 339835 187245.71 + 142.847 0.999995 339835 218453.33 + 142.847 0.999996 339835 262144.00 + 142.847 0.999997 339835 291271.11 + 142.847 0.999997 339835 327680.00 + 147.455 0.999997 339836 374491.43 + 147.455 1.000000 339836 inf +#[Mean = 1.873, StdDeviation = 1.986] +#[Max = 147.328, Total count = 339836] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 359951 requests in 3.00m, 23.00MB read + Non-2xx or 3xx responses: 36 +Requests/sec: 1999.74 +Transfer/sec: 130.84KB diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putCpu.png b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putCpu.png new file mode 100644 index 000000000..c5833946f Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putCpu.png differ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putLock.html b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putLock.html new file mode 100644 index 000000000..7888b4b04 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putLock.html @@ -0,0 +1,740 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

Matched:

+

 

+ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putMemory.png b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putMemory.png new file mode 100644 index 000000000..d66d9a88b Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/putMemory.png differ diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/report.md b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/report.md new file mode 100644 index 000000000..32718b4f2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/report/stage5/report.md @@ -0,0 +1,57 @@ +# Отчёт о нагрузочном тестировании +## Этап 5 + +* Тестирование производилось при 2000 RPS(GET) и 2000 RPS(PUT) на 4 потока с одним 64 соединяниями. +* Кластер из трёх баз данных (localhost:8080, localhost:8081, localhost:8080) +* ack=2, from=3 +* flushThresholdBytes 10Mb +* База заполнена на 65 Mb всеми ключами от 0 до 100000. +* Обработкой запросов занимется ThreadPoolExecutor с очередью на 100000 задач, + пулом от 1 до 12 потоков +* Для тестирования была использована утилита wrk2. +* Для профилирования был использован async-profiler внутри IntelliJ IDEA + + +### Скрипты +* [get.lua](../scripts/get.lua) +* [put.lua](../scripts/put.lua) + +### Результаты +[Вывод wrk2 для GET](get.txt) + +[Вывод wrk2 для PUT](put.txt) + +![](Histogram.png) + + +#### Флеймграфы для GET запросов +##### CPU +![](getCpu.png) +[html](getcpu.html) + +##### Allocations +![](getMemory.png) +[html](getalloc.html) + +##### Lock +[\*тык\*](getLock.html) + + +#### Флеймграфы для PUT запросов +##### CPU +![](putCpu.png) + +##### Allocations +![](putMemory.png) + +##### Lock +[\*тык\*](putLock.html) + +### Вывод +Удалось повысить RPS по сравнению с прошлым этапом. +Всё ещё есть значительное ухудшение по сравнению с 3 этапом, +что можно объяснить бОльшим количеством нод и +там что запросы идут на все из них. + +Новая реализация не использует блокировки и поэтому в профайлере +никак не отображается. \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/service/DatabaseHttpServer.java b/src/main/java/ru/vk/itmo/test/kislovdanil/service/DatabaseHttpServer.java index 2ce2af5ac..bd3955943 100644 --- a/src/main/java/ru/vk/itmo/test/kislovdanil/service/DatabaseHttpServer.java +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/service/DatabaseHttpServer.java @@ -12,16 +12,16 @@ import ru.vk.itmo.test.kislovdanil.dao.BaseEntry; import ru.vk.itmo.test.kislovdanil.dao.Entry; import ru.vk.itmo.test.kislovdanil.dao.PersistentDao; +import ru.vk.itmo.test.kislovdanil.service.sharding.RequestsManager; import ru.vk.itmo.test.kislovdanil.service.sharding.Sharder; import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -30,7 +30,7 @@ public class DatabaseHttpServer extends HttpServer { private final PersistentDao dao; private final Sharder sharder; private static final String ENTITY_ACCESS_URL = "/v0/entity"; - private static final int CORE_POOL_SIZE = 1; + private static final int CORE_POOL_SIZE = 12; private static final int MAX_POOL_SIZE = 12; private static final int KEEP_ALIVE_TIME_MS = 50; private final ThreadPoolExecutor queryExecutor = new ThreadPoolExecutor(CORE_POOL_SIZE, MAX_POOL_SIZE, @@ -72,34 +72,30 @@ private void handleEntityRequestTask(int method, String entityKey, } MemorySegment key = fromString(entityKey); List proxiedUrls = sharder.defineRequestProxyUrls(entityKey, from); - List responses = new ArrayList<>(from); - + CompletableFuture currentNodeResponse = null; if (proxiedUrls.contains(selfUrl)) { - Response currentNodeResponse = switch (method) { + currentNodeResponse = CompletableFuture.supplyAsync(() -> switch (method) { case Request.METHOD_GET -> getEntity(key); case Request.METHOD_PUT -> putEntity(key, body); case Request.METHOD_DELETE -> deleteEntity(key); default -> new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); - }; + }); if (notProxy) { - sendResponse(currentNodeResponse, session); - return; + try { + sendResponse(currentNodeResponse.get(), session); + return; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + sendResponse(new Response(Response.INTERNAL_ERROR, Response.EMPTY), session); + } } proxiedUrls.remove(selfUrl); - responses.add(currentNodeResponse); } - - for (Future future : sharder.proxyRequest(method, entityKey, body, proxiedUrls)) { - try { - responses.add(future.get()); - } catch (ExecutionException e) { - responses.add(new Response(Response.SERVICE_UNAVAILABLE, Response.EMPTY)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - sendResponse(sharder.makeDecision(responses, acknowledge, method), session); - + List> responses = sharder.proxyRequest(method, entityKey, body, proxiedUrls); + if (currentNodeResponse != null) responses.add(currentNodeResponse); + // RequestManager appends response sending async operations inside the constructor + new RequestsManager(responses, session, acknowledge, method); } @Path(ENTITY_ACCESS_URL) diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/service/DatabaseServiceFactory.java b/src/main/java/ru/vk/itmo/test/kislovdanil/service/DatabaseServiceFactory.java index a622fcafd..e1bada441 100644 --- a/src/main/java/ru/vk/itmo/test/kislovdanil/service/DatabaseServiceFactory.java +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/service/DatabaseServiceFactory.java @@ -9,7 +9,7 @@ import java.io.UncheckedIOException; import java.nio.file.Paths; -@ServiceFactory(stage = 4) +@ServiceFactory(stage = 5) public class DatabaseServiceFactory implements ServiceFactory.Factory { @Override public Service create(ServiceConfig serverConfig) { diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/BaseSharder.java b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/BaseSharder.java index 9798c4a56..1734a7937 100644 --- a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/BaseSharder.java +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/BaseSharder.java @@ -13,17 +13,15 @@ import java.util.concurrent.CompletableFuture; public abstract class BaseSharder implements Sharder { - protected static final String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; - private static final HttpRequest.Builder requestBuilder = HttpRequest.newBuilder(); private final HttpClient client; private static final String TIMESTAMP_HEADER_LITERAL = TIMESTAMP_HEADER + " "; - + protected BaseSharder(HttpClient client) { this.client = client; } - protected Response handleProxiedResponse(int statusCode, byte[] body, Long timestamp) { + public static Response handleProxiedResponse(int statusCode, byte[] body, Long timestamp) { Response response = switch (statusCode) { case 200 -> new Response(Response.OK, body); case 201 -> new Response(Response.CREATED, Response.EMPTY); @@ -67,7 +65,9 @@ private Response handleSendingException(HttpResponse response, @Override public List> proxyRequest(int method, String entityKey, byte[] body, List baseUrls) { - List> futures = new ArrayList<>(baseUrls.size()); + /* For the case if our node participates in request it's better to allocate + room for 1 object then in worst case for 2*n */ + List> futures = new ArrayList<>(baseUrls.size() + 1); for (String baseUrl : baseUrls) { String entityUrl = baseUrl + "/v0/entity?id=" + entityKey + "¬-proxy=true"; URI uri = URI.create(entityUrl); diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/RandevouzSharder.java b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/RandevouzSharder.java index 3d176757a..a1caba577 100644 --- a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/RandevouzSharder.java +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/RandevouzSharder.java @@ -8,7 +8,7 @@ import java.util.Random; import java.util.stream.Collectors; -public class RandevouzSharder extends ReplicativeBaseSharder { +public class RandevouzSharder extends BaseSharder { private final List urls; private final Map urlsHash = new HashMap<>(); diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/ReplicativeBaseSharder.java b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/ReplicativeBaseSharder.java deleted file mode 100644 index 929fdfa65..000000000 --- a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/ReplicativeBaseSharder.java +++ /dev/null @@ -1,67 +0,0 @@ -package ru.vk.itmo.test.kislovdanil.service.sharding; - -import one.nio.http.Request; -import one.nio.http.Response; - -import java.net.http.HttpClient; -import java.util.Comparator; -import java.util.List; -import java.util.Set; -import java.util.stream.Stream; - -public abstract class ReplicativeBaseSharder extends BaseSharder { - - private static final Comparator timestampComparator = Comparator - .comparingLong(ReplicativeBaseSharder::extractTimestampHeader).reversed(); - - private static long extractTimestampHeader(Response response) { - return Long.parseLong(response.getHeader(TIMESTAMP_HEADER).substring(2)); - } - - protected ReplicativeBaseSharder(HttpClient client) { - super(client); - } - - private boolean isSuccessful(List responses, int acknowledge, Set validStatuses) { - int approved = 0; - for (Response response : responses) { - if (validStatuses.contains(response.getStatus())) { - approved++; - } - if (approved >= acknowledge) { - return true; - } - } - return false; - } - - private Response chooseResponse(List responses, Set validStatuses, boolean useTimestamps) { - Stream validResponses = responses.stream() - .filter(response -> validStatuses.contains(response.getStatus())); - Response errorResponse = new Response(Response.INTERNAL_ERROR, Response.EMPTY); - return useTimestamps ? validResponses - .max(timestampComparator).orElse(errorResponse) : - validResponses.findAny().orElse(errorResponse); - } - - private Response makeMethodDecision(List responses, - int acknowledge, Set validStatuses, - boolean useTimestamps) { - if (isSuccessful(responses, acknowledge, validStatuses)) { - return chooseResponse(responses, validStatuses, useTimestamps); - } - return new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY); - } - - @Override - public Response makeDecision(List responses, int acknowledge, int method) { - Response basicResponse = switch (method) { - case Request.METHOD_GET -> makeMethodDecision(responses, acknowledge, Set.of(200, 404), true); - case Request.METHOD_PUT -> makeMethodDecision(responses, acknowledge, Set.of(201), false); - case Request.METHOD_DELETE -> makeMethodDecision(responses, acknowledge, Set.of(202), false); - default -> new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); - }; - // To remove X-Timeout header for response for client - return handleProxiedResponse(basicResponse.getStatus(), basicResponse.getBody(), null); - } -} diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/RequestsManager.java b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/RequestsManager.java new file mode 100644 index 000000000..7c6175fa4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/RequestsManager.java @@ -0,0 +1,118 @@ +package ru.vk.itmo.test.kislovdanil.service.sharding; + +import one.nio.http.HttpSession; +import one.nio.http.Request; +import one.nio.http.Response; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static ru.vk.itmo.test.kislovdanil.service.DatabaseHttpServer.sendResponse; +import static ru.vk.itmo.test.kislovdanil.service.sharding.BaseSharder.handleProxiedResponse; +import static ru.vk.itmo.test.kislovdanil.service.sharding.Sharder.NOT_ENOUGH_REPLICAS; +import static ru.vk.itmo.test.kislovdanil.service.sharding.Sharder.TIMESTAMP_HEADER; + +// Thread-safe lock-free requests manager +public class RequestsManager { + private final HttpSession session; + private final AtomicInteger successCount = new AtomicInteger(0); + private final AtomicInteger checkedCount = new AtomicInteger(0); + private final AtomicReference actualResponse = new AtomicReference<>(); + private final int ack; + private final int method; + private final int from; + private final Iterable> futures; + private final AtomicBoolean hasResponseSent = new AtomicBoolean(false); + private static final Comparator timestampComparator = Comparator + .comparingLong(RequestsManager::extractTimestampHeader).reversed(); + private static final Map> SUCCESS_STATUSES = Map.of( + Request.METHOD_GET, Set.of(200, 404), + Request.METHOD_PUT, Set.of(201), + Request.METHOD_DELETE, Set.of(202) + ); + + @SuppressWarnings("FutureReturnValueIgnored") + public RequestsManager(Collection> futures, HttpSession session, + int ack, int method) { + this.session = session; + this.ack = ack; + this.method = method; + this.from = futures.size(); + this.futures = futures; + for (CompletableFuture future : futures) { + future.thenAccept(this::makeDecision); + } + } + + private static long extractTimestampHeader(Response response) { + if (response == null) { + return Long.MAX_VALUE; + } + return Long.parseLong(response.getHeader(TIMESTAMP_HEADER).substring(2)); + } + + private void makeMethodDecision(Response response, + Set validStatuses, + boolean useTimestamps) { + if (validStatuses.contains(response.getStatus())) { + /* Update actual response in CAS loop. Even if we get ack responses before this one + it's okay to send this response if it contains newer data */ + while (true) { + if (!useTimestamps) { + // No difference which one would be set + actualResponse.set(response); + break; + } + Response currentActualResponse = actualResponse.get(); + if (timestampComparator.compare(response, currentActualResponse) < 0 + || actualResponse.compareAndSet(currentActualResponse, response)) { + break; + } + } + int currentSuccessCount = successCount.incrementAndGet(); + if (currentSuccessCount == ack) { + final Response finalResponse = actualResponse.get(); + sendSingleResponse(handleProxiedResponse(finalResponse.getStatus(), + finalResponse.getBody(), + null)); + return; + } else if (currentSuccessCount > ack) { + return; + } + } + if (checkedCount.incrementAndGet() >= from) { + sendSingleResponse(new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY)); + } + } + + private void sendSingleResponse(Response response) { + if (hasResponseSent.compareAndSet(false, true)) { + sendResponse(response, session); + // No need to wait for all get requests + if (method == Request.METHOD_GET) { + for (CompletableFuture future : futures) { + future.cancel(true); + } + } + } + } + + public void makeDecision(Response response) { + switch (method) { + case Request.METHOD_GET -> makeMethodDecision(response, + SUCCESS_STATUSES.get(Request.METHOD_GET), true); + case Request.METHOD_PUT -> makeMethodDecision(response, + SUCCESS_STATUSES.get(Request.METHOD_PUT), false); + case Request.METHOD_DELETE -> makeMethodDecision(response, + SUCCESS_STATUSES.get(Request.METHOD_DELETE), false); + default -> sendSingleResponse(new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY)); + } + } + +} diff --git a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/Sharder.java b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/Sharder.java index 872259c60..7d9e3ad91 100644 --- a/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/Sharder.java +++ b/src/main/java/ru/vk/itmo/test/kislovdanil/service/sharding/Sharder.java @@ -7,11 +7,10 @@ public interface Sharder { String TIMESTAMP_HEADER = "X-Timestamp"; + String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; List defineRequestProxyUrls(String entityKey, int from); List> proxyRequest(int method, String entityKey, byte[] body, List baseUrls); - - Response makeDecision(List responses, int acknowledge, int method); } diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_alloc.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_alloc.html new file mode 100644 index 000000000..5a04b9877 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_alloc.html @@ -0,0 +1,1238 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_cpu.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_cpu.html new file mode 100644 index 000000000..50d12da33 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_cpu.html @@ -0,0 +1,6421 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_lock.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_lock.html new file mode 100644 index 000000000..1f254d752 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_lock.html @@ -0,0 +1,691 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_wrk.txt new file mode 100644 index 000000000..cf373ef03 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_get_34000_wrk.txt @@ -0,0 +1,157 @@ +Running 5m test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.549ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.540ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.557ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.545ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.542ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.551ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.541ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.552ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.36ms 1.27ms 19.79ms 93.17% + Req/Sec 4.48k 409.41 7.89k 84.61% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.11ms + 75.000% 1.52ms + 90.000% 2.08ms + 99.000% 7.64ms + 99.900% 10.04ms + 99.990% 17.87ms + 99.999% 18.77ms +100.000% 19.81ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.040 0.000000 1 1.00 + 0.425 0.100000 988072 1.11 + 0.625 0.200000 1974847 1.25 + 0.795 0.300000 2958430 1.43 + 0.953 0.400000 3945084 1.67 + 1.108 0.500000 4934359 2.00 + 1.186 0.550000 5426130 2.22 + 1.265 0.600000 5916751 2.50 + 1.346 0.650000 6412566 2.86 + 1.429 0.700000 6901384 3.33 + 1.524 0.750000 7396247 4.00 + 1.579 0.775000 7641735 4.44 + 1.642 0.800000 7889099 5.00 + 1.715 0.825000 8135245 5.71 + 1.802 0.850000 8378924 6.67 + 1.915 0.875000 8625217 8.00 + 1.988 0.887500 8749437 8.89 + 2.079 0.900000 8871631 10.00 + 2.207 0.912500 8994914 11.43 + 2.423 0.925000 9118308 13.33 + 2.851 0.937500 9241195 16.00 + 3.157 0.943750 9302810 17.78 + 3.503 0.950000 9364506 20.00 + 3.887 0.956250 9426091 22.86 + 4.319 0.962500 9487869 26.67 + 4.811 0.968750 9549550 32.00 + 5.095 0.971875 9580436 35.56 + 5.415 0.975000 9610970 40.00 + 5.783 0.978125 9641901 45.71 + 6.207 0.981250 9672613 53.33 + 6.675 0.984375 9703235 64.00 + 6.935 0.985938 9718677 71.11 + 7.203 0.987500 9734294 80.00 + 7.471 0.989062 9749454 91.43 + 7.747 0.990625 9764963 106.67 + 8.023 0.992188 9780283 128.00 + 8.171 0.992969 9788070 142.22 + 8.319 0.993750 9795829 160.00 + 8.471 0.994531 9803388 182.86 + 8.639 0.995313 9811183 213.33 + 8.823 0.996094 9819050 256.00 + 8.919 0.996484 9822751 284.44 + 9.023 0.996875 9826489 320.00 + 9.143 0.997266 9830376 365.71 + 9.279 0.997656 9834242 426.67 + 9.439 0.998047 9838146 512.00 + 9.527 0.998242 9840015 568.89 + 9.623 0.998437 9841863 640.00 + 9.743 0.998633 9843858 731.43 + 9.879 0.998828 9845753 853.33 + 10.055 0.999023 9847632 1024.00 + 10.167 0.999121 9848596 1137.78 + 10.303 0.999219 9849566 1280.00 + 10.471 0.999316 9850544 1462.86 + 10.703 0.999414 9851497 1706.67 + 11.087 0.999512 9852443 2048.00 + 11.519 0.999561 9852929 2275.56 + 12.687 0.999609 9853406 2560.00 + 14.143 0.999658 9853885 2925.71 + 15.527 0.999707 9854370 3413.33 + 16.559 0.999756 9854856 4096.00 + 16.959 0.999780 9855093 4551.11 + 17.231 0.999805 9855342 5120.00 + 17.407 0.999829 9855570 5851.43 + 17.583 0.999854 9855817 6826.67 + 17.727 0.999878 9856060 8192.00 + 17.807 0.999890 9856175 9102.22 + 17.887 0.999902 9856305 10240.00 + 17.967 0.999915 9856416 11702.86 + 18.063 0.999927 9856559 13653.33 + 18.143 0.999939 9856664 16384.00 + 18.191 0.999945 9856718 18204.44 + 18.239 0.999951 9856780 20480.00 + 18.303 0.999957 9856853 23405.71 + 18.351 0.999963 9856898 27306.67 + 18.415 0.999969 9856954 32768.00 + 18.463 0.999973 9856988 36408.89 + 18.495 0.999976 9857020 40960.00 + 18.559 0.999979 9857053 46811.43 + 18.607 0.999982 9857082 54613.33 + 18.639 0.999985 9857106 65536.00 + 18.687 0.999986 9857123 72817.78 + 18.719 0.999988 9857138 81920.00 + 18.751 0.999989 9857149 93622.86 + 18.783 0.999991 9857164 109226.67 + 18.831 0.999992 9857180 131072.00 + 18.879 0.999993 9857189 145635.56 + 18.911 0.999994 9857203 163840.00 + 18.911 0.999995 9857203 187245.71 + 18.943 0.999995 9857210 218453.33 + 18.991 0.999996 9857217 262144.00 + 19.055 0.999997 9857222 291271.11 + 19.071 0.999997 9857224 327680.00 + 19.135 0.999997 9857229 374491.43 + 19.183 0.999998 9857233 436906.67 + 19.215 0.999998 9857236 524288.00 + 19.247 0.999998 9857238 582542.22 + 19.279 0.999998 9857240 655360.00 + 19.295 0.999999 9857241 748982.86 + 19.327 0.999999 9857243 873813.33 + 19.407 0.999999 9857245 1048576.00 + 19.423 0.999999 9857246 1165084.44 + 19.439 0.999999 9857247 1310720.00 + 19.487 0.999999 9857248 1497965.71 + 19.535 0.999999 9857249 1747626.67 + 19.567 1.000000 9857251 2097152.00 + 19.567 1.000000 9857251 2330168.89 + 19.567 1.000000 9857251 2621440.00 + 19.567 1.000000 9857251 2995931.43 + 19.679 1.000000 9857252 3495253.33 + 19.679 1.000000 9857252 4194304.00 + 19.679 1.000000 9857252 4660337.78 + 19.711 1.000000 9857253 5242880.00 + 19.711 1.000000 9857253 5991862.86 + 19.711 1.000000 9857253 6990506.67 + 19.711 1.000000 9857253 8388608.00 + 19.711 1.000000 9857253 9320675.55 + 19.807 1.000000 9857254 10485760.00 + 19.807 1.000000 9857254 inf +#[Mean = 1.355, StdDeviation = 1.268] +#[Max = 19.792, Total count = 9857254] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 10198723 requests in 5.00m, 1.99GB read + Non-2xx or 3xx responses: 5258919 +Requests/sec: 33995.77 +Transfer/sec: 6.78MB +------------------------------ + +HTTP Status 404 Count: 5258919 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_alloc.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_alloc.html new file mode 100644 index 000000000..aab74d698 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_alloc.html @@ -0,0 +1,1238 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_cpu.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_cpu.html new file mode 100644 index 000000000..e3ca0cacf --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_cpu.html @@ -0,0 +1,5384 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_lock.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_lock.html new file mode 100644 index 000000000..3939be84a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_lock.html @@ -0,0 +1,638 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_wrk.txt new file mode 100644 index 000000000..ef8b03125 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_16_stable_put_30000_wrk.txt @@ -0,0 +1,151 @@ +Running 5m test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.695ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.691ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.701ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.690ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.670ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.701ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.705ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.673ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.72ms 1.95ms 40.61ms 96.29% + Req/Sec 3.95k 392.61 12.50k 80.76% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.40ms + 75.000% 1.90ms + 90.000% 2.43ms + 99.000% 10.30ms + 99.900% 27.82ms + 99.990% 34.62ms + 99.999% 37.47ms +100.000% 40.64ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.045 0.000000 1 1.00 + 0.680 0.100000 873035 1.11 + 0.883 0.200000 1743027 1.25 + 1.056 0.300000 2613258 1.43 + 1.224 0.400000 3482644 1.67 + 1.396 0.500000 4353126 2.00 + 1.485 0.550000 4783844 2.22 + 1.579 0.600000 5220810 2.50 + 1.678 0.650000 5656052 2.86 + 1.785 0.700000 6089607 3.33 + 1.905 0.750000 6526467 4.00 + 1.971 0.775000 6742355 4.44 + 2.043 0.800000 6959960 5.00 + 2.121 0.825000 7177864 5.71 + 2.207 0.850000 7394676 6.67 + 2.307 0.875000 7612382 8.00 + 2.365 0.887500 7720905 8.89 + 2.431 0.900000 7828658 10.00 + 2.511 0.912500 7937051 11.43 + 2.613 0.925000 8045297 13.33 + 2.757 0.937500 8154190 16.00 + 2.861 0.943750 8208365 17.78 + 3.011 0.950000 8262789 20.00 + 3.223 0.956250 8317039 22.86 + 3.629 0.962500 8371485 26.67 + 4.671 0.968750 8425768 32.00 + 5.295 0.971875 8453088 35.56 + 5.939 0.975000 8480264 40.00 + 6.631 0.978125 8507393 45.71 + 7.399 0.981250 8534562 53.33 + 8.287 0.984375 8561787 64.00 + 8.783 0.985938 8575347 71.11 + 9.319 0.987500 8588951 80.00 + 9.903 0.989062 8602532 91.43 + 10.575 0.990625 8616037 106.67 + 11.399 0.992188 8629676 128.00 + 11.879 0.992969 8636439 142.22 + 12.439 0.993750 8643246 160.00 + 13.111 0.994531 8650055 182.86 + 13.951 0.995313 8656819 213.33 + 15.199 0.996094 8663586 256.00 + 16.143 0.996484 8666994 284.44 + 17.407 0.996875 8670377 320.00 + 19.023 0.997266 8673790 365.71 + 20.863 0.997656 8677169 426.67 + 22.815 0.998047 8680578 512.00 + 23.759 0.998242 8682279 568.89 + 24.703 0.998437 8683984 640.00 + 25.679 0.998633 8685680 731.43 + 26.799 0.998828 8687364 853.33 + 27.967 0.999023 8689075 1024.00 + 28.591 0.999121 8689910 1137.78 + 29.199 0.999219 8690760 1280.00 + 29.823 0.999316 8691611 1462.86 + 30.463 0.999414 8692455 1706.67 + 31.151 0.999512 8693309 2048.00 + 31.503 0.999561 8693737 2275.56 + 31.871 0.999609 8694165 2560.00 + 32.223 0.999658 8694580 2925.71 + 32.607 0.999707 8695016 3413.33 + 33.023 0.999756 8695440 4096.00 + 33.247 0.999780 8695657 4551.11 + 33.471 0.999805 8695856 5120.00 + 33.695 0.999829 8696068 5851.43 + 33.983 0.999854 8696279 6826.67 + 34.303 0.999878 8696490 8192.00 + 34.495 0.999890 8696617 9102.22 + 34.623 0.999902 8696700 10240.00 + 34.815 0.999915 8696810 11702.86 + 35.007 0.999927 8696914 13653.33 + 35.231 0.999939 8697034 16384.00 + 35.327 0.999945 8697079 18204.44 + 35.423 0.999951 8697127 20480.00 + 35.583 0.999957 8697181 23405.71 + 35.775 0.999963 8697238 27306.67 + 35.967 0.999969 8697286 32768.00 + 36.095 0.999973 8697312 36408.89 + 36.255 0.999976 8697338 40960.00 + 36.575 0.999979 8697366 46811.43 + 36.767 0.999982 8697391 54613.33 + 36.927 0.999985 8697419 65536.00 + 37.087 0.999986 8697430 72817.78 + 37.247 0.999988 8697447 81920.00 + 37.439 0.999989 8697458 93622.86 + 37.567 0.999991 8697472 109226.67 + 37.663 0.999992 8697483 131072.00 + 37.791 0.999993 8697494 145635.56 + 37.855 0.999994 8697498 163840.00 + 37.887 0.999995 8697503 187245.71 + 38.047 0.999995 8697512 218453.33 + 38.143 0.999996 8697516 262144.00 + 38.335 0.999997 8697520 291271.11 + 38.431 0.999997 8697523 327680.00 + 38.527 0.999997 8697526 374491.43 + 38.687 0.999998 8697530 436906.67 + 38.943 0.999998 8697534 524288.00 + 39.007 0.999998 8697535 582542.22 + 39.103 0.999998 8697536 655360.00 + 39.423 0.999999 8697539 748982.86 + 39.519 0.999999 8697540 873813.33 + 39.583 0.999999 8697541 1048576.00 + 39.711 0.999999 8697542 1165084.44 + 39.775 0.999999 8697543 1310720.00 + 39.839 0.999999 8697544 1497965.71 + 39.935 0.999999 8697545 1747626.67 + 39.935 1.000000 8697545 2097152.00 + 40.255 1.000000 8697546 2330168.89 + 40.255 1.000000 8697546 2621440.00 + 40.575 1.000000 8697547 2995931.43 + 40.575 1.000000 8697547 3495253.33 + 40.575 1.000000 8697547 4194304.00 + 40.607 1.000000 8697548 4660337.78 + 40.607 1.000000 8697548 5242880.00 + 40.607 1.000000 8697548 5991862.86 + 40.607 1.000000 8697548 6990506.67 + 40.607 1.000000 8697548 8388608.00 + 40.639 1.000000 8697549 9320675.55 + 40.639 1.000000 8697549 inf +#[Mean = 1.716, StdDeviation = 1.949] +#[Max = 40.608, Total count = 8697549] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 8998838 requests in 5.00m, 768.01MB read +Requests/sec: 29996.32 +Transfer/sec: 2.56MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_alloc.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_alloc.html new file mode 100644 index 000000000..11390ff01 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_alloc.html @@ -0,0 +1,1143 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_cpu.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_cpu.html new file mode 100644 index 000000000..2138f7fde --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_cpu.html @@ -0,0 +1,7165 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_lock.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_lock.html new file mode 100644 index 000000000..b27d5709b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_lock.html @@ -0,0 +1,672 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_wrk.txt new file mode 100644 index 000000000..686985894 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_get_wrk.txt @@ -0,0 +1,161 @@ +Running 5m test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.535ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.644ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.536ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.539ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.518ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.561ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.528ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.528ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.56ms 1.27ms 32.08ms 93.27% + Req/Sec 7.90k 805.23 31.67k 84.77% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.34ms + 75.000% 1.85ms + 90.000% 2.40ms + 99.000% 7.93ms + 99.900% 12.30ms + 99.990% 21.52ms + 99.999% 26.35ms +100.000% 32.10ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.030 0.000000 1 1.00 + 0.607 0.100000 1741347 1.11 + 0.813 0.200000 3481797 1.25 + 0.990 0.300000 5221775 1.43 + 1.163 0.400000 6965472 1.67 + 1.338 0.500000 8706040 2.00 + 1.428 0.550000 9574916 2.22 + 1.521 0.600000 10441486 2.50 + 1.620 0.650000 11313689 2.86 + 1.726 0.700000 12178036 3.33 + 1.846 0.750000 13049263 4.00 + 1.913 0.775000 13483680 4.44 + 1.987 0.800000 13919403 5.00 + 2.069 0.825000 14354928 5.71 + 2.161 0.850000 14789924 6.67 + 2.269 0.875000 15224740 8.00 + 2.333 0.887500 15444200 8.89 + 2.405 0.900000 15659809 10.00 + 2.489 0.912500 15874271 11.43 + 2.593 0.925000 16093150 13.33 + 2.727 0.937500 16309011 16.00 + 2.815 0.943750 16418522 17.78 + 2.925 0.950000 16525593 20.00 + 3.075 0.956250 16634588 22.86 + 3.299 0.962500 16743274 26.67 + 3.739 0.968750 16851844 32.00 + 4.139 0.971875 16906306 35.56 + 4.667 0.975000 16960633 40.00 + 5.271 0.978125 17014860 45.71 + 5.915 0.981250 17069233 53.33 + 6.583 0.984375 17123652 64.00 + 6.935 0.985938 17150876 71.11 + 7.299 0.987500 17177882 80.00 + 7.683 0.989062 17205118 91.43 + 8.091 0.990625 17232267 106.67 + 8.527 0.992188 17259655 128.00 + 8.751 0.992969 17273067 142.22 + 8.983 0.993750 17286762 160.00 + 9.223 0.994531 17300294 182.86 + 9.479 0.995313 17314072 213.33 + 9.751 0.996094 17327737 256.00 + 9.895 0.996484 17334390 284.44 + 10.055 0.996875 17341168 320.00 + 10.231 0.997266 17348019 365.71 + 10.431 0.997656 17354658 426.67 + 10.679 0.998047 17361430 512.00 + 10.839 0.998242 17364861 568.89 + 11.031 0.998437 17368227 640.00 + 11.287 0.998633 17371635 731.43 + 11.679 0.998828 17374994 853.33 + 12.399 0.999023 17378351 1024.00 + 12.927 0.999121 17380035 1137.78 + 13.559 0.999219 17381747 1280.00 + 14.287 0.999316 17383433 1462.86 + 15.103 0.999414 17385138 1706.67 + 15.991 0.999512 17386833 2048.00 + 16.479 0.999561 17387682 2275.56 + 17.007 0.999609 17388539 2560.00 + 17.583 0.999658 17389384 2925.71 + 18.175 0.999707 17390246 3413.33 + 18.847 0.999756 17391086 4096.00 + 19.231 0.999780 17391509 4551.11 + 19.615 0.999805 17391940 5120.00 + 20.047 0.999829 17392370 5851.43 + 20.495 0.999854 17392792 6826.67 + 21.007 0.999878 17393209 8192.00 + 21.279 0.999890 17393415 9102.22 + 21.567 0.999902 17393637 10240.00 + 21.871 0.999915 17393842 11702.86 + 22.255 0.999927 17394051 13653.33 + 22.719 0.999939 17394267 16384.00 + 22.943 0.999945 17394368 18204.44 + 23.215 0.999951 17394480 20480.00 + 23.503 0.999957 17394586 23405.71 + 23.839 0.999963 17394686 27306.67 + 24.255 0.999969 17394793 32768.00 + 24.543 0.999973 17394847 36408.89 + 24.783 0.999976 17394902 40960.00 + 24.991 0.999979 17394952 46811.43 + 25.343 0.999982 17395007 54613.33 + 25.663 0.999985 17395059 65536.00 + 25.823 0.999986 17395085 72817.78 + 25.983 0.999988 17395112 81920.00 + 26.223 0.999989 17395139 93622.86 + 26.447 0.999991 17395164 109226.67 + 26.783 0.999992 17395192 131072.00 + 26.879 0.999993 17395204 145635.56 + 27.007 0.999994 17395217 163840.00 + 27.247 0.999995 17395233 187245.71 + 27.407 0.999995 17395245 218453.33 + 27.647 0.999996 17395257 262144.00 + 27.871 0.999997 17395265 291271.11 + 27.983 0.999997 17395271 327680.00 + 28.175 0.999997 17395277 374491.43 + 28.319 0.999998 17395284 436906.67 + 28.527 0.999998 17395291 524288.00 + 28.655 0.999998 17395294 582542.22 + 28.943 0.999998 17395299 655360.00 + 28.991 0.999999 17395300 748982.86 + 29.167 0.999999 17395306 873813.33 + 29.199 0.999999 17395307 1048576.00 + 29.375 0.999999 17395309 1165084.44 + 29.535 0.999999 17395310 1310720.00 + 29.679 0.999999 17395312 1497965.71 + 29.727 0.999999 17395314 1747626.67 + 29.775 1.000000 17395316 2097152.00 + 29.775 1.000000 17395316 2330168.89 + 29.807 1.000000 17395317 2621440.00 + 29.839 1.000000 17395318 2995931.43 + 30.351 1.000000 17395319 3495253.33 + 30.351 1.000000 17395319 4194304.00 + 30.607 1.000000 17395320 4660337.78 + 30.607 1.000000 17395320 5242880.00 + 30.959 1.000000 17395321 5991862.86 + 30.959 1.000000 17395321 6990506.67 + 30.959 1.000000 17395321 8388608.00 + 31.151 1.000000 17395322 9320675.55 + 31.151 1.000000 17395322 10485760.00 + 31.151 1.000000 17395322 11983725.71 + 31.151 1.000000 17395322 13981013.34 + 31.151 1.000000 17395322 16777216.00 + 32.095 1.000000 17395323 18641351.10 + 32.095 1.000000 17395323 inf +#[Mean = 1.557, StdDeviation = 1.268] +#[Max = 32.080, Total count = 17395323] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 17997675 requests in 5.00m, 3.80GB read + Non-2xx or 3xx responses: 6306857 +Requests/sec: 59992.28 +Transfer/sec: 12.97MB +------------------------------ + +HTTP Status 404 Count: 6306857 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_alloc.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_alloc.html new file mode 100644 index 000000000..260d2a613 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_alloc.html @@ -0,0 +1,1184 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_cpu.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_cpu.html new file mode 100644 index 000000000..b41001f9e --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_cpu.html @@ -0,0 +1,5849 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_lock.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_lock.html new file mode 100644 index 000000000..2186d6f28 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_lock.html @@ -0,0 +1,690 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_wrk.txt new file mode 100644 index 000000000..2067ddb05 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_1_put_wrk.txt @@ -0,0 +1,154 @@ +Running 5m test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.464ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.466ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.405ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.418ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.543ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.428ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.518ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.398ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.60ms 2.31ms 106.11ms 97.97% + Req/Sec 6.33k 655.11 22.56k 80.14% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.34ms + 75.000% 1.85ms + 90.000% 2.37ms + 99.000% 6.70ms + 99.900% 36.35ms + 99.990% 76.54ms + 99.999% 102.97ms +100.000% 106.18ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.037 0.000000 2 1.00 + 0.620 0.100000 1396356 1.11 + 0.817 0.200000 2789696 1.25 + 0.988 0.300000 4178117 1.43 + 1.159 0.400000 5574397 1.67 + 1.336 0.500000 6965553 2.00 + 1.430 0.550000 7661013 2.22 + 1.527 0.600000 8356277 2.50 + 1.627 0.650000 9048574 2.86 + 1.735 0.700000 9745853 3.33 + 1.853 0.750000 10439525 4.00 + 1.917 0.775000 10785010 4.44 + 1.987 0.800000 11134152 5.00 + 2.065 0.825000 11485236 5.71 + 2.151 0.850000 11833490 6.67 + 2.249 0.875000 12181438 8.00 + 2.305 0.887500 12356395 8.89 + 2.367 0.900000 12528632 10.00 + 2.439 0.912500 12702015 11.43 + 2.525 0.925000 12875882 13.33 + 2.631 0.937500 13047369 16.00 + 2.697 0.943750 13133713 17.78 + 2.777 0.950000 13220513 20.00 + 2.877 0.956250 13307637 22.86 + 3.011 0.962500 13394773 26.67 + 3.203 0.968750 13481248 32.00 + 3.337 0.971875 13525168 35.56 + 3.511 0.975000 13568223 40.00 + 3.753 0.978125 13611717 45.71 + 4.099 0.981250 13655580 53.33 + 4.607 0.984375 13698692 64.00 + 4.963 0.985938 13720554 71.11 + 5.431 0.987500 13742259 80.00 + 6.111 0.989062 13763947 91.43 + 7.179 0.990625 13785686 106.67 + 8.735 0.992188 13807456 128.00 + 9.711 0.992969 13818291 142.22 + 10.839 0.993750 13829180 160.00 + 12.223 0.994531 13840056 182.86 + 13.831 0.995313 13850920 213.33 + 15.783 0.996094 13861788 256.00 + 16.927 0.996484 13867250 284.44 + 18.287 0.996875 13872679 320.00 + 19.935 0.997266 13878082 365.71 + 22.079 0.997656 13883517 426.67 + 24.751 0.998047 13888982 512.00 + 26.271 0.998242 13891669 568.89 + 27.951 0.998437 13894382 640.00 + 30.095 0.998633 13897108 731.43 + 32.895 0.998828 13899840 853.33 + 36.895 0.999023 13902540 1024.00 + 39.231 0.999121 13903909 1137.78 + 41.375 0.999219 13905260 1280.00 + 43.519 0.999316 13906622 1462.86 + 45.567 0.999414 13907977 1706.67 + 48.095 0.999512 13909344 2048.00 + 49.887 0.999561 13910017 2275.56 + 51.807 0.999609 13910693 2560.00 + 53.599 0.999658 13911375 2925.71 + 55.679 0.999707 13912051 3413.33 + 58.527 0.999756 13912732 4096.00 + 60.831 0.999780 13913069 4551.11 + 63.551 0.999805 13913407 5120.00 + 66.303 0.999829 13913753 5851.43 + 69.119 0.999854 13914091 6826.67 + 71.935 0.999878 13914431 8192.00 + 74.559 0.999890 13914597 9102.22 + 76.927 0.999902 13914767 10240.00 + 78.719 0.999915 13914936 11702.86 + 80.639 0.999927 13915111 13653.33 + 84.479 0.999939 13915275 16384.00 + 87.167 0.999945 13915360 18204.44 + 88.831 0.999951 13915445 20480.00 + 90.367 0.999957 13915536 23405.71 + 91.775 0.999963 13915617 27306.67 + 94.079 0.999969 13915700 32768.00 + 96.319 0.999973 13915743 36408.89 + 98.751 0.999976 13915786 40960.00 + 100.223 0.999979 13915827 46811.43 + 101.503 0.999982 13915871 54613.33 + 102.207 0.999985 13915914 65536.00 + 102.463 0.999986 13915934 72817.78 + 102.719 0.999988 13915959 81920.00 + 102.911 0.999989 13915978 93622.86 + 103.167 0.999991 13916004 109226.67 + 103.423 0.999992 13916019 131072.00 + 103.679 0.999993 13916034 145635.56 + 103.807 0.999994 13916045 163840.00 + 103.935 0.999995 13916054 187245.71 + 104.063 0.999995 13916066 218453.33 + 104.191 0.999996 13916074 262144.00 + 104.255 0.999997 13916077 291271.11 + 104.319 0.999997 13916082 327680.00 + 104.447 0.999997 13916092 374491.43 + 104.511 0.999998 13916096 436906.67 + 104.575 0.999998 13916098 524288.00 + 104.703 0.999998 13916106 582542.22 + 104.703 0.999998 13916106 655360.00 + 104.703 0.999999 13916106 748982.86 + 104.767 0.999999 13916110 873813.33 + 104.831 0.999999 13916111 1048576.00 + 104.895 0.999999 13916115 1165084.44 + 104.895 0.999999 13916115 1310720.00 + 104.895 0.999999 13916115 1497965.71 + 105.151 0.999999 13916119 1747626.67 + 105.151 1.000000 13916119 2097152.00 + 105.151 1.000000 13916119 2330168.89 + 105.151 1.000000 13916119 2621440.00 + 105.215 1.000000 13916120 2995931.43 + 105.407 1.000000 13916121 3495253.33 + 105.407 1.000000 13916121 4194304.00 + 105.663 1.000000 13916122 4660337.78 + 105.663 1.000000 13916122 5242880.00 + 105.663 1.000000 13916122 5991862.86 + 105.919 1.000000 13916123 6990506.67 + 105.919 1.000000 13916123 8388608.00 + 105.919 1.000000 13916123 9320675.55 + 105.919 1.000000 13916123 10485760.00 + 105.919 1.000000 13916123 11983725.71 + 106.175 1.000000 13916124 13981013.34 + 106.175 1.000000 13916124 inf +#[Mean = 1.600, StdDeviation = 2.309] +#[Max = 106.112, Total count = 13916124] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 14398154 requests in 5.00m, 0.90GB read +Requests/sec: 47993.94 +Transfer/sec: 3.07MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_alloc.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_alloc.html new file mode 100644 index 000000000..9bf7ec5f1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_alloc.html @@ -0,0 +1,1245 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_cpu.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_cpu.html new file mode 100644 index 000000000..37bb5e1d7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_cpu.html @@ -0,0 +1,4601 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_lock.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_lock.html new file mode 100644 index 000000000..de097cd3d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_lock.html @@ -0,0 +1,636 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_wrk.txt new file mode 100644 index 000000000..ce5ae6540 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_get_stable_40000_wrk.txt @@ -0,0 +1,151 @@ +Running 2m test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 3.524ms, rate sampling interval: 15ms + Thread calibration: mean lat.: 3.469ms, rate sampling interval: 14ms + Thread calibration: mean lat.: 3.449ms, rate sampling interval: 14ms + Thread calibration: mean lat.: 3.463ms, rate sampling interval: 14ms + Thread calibration: mean lat.: 3.446ms, rate sampling interval: 14ms + Thread calibration: mean lat.: 3.447ms, rate sampling interval: 14ms + Thread calibration: mean lat.: 3.444ms, rate sampling interval: 14ms + Thread calibration: mean lat.: 3.472ms, rate sampling interval: 15ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.49ms 3.18ms 71.87ms 97.75% + Req/Sec 5.18k 361.86 9.38k 89.05% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.10ms + 75.000% 1.49ms + 90.000% 1.89ms + 99.000% 13.10ms + 99.900% 44.38ms + 99.990% 66.37ms + 99.999% 69.25ms +100.000% 71.93ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.045 0.000000 2 1.00 + 0.458 0.100000 439720 1.11 + 0.644 0.200000 880855 1.25 + 0.800 0.300000 1320299 1.43 + 0.949 0.400000 1759833 1.67 + 1.097 0.500000 2199428 2.00 + 1.172 0.550000 2420322 2.22 + 1.248 0.600000 2638878 2.50 + 1.326 0.650000 2859886 2.86 + 1.406 0.700000 3079388 3.33 + 1.493 0.750000 3298905 4.00 + 1.541 0.775000 3408611 4.44 + 1.593 0.800000 3517658 5.00 + 1.651 0.825000 3627398 5.71 + 1.717 0.850000 3737414 6.67 + 1.795 0.875000 3848370 8.00 + 1.839 0.887500 3902851 8.89 + 1.889 0.900000 3957530 10.00 + 1.947 0.912500 4012095 11.43 + 2.017 0.925000 4067444 13.33 + 2.103 0.937500 4122266 16.00 + 2.157 0.943750 4149514 17.78 + 2.225 0.950000 4177653 20.00 + 2.311 0.956250 4204601 22.86 + 2.447 0.962500 4232190 26.67 + 2.729 0.968750 4259382 32.00 + 3.103 0.971875 4273137 35.56 + 3.833 0.975000 4286856 40.00 + 4.947 0.978125 4300614 45.71 + 6.651 0.981250 4314335 53.33 + 8.503 0.984375 4328081 64.00 + 9.527 0.985938 4334946 71.11 + 10.607 0.987500 4341843 80.00 + 11.887 0.989062 4348702 91.43 + 14.231 0.990625 4355552 106.67 + 16.543 0.992188 4362423 128.00 + 18.223 0.992969 4365880 142.22 + 20.687 0.993750 4369302 160.00 + 24.463 0.994531 4372719 182.86 + 28.463 0.995313 4376165 213.33 + 32.495 0.996094 4379591 256.00 + 34.143 0.996484 4381304 284.44 + 35.455 0.996875 4383047 320.00 + 36.671 0.997266 4384746 365.71 + 37.855 0.997656 4386474 426.67 + 38.943 0.998047 4388191 512.00 + 39.551 0.998242 4389058 568.89 + 40.383 0.998437 4389916 640.00 + 42.527 0.998633 4390762 731.43 + 43.615 0.998828 4391614 853.33 + 44.479 0.999023 4392478 1024.00 + 44.927 0.999121 4392901 1137.78 + 45.535 0.999219 4393336 1280.00 + 46.655 0.999316 4393761 1462.86 + 49.759 0.999414 4394187 1706.67 + 53.567 0.999512 4394615 2048.00 + 55.903 0.999561 4394831 2275.56 + 57.695 0.999609 4395044 2560.00 + 59.583 0.999658 4395262 2925.71 + 61.279 0.999707 4395473 3413.33 + 62.399 0.999756 4395690 4096.00 + 62.879 0.999780 4395799 4551.11 + 63.391 0.999805 4395903 5120.00 + 64.095 0.999829 4396010 5851.43 + 64.991 0.999854 4396122 6826.67 + 65.791 0.999878 4396226 8192.00 + 66.111 0.999890 4396278 9102.22 + 66.431 0.999902 4396337 10240.00 + 66.751 0.999915 4396388 11702.86 + 67.071 0.999927 4396442 13653.33 + 67.391 0.999939 4396495 16384.00 + 67.519 0.999945 4396520 18204.44 + 67.711 0.999951 4396559 20480.00 + 67.839 0.999957 4396574 23405.71 + 68.031 0.999963 4396606 27306.67 + 68.223 0.999969 4396627 32768.00 + 68.351 0.999973 4396641 36408.89 + 68.479 0.999976 4396656 40960.00 + 68.607 0.999979 4396675 46811.43 + 68.671 0.999982 4396683 54613.33 + 68.799 0.999985 4396697 65536.00 + 68.927 0.999986 4396704 72817.78 + 68.991 0.999988 4396708 81920.00 + 69.183 0.999989 4396716 93622.86 + 69.247 0.999991 4396723 109226.67 + 69.375 0.999992 4396729 131072.00 + 69.439 0.999993 4396733 145635.56 + 69.503 0.999994 4396737 163840.00 + 69.567 0.999995 4396740 187245.71 + 69.631 0.999995 4396743 218453.33 + 69.695 0.999996 4396745 262144.00 + 69.823 0.999997 4396746 291271.11 + 69.951 0.999997 4396749 327680.00 + 70.015 0.999997 4396750 374491.43 + 70.271 0.999998 4396752 436906.67 + 70.335 0.999998 4396753 524288.00 + 70.463 0.999998 4396754 582542.22 + 70.527 0.999998 4396755 655360.00 + 70.591 0.999999 4396756 748982.86 + 70.591 0.999999 4396756 873813.33 + 70.719 0.999999 4396757 1048576.00 + 70.847 0.999999 4396759 1165084.44 + 70.847 0.999999 4396759 1310720.00 + 70.847 0.999999 4396759 1497965.71 + 70.847 0.999999 4396759 1747626.67 + 70.847 1.000000 4396759 2097152.00 + 71.423 1.000000 4396760 2330168.89 + 71.423 1.000000 4396760 2621440.00 + 71.423 1.000000 4396760 2995931.43 + 71.423 1.000000 4396760 3495253.33 + 71.423 1.000000 4396760 4194304.00 + 71.935 1.000000 4396761 4660337.78 + 71.935 1.000000 4396761 inf +#[Mean = 1.490, StdDeviation = 3.182] +#[Max = 71.872, Total count = 4396761] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 4798471 requests in 2.00m, 0.96GB read + Non-2xx or 3xx responses: 2174015 +Requests/sec: 39987.44 +Transfer/sec: 8.15MB +------------------------------ + +HTTP Status 404 Count: 2174015 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_alloc.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_alloc.html new file mode 100644 index 000000000..1b103f90c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_alloc.html @@ -0,0 +1,1286 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_cpu.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_cpu.html new file mode 100644 index 000000000..4aec30ab4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_cpu.html @@ -0,0 +1,6892 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_lock.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_lock.html new file mode 100644 index 000000000..dbe3cf452 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_lock.html @@ -0,0 +1,714 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_wrk.txt new file mode 100644 index 000000000..0fe96dd99 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_2_stable_put_36000_wrk.txt @@ -0,0 +1,152 @@ +Running 5m test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.756ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.730ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.761ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.859ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.801ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.770ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.814ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.738ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.70ms 1.34ms 46.72ms 91.35% + Req/Sec 4.74k 518.25 16.89k 78.25% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.46ms + 75.000% 2.04ms + 90.000% 2.72ms + 99.000% 6.64ms + 99.900% 16.48ms + 99.990% 30.42ms + 99.999% 41.06ms +100.000% 46.75ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.041 0.000000 1 1.00 + 0.676 0.100000 1046642 1.11 + 0.892 0.200000 2089913 1.25 + 1.080 0.300000 3133042 1.43 + 1.266 0.400000 4178589 1.67 + 1.458 0.500000 5221280 2.00 + 1.558 0.550000 5741388 2.22 + 1.663 0.600000 6264456 2.50 + 1.775 0.650000 6785434 2.86 + 1.898 0.700000 7307339 3.33 + 2.037 0.750000 7829201 4.00 + 2.115 0.775000 8093336 4.44 + 2.197 0.800000 8349861 5.00 + 2.291 0.825000 8612089 5.71 + 2.399 0.850000 8872090 6.67 + 2.535 0.875000 9135094 8.00 + 2.619 0.887500 9265308 8.89 + 2.719 0.900000 9393906 10.00 + 2.845 0.912500 9523833 11.43 + 3.007 0.925000 9654481 13.33 + 3.207 0.937500 9784907 16.00 + 3.325 0.943750 9850415 17.78 + 3.465 0.950000 9915379 20.00 + 3.645 0.956250 9980464 22.86 + 3.889 0.962500 10046159 26.67 + 4.203 0.968750 10111227 32.00 + 4.391 0.971875 10143577 35.56 + 4.611 0.975000 10176647 40.00 + 4.867 0.978125 10209091 45.71 + 5.171 0.981250 10241353 53.33 + 5.567 0.984375 10274130 64.00 + 5.807 0.985938 10290381 71.11 + 6.087 0.987500 10306674 80.00 + 6.419 0.989062 10323061 91.43 + 6.807 0.990625 10339325 106.67 + 7.291 0.992188 10355616 128.00 + 7.579 0.992969 10363694 142.22 + 7.903 0.993750 10371869 160.00 + 8.287 0.994531 10380136 182.86 + 8.727 0.995313 10388195 213.33 + 9.271 0.996094 10396354 256.00 + 9.607 0.996484 10400374 284.44 + 9.999 0.996875 10404498 320.00 + 10.495 0.997266 10408522 365.71 + 11.143 0.997656 10412589 426.67 + 12.031 0.998047 10416674 512.00 + 12.623 0.998242 10418718 568.89 + 13.343 0.998437 10420744 640.00 + 14.255 0.998633 10422778 731.43 + 15.351 0.998828 10424825 853.33 + 16.655 0.999023 10426862 1024.00 + 17.391 0.999121 10427884 1137.78 + 18.207 0.999219 10428909 1280.00 + 19.055 0.999316 10429922 1462.86 + 20.015 0.999414 10430946 1706.67 + 21.007 0.999512 10431957 2048.00 + 21.503 0.999561 10432472 2275.56 + 22.047 0.999609 10432980 2560.00 + 22.687 0.999658 10433496 2925.71 + 23.471 0.999707 10433991 3413.33 + 24.607 0.999756 10434501 4096.00 + 25.263 0.999780 10434761 4551.11 + 26.127 0.999805 10435010 5120.00 + 27.087 0.999829 10435265 5851.43 + 28.111 0.999854 10435521 6826.67 + 29.215 0.999878 10435774 8192.00 + 29.855 0.999890 10435903 9102.22 + 30.527 0.999902 10436028 10240.00 + 31.407 0.999915 10436157 11702.86 + 32.303 0.999927 10436283 13653.33 + 33.215 0.999939 10436410 16384.00 + 33.759 0.999945 10436476 18204.44 + 34.431 0.999951 10436540 20480.00 + 34.975 0.999957 10436602 23405.71 + 35.743 0.999963 10436666 27306.67 + 36.639 0.999969 10436729 32768.00 + 37.279 0.999973 10436763 36408.89 + 37.727 0.999976 10436793 40960.00 + 38.207 0.999979 10436825 46811.43 + 38.911 0.999982 10436856 54613.33 + 39.647 0.999985 10436888 65536.00 + 39.903 0.999986 10436905 72817.78 + 40.383 0.999988 10436920 81920.00 + 40.895 0.999989 10436936 93622.86 + 41.407 0.999991 10436952 109226.67 + 41.855 0.999992 10436970 131072.00 + 42.239 0.999993 10436976 145635.56 + 42.527 0.999994 10436986 163840.00 + 42.719 0.999995 10436994 187245.71 + 43.039 0.999995 10437000 218453.33 + 43.327 0.999996 10437009 262144.00 + 43.583 0.999997 10437012 291271.11 + 43.967 0.999997 10437018 327680.00 + 44.159 0.999997 10437020 374491.43 + 44.319 0.999998 10437025 436906.67 + 44.479 0.999998 10437028 524288.00 + 44.607 0.999998 10437030 582542.22 + 44.895 0.999998 10437032 655360.00 + 45.087 0.999999 10437034 748982.86 + 45.535 0.999999 10437036 873813.33 + 45.631 0.999999 10437038 1048576.00 + 45.695 0.999999 10437040 1165084.44 + 45.695 0.999999 10437040 1310720.00 + 45.823 0.999999 10437041 1497965.71 + 45.983 0.999999 10437042 1747626.67 + 46.015 1.000000 10437043 2097152.00 + 46.015 1.000000 10437043 2330168.89 + 46.079 1.000000 10437045 2621440.00 + 46.079 1.000000 10437045 2995931.43 + 46.079 1.000000 10437045 3495253.33 + 46.079 1.000000 10437045 4194304.00 + 46.079 1.000000 10437045 4660337.78 + 46.591 1.000000 10437046 5242880.00 + 46.591 1.000000 10437046 5991862.86 + 46.591 1.000000 10437046 6990506.67 + 46.591 1.000000 10437046 8388608.00 + 46.591 1.000000 10437046 9320675.55 + 46.751 1.000000 10437047 10485760.00 + 46.751 1.000000 10437047 inf +#[Mean = 1.696, StdDeviation = 1.339] +#[Max = 46.720, Total count = 10437047] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 10798595 requests in 5.00m, 813.57MB read +Requests/sec: 35995.44 +Transfer/sec: 2.71MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_35000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_35000_wrk.txt new file mode 100644 index 000000000..a103baec3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_35000_wrk.txt @@ -0,0 +1,133 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.475ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.459ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.475ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.460ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.467ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.455ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.460ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.474ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.27ms 0.90ms 13.45ms 83.84% + Req/Sec 4.62k 375.86 7.11k 73.54% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.14ms + 75.000% 1.57ms + 90.000% 2.09ms + 99.000% 4.38ms + 99.900% 11.10ms + 99.990% 12.42ms + 99.999% 13.04ms +100.000% 13.45ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.052 0.000000 1 1.00 + 0.440 0.100000 34860 1.11 + 0.653 0.200000 69477 1.25 + 0.829 0.300000 104312 1.43 + 0.987 0.400000 138879 1.67 + 1.141 0.500000 173684 2.00 + 1.220 0.550000 191122 2.22 + 1.299 0.600000 208365 2.50 + 1.379 0.650000 225806 2.86 + 1.465 0.700000 243075 3.33 + 1.568 0.750000 260534 4.00 + 1.627 0.775000 269195 4.44 + 1.692 0.800000 277734 5.00 + 1.768 0.825000 286459 5.71 + 1.854 0.850000 295141 6.67 + 1.957 0.875000 303850 8.00 + 2.019 0.887500 308167 8.89 + 2.091 0.900000 312546 10.00 + 2.173 0.912500 316805 11.43 + 2.275 0.925000 321138 13.33 + 2.415 0.937500 325484 16.00 + 2.499 0.943750 327642 17.78 + 2.599 0.950000 329832 20.00 + 2.715 0.956250 331990 22.86 + 2.861 0.962500 334153 26.67 + 3.047 0.968750 336334 32.00 + 3.151 0.971875 337401 35.56 + 3.277 0.975000 338500 40.00 + 3.427 0.978125 339572 45.71 + 3.607 0.981250 340664 53.33 + 3.813 0.984375 341745 64.00 + 3.939 0.985938 342285 71.11 + 4.085 0.987500 342827 80.00 + 4.255 0.989062 343373 91.43 + 4.471 0.990625 343920 106.67 + 4.799 0.992188 344455 128.00 + 5.007 0.992969 344724 142.22 + 5.275 0.993750 344998 160.00 + 5.611 0.994531 345271 182.86 + 5.955 0.995313 345538 213.33 + 6.279 0.996094 345812 256.00 + 6.471 0.996484 345946 284.44 + 6.699 0.996875 346081 320.00 + 7.099 0.997266 346217 365.71 + 7.939 0.997656 346352 426.67 + 9.031 0.998047 346487 512.00 + 9.575 0.998242 346555 568.89 + 10.055 0.998437 346623 640.00 + 10.479 0.998633 346692 731.43 + 10.831 0.998828 346759 853.33 + 11.135 0.999023 346827 1024.00 + 11.239 0.999121 346861 1137.78 + 11.343 0.999219 346895 1280.00 + 11.479 0.999316 346931 1462.86 + 11.615 0.999414 346964 1706.67 + 11.751 0.999512 346997 2048.00 + 11.815 0.999561 347014 2275.56 + 11.871 0.999609 347033 2560.00 + 11.943 0.999658 347048 2925.71 + 12.015 0.999707 347064 3413.33 + 12.087 0.999756 347083 4096.00 + 12.119 0.999780 347089 4551.11 + 12.191 0.999805 347098 5120.00 + 12.263 0.999829 347107 5851.43 + 12.303 0.999854 347115 6826.67 + 12.351 0.999878 347124 8192.00 + 12.399 0.999890 347128 9102.22 + 12.455 0.999902 347134 10240.00 + 12.503 0.999915 347136 11702.86 + 12.551 0.999927 347142 13653.33 + 12.591 0.999939 347144 16384.00 + 12.607 0.999945 347147 18204.44 + 12.631 0.999951 347149 20480.00 + 12.655 0.999957 347151 23405.71 + 12.727 0.999963 347153 27306.67 + 12.887 0.999969 347158 32768.00 + 12.887 0.999973 347158 36408.89 + 12.887 0.999976 347158 40960.00 + 12.887 0.999979 347158 46811.43 + 12.935 0.999982 347159 54613.33 + 12.943 0.999985 347160 65536.00 + 12.999 0.999986 347161 72817.78 + 12.999 0.999988 347161 81920.00 + 13.039 0.999989 347162 93622.86 + 13.039 0.999991 347162 109226.67 + 13.103 0.999992 347163 131072.00 + 13.103 0.999993 347163 145635.56 + 13.103 0.999994 347163 163840.00 + 13.167 0.999995 347164 187245.71 + 13.167 0.999995 347164 218453.33 + 13.167 0.999996 347164 262144.00 + 13.167 0.999997 347164 291271.11 + 13.167 0.999997 347164 327680.00 + 13.455 0.999997 347165 374491.43 + 13.455 1.000000 347165 inf +#[Mean = 1.270, StdDeviation = 0.896] +#[Max = 13.448, Total count = 347165] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 698669 requests in 20.00s, 145.91MB read + Non-2xx or 3xx responses: 317118 +Requests/sec: 34934.65 +Transfer/sec: 7.30MB +------------------------------ + +HTTP Status 404 Count: 317118 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_36000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_36000_wrk.txt new file mode 100644 index 000000000..236bad0d7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_36000_wrk.txt @@ -0,0 +1,133 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.493ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.496ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.489ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.490ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.489ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.486ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.493ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.487ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.34ms 0.99ms 11.41ms 86.02% + Req/Sec 4.74k 442.61 8.89k 78.58% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.17ms + 75.000% 1.61ms + 90.000% 2.19ms + 99.000% 5.56ms + 99.900% 9.49ms + 99.990% 10.74ms + 99.999% 11.19ms +100.000% 11.41ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.054 0.000000 1 1.00 + 0.453 0.100000 35866 1.11 + 0.663 0.200000 71515 1.25 + 0.843 0.300000 107316 1.43 + 1.005 0.400000 142847 1.67 + 1.167 0.500000 178642 2.00 + 1.248 0.550000 196586 2.22 + 1.329 0.600000 214328 2.50 + 1.411 0.650000 232285 2.86 + 1.502 0.700000 250025 3.33 + 1.610 0.750000 267899 4.00 + 1.672 0.775000 276835 4.44 + 1.742 0.800000 285708 5.00 + 1.822 0.825000 294633 5.71 + 1.917 0.850000 303584 6.67 + 2.034 0.875000 312493 8.00 + 2.105 0.887500 317015 8.89 + 2.187 0.900000 321384 10.00 + 2.295 0.912500 325885 11.43 + 2.437 0.925000 330357 13.33 + 2.633 0.937500 334807 16.00 + 2.771 0.943750 337029 17.78 + 2.943 0.950000 339254 20.00 + 3.145 0.956250 341475 22.86 + 3.407 0.962500 343719 26.67 + 3.721 0.968750 345953 32.00 + 3.903 0.971875 347050 35.56 + 4.103 0.975000 348172 40.00 + 4.323 0.978125 349282 45.71 + 4.575 0.981250 350406 53.33 + 4.851 0.984375 351522 64.00 + 5.031 0.985938 352076 71.11 + 5.215 0.987500 352637 80.00 + 5.419 0.989062 353189 91.43 + 5.679 0.990625 353747 106.67 + 5.995 0.992188 354304 128.00 + 6.171 0.992969 354584 142.22 + 6.359 0.993750 354862 160.00 + 6.587 0.994531 355144 182.86 + 6.855 0.995313 355421 213.33 + 7.143 0.996094 355700 256.00 + 7.307 0.996484 355842 284.44 + 7.515 0.996875 355979 320.00 + 7.719 0.997266 356117 365.71 + 7.975 0.997656 356258 426.67 + 8.327 0.998047 356396 512.00 + 8.527 0.998242 356468 568.89 + 8.775 0.998437 356538 640.00 + 9.015 0.998633 356607 731.43 + 9.279 0.998828 356675 853.33 + 9.535 0.999023 356747 1024.00 + 9.663 0.999121 356781 1137.78 + 9.783 0.999219 356816 1280.00 + 9.911 0.999316 356850 1462.86 + 10.023 0.999414 356884 1706.67 + 10.159 0.999512 356923 2048.00 + 10.215 0.999561 356938 2275.56 + 10.263 0.999609 356956 2560.00 + 10.319 0.999658 356973 2925.71 + 10.383 0.999707 356990 3413.33 + 10.447 0.999756 357009 4096.00 + 10.495 0.999780 357017 4551.11 + 10.535 0.999805 357024 5120.00 + 10.591 0.999829 357033 5851.43 + 10.639 0.999854 357041 6826.67 + 10.711 0.999878 357050 8192.00 + 10.735 0.999890 357055 9102.22 + 10.743 0.999902 357059 10240.00 + 10.767 0.999915 357064 11702.86 + 10.815 0.999927 357069 13653.33 + 10.855 0.999939 357072 16384.00 + 10.879 0.999945 357074 18204.44 + 10.903 0.999951 357076 20480.00 + 10.911 0.999957 357078 23405.71 + 10.943 0.999963 357080 27306.67 + 10.975 0.999969 357083 32768.00 + 11.015 0.999973 357084 36408.89 + 11.055 0.999976 357086 40960.00 + 11.055 0.999979 357086 46811.43 + 11.071 0.999982 357087 54613.33 + 11.183 0.999985 357088 65536.00 + 11.191 0.999986 357089 72817.78 + 11.191 0.999988 357089 81920.00 + 11.215 0.999989 357090 93622.86 + 11.215 0.999991 357090 109226.67 + 11.287 0.999992 357091 131072.00 + 11.287 0.999993 357091 145635.56 + 11.287 0.999994 357091 163840.00 + 11.295 0.999995 357092 187245.71 + 11.295 0.999995 357092 218453.33 + 11.295 0.999996 357092 262144.00 + 11.295 0.999997 357092 291271.11 + 11.295 0.999997 357092 327680.00 + 11.415 0.999997 357093 374491.43 + 11.415 1.000000 357093 inf +#[Mean = 1.337, StdDeviation = 0.992] +#[Max = 11.408, Total count = 357093] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 718635 requests in 20.00s, 150.03MB read + Non-2xx or 3xx responses: 326189 +Requests/sec: 35932.08 +Transfer/sec: 7.50MB +------------------------------ + +HTTP Status 404 Count: 326189 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_38000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_38000_wrk.txt new file mode 100644 index 000000000..e59280fcb --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_38000_wrk.txt @@ -0,0 +1,134 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 2.178ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.175ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.174ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.177ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.176ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.174ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.179ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.167ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.50ms 1.30ms 13.60ms 90.85% + Req/Sec 5.01k 518.82 9.00k 82.56% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.23ms + 75.000% 1.72ms + 90.000% 2.52ms + 99.000% 7.80ms + 99.900% 11.16ms + 99.990% 12.64ms + 99.999% 13.18ms +100.000% 13.61ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.064 0.000000 2 1.00 + 0.491 0.100000 37719 1.11 + 0.701 0.200000 75483 1.25 + 0.884 0.300000 113095 1.43 + 1.056 0.400000 150854 1.67 + 1.226 0.500000 188641 2.00 + 1.312 0.550000 207336 2.22 + 1.400 0.600000 226311 2.50 + 1.492 0.650000 245190 2.86 + 1.597 0.700000 263912 3.33 + 1.724 0.750000 282808 4.00 + 1.799 0.775000 292220 4.44 + 1.885 0.800000 301640 5.00 + 1.986 0.825000 311024 5.71 + 2.111 0.850000 320471 6.67 + 2.273 0.875000 329865 8.00 + 2.381 0.887500 334564 8.89 + 2.519 0.900000 339299 10.00 + 2.695 0.912500 343960 11.43 + 2.939 0.925000 348673 13.33 + 3.267 0.937500 353378 16.00 + 3.479 0.943750 355749 17.78 + 3.725 0.950000 358095 20.00 + 4.003 0.956250 360455 22.86 + 4.319 0.962500 362825 26.67 + 4.699 0.968750 365167 32.00 + 4.915 0.971875 366337 35.56 + 5.179 0.975000 367512 40.00 + 5.491 0.978125 368690 45.71 + 5.927 0.981250 369874 53.33 + 6.627 0.984375 371045 64.00 + 6.939 0.985938 371633 71.11 + 7.263 0.987500 372222 80.00 + 7.587 0.989062 372816 91.43 + 7.939 0.990625 373401 106.67 + 8.335 0.992188 373988 128.00 + 8.559 0.992969 374282 142.22 + 8.807 0.993750 374586 160.00 + 9.079 0.994531 374873 182.86 + 9.359 0.995313 375173 213.33 + 9.623 0.996094 375460 256.00 + 9.759 0.996484 375609 284.44 + 9.919 0.996875 375763 320.00 + 10.103 0.997266 375904 365.71 + 10.279 0.997656 376051 426.67 + 10.455 0.998047 376197 512.00 + 10.559 0.998242 376272 568.89 + 10.679 0.998437 376345 640.00 + 10.823 0.998633 376422 731.43 + 10.983 0.998828 376495 853.33 + 11.175 0.999023 376565 1024.00 + 11.279 0.999121 376603 1137.78 + 11.439 0.999219 376638 1280.00 + 11.583 0.999316 376679 1462.86 + 11.759 0.999414 376712 1706.67 + 11.927 0.999512 376751 2048.00 + 11.967 0.999561 376768 2275.56 + 12.047 0.999609 376787 2560.00 + 12.119 0.999658 376807 2925.71 + 12.183 0.999707 376822 3413.33 + 12.271 0.999756 376841 4096.00 + 12.351 0.999780 376850 4551.11 + 12.423 0.999805 376861 5120.00 + 12.495 0.999829 376871 5851.43 + 12.527 0.999854 376878 6826.67 + 12.575 0.999878 376886 8192.00 + 12.623 0.999890 376891 9102.22 + 12.647 0.999902 376897 10240.00 + 12.687 0.999915 376900 11702.86 + 12.727 0.999927 376907 13653.33 + 12.743 0.999939 376910 16384.00 + 12.759 0.999945 376912 18204.44 + 12.767 0.999951 376915 20480.00 + 12.815 0.999957 376916 23405.71 + 12.895 0.999963 376919 27306.67 + 12.943 0.999969 376921 32768.00 + 12.959 0.999973 376922 36408.89 + 12.983 0.999976 376923 40960.00 + 12.991 0.999979 376924 46811.43 + 13.055 0.999982 376926 54613.33 + 13.079 0.999985 376927 65536.00 + 13.079 0.999986 376927 72817.78 + 13.183 0.999988 376928 81920.00 + 13.183 0.999989 376928 93622.86 + 13.319 0.999991 376929 109226.67 + 13.439 0.999992 376930 131072.00 + 13.439 0.999993 376930 145635.56 + 13.439 0.999994 376930 163840.00 + 13.439 0.999995 376930 187245.71 + 13.559 0.999995 376931 218453.33 + 13.559 0.999996 376931 262144.00 + 13.559 0.999997 376931 291271.11 + 13.559 0.999997 376931 327680.00 + 13.559 0.999997 376931 374491.43 + 13.607 0.999998 376932 436906.67 + 13.607 1.000000 376932 inf +#[Mean = 1.500, StdDeviation = 1.298] +#[Max = 13.600, Total count = 376932] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 758562 requests in 20.00s, 158.36MB read + Non-2xx or 3xx responses: 344497 +Requests/sec: 37928.70 +Transfer/sec: 7.92MB +------------------------------ + +HTTP Status 404 Count: 344497 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_40000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_40000_wrk.txt new file mode 100644 index 000000000..7c1296581 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_40000_wrk.txt @@ -0,0 +1,134 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.888ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.894ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.882ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.881ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.879ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.884ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.873ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.873ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.97ms 2.26ms 19.90ms 90.55% + Req/Sec 5.27k 718.39 9.11k 80.71% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.31ms + 75.000% 1.96ms + 90.000% 4.05ms + 99.000% 12.05ms + 99.900% 18.05ms + 99.990% 19.20ms + 99.999% 19.77ms +100.000% 19.92ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.056 0.000000 2 1.00 + 0.526 0.100000 39789 1.11 + 0.747 0.200000 79555 1.25 + 0.939 0.300000 119217 1.43 + 1.124 0.400000 158859 1.67 + 1.312 0.500000 198438 2.00 + 1.410 0.550000 218421 2.22 + 1.511 0.600000 238148 2.50 + 1.629 0.650000 258074 2.86 + 1.773 0.700000 277823 3.33 + 1.964 0.750000 297637 4.00 + 2.095 0.775000 307646 4.44 + 2.263 0.800000 317494 5.00 + 2.511 0.825000 327454 5.71 + 2.873 0.850000 337346 6.67 + 3.375 0.875000 347244 8.00 + 3.685 0.887500 352210 8.89 + 4.055 0.900000 357182 10.00 + 4.495 0.912500 362151 11.43 + 5.063 0.925000 367099 13.33 + 5.763 0.937500 372060 16.00 + 6.159 0.943750 374550 17.78 + 6.643 0.950000 377015 20.00 + 7.203 0.956250 379497 22.86 + 7.883 0.962500 381971 26.67 + 8.599 0.968750 384450 32.00 + 8.975 0.971875 385697 35.56 + 9.367 0.975000 386928 40.00 + 9.799 0.978125 388166 45.71 + 10.263 0.981250 389415 53.33 + 10.783 0.984375 390651 64.00 + 11.095 0.985938 391275 71.11 + 11.423 0.987500 391888 80.00 + 11.783 0.989062 392516 91.43 + 12.231 0.990625 393130 106.67 + 12.815 0.992188 393746 128.00 + 13.191 0.992969 394055 142.22 + 13.599 0.993750 394367 160.00 + 13.959 0.994531 394678 182.86 + 14.367 0.995313 394989 213.33 + 14.847 0.996094 395295 256.00 + 15.199 0.996484 395450 284.44 + 15.807 0.996875 395605 320.00 + 16.415 0.997266 395761 365.71 + 16.943 0.997656 395918 426.67 + 17.295 0.998047 396075 512.00 + 17.439 0.998242 396153 568.89 + 17.583 0.998437 396234 640.00 + 17.743 0.998633 396306 731.43 + 17.887 0.998828 396385 853.33 + 18.063 0.999023 396462 1024.00 + 18.143 0.999121 396501 1137.78 + 18.223 0.999219 396538 1280.00 + 18.303 0.999316 396579 1462.86 + 18.383 0.999414 396618 1706.67 + 18.463 0.999512 396653 2048.00 + 18.543 0.999561 396676 2275.56 + 18.607 0.999609 396693 2560.00 + 18.687 0.999658 396711 2925.71 + 18.783 0.999707 396734 3413.33 + 18.831 0.999756 396749 4096.00 + 18.895 0.999780 396761 4551.11 + 18.927 0.999805 396768 5120.00 + 19.023 0.999829 396778 5851.43 + 19.071 0.999854 396787 6826.67 + 19.167 0.999878 396799 8192.00 + 19.183 0.999890 396802 9102.22 + 19.199 0.999902 396808 10240.00 + 19.231 0.999915 396812 11702.86 + 19.247 0.999927 396819 13653.33 + 19.279 0.999939 396821 16384.00 + 19.343 0.999945 396826 18204.44 + 19.343 0.999951 396826 20480.00 + 19.391 0.999957 396829 23405.71 + 19.439 0.999963 396831 27306.67 + 19.535 0.999969 396833 32768.00 + 19.551 0.999973 396835 36408.89 + 19.599 0.999976 396836 40960.00 + 19.679 0.999979 396837 46811.43 + 19.727 0.999982 396839 54613.33 + 19.727 0.999985 396839 65536.00 + 19.759 0.999986 396840 72817.78 + 19.775 0.999988 396841 81920.00 + 19.775 0.999989 396841 93622.86 + 19.823 0.999991 396842 109226.67 + 19.823 0.999992 396842 131072.00 + 19.871 0.999993 396843 145635.56 + 19.871 0.999994 396843 163840.00 + 19.871 0.999995 396843 187245.71 + 19.887 0.999995 396844 218453.33 + 19.887 0.999996 396844 262144.00 + 19.887 0.999997 396844 291271.11 + 19.887 0.999997 396844 327680.00 + 19.887 0.999997 396844 374491.43 + 19.919 0.999998 396845 436906.67 + 19.919 1.000000 396845 inf +#[Mean = 1.971, StdDeviation = 2.260] +#[Max = 19.904, Total count = 396845] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 798482 requests in 20.00s, 166.82MB read + Non-2xx or 3xx responses: 362022 +Requests/sec: 39924.79 +Transfer/sec: 8.34MB +------------------------------ + +HTTP Status 404 Count: 362022 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_80000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_80000_wrk.txt new file mode 100644 index 000000000..aa60fb4a5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_get_80000_wrk.txt @@ -0,0 +1,115 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 890.230ms, rate sampling interval: 2787ms + Thread calibration: mean lat.: 890.300ms, rate sampling interval: 2785ms + Thread calibration: mean lat.: 890.301ms, rate sampling interval: 2783ms + Thread calibration: mean lat.: 889.136ms, rate sampling interval: 2781ms + Thread calibration: mean lat.: 928.200ms, rate sampling interval: 2871ms + Thread calibration: mean lat.: 889.076ms, rate sampling interval: 2785ms + Thread calibration: mean lat.: 890.089ms, rate sampling interval: 2789ms + Thread calibration: mean lat.: 888.273ms, rate sampling interval: 2781ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.19s 383.68ms 2.95s 58.23% + Req/Sec 6.50k 20.47 6.52k 50.00% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 2.19s + 75.000% 2.52s + 90.000% 2.72s + 99.000% 2.86s + 99.900% 2.91s + 99.990% 2.95s + 99.999% 2.96s +100.000% 2.96s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 1486.847 0.000000 3 1.00 + 1659.903 0.100000 51833 1.11 + 1793.023 0.200000 103015 1.25 + 1926.143 0.300000 154645 1.43 + 2057.215 0.400000 206037 1.67 + 2189.311 0.500000 257793 2.00 + 2250.751 0.550000 283138 2.22 + 2316.287 0.600000 309034 2.50 + 2381.823 0.650000 334497 2.86 + 2453.503 0.700000 360923 3.33 + 2519.039 0.750000 386105 4.00 + 2553.855 0.775000 398966 4.44 + 2586.623 0.800000 412235 5.00 + 2619.391 0.825000 425011 5.71 + 2654.207 0.850000 437767 6.67 + 2684.927 0.875000 450244 8.00 + 2705.407 0.887500 456699 8.89 + 2723.839 0.900000 463662 10.00 + 2740.223 0.912500 469846 11.43 + 2760.703 0.925000 476419 13.33 + 2779.135 0.937500 483232 16.00 + 2787.327 0.943750 486368 17.78 + 2793.471 0.950000 488956 20.00 + 2801.663 0.956250 492021 22.86 + 2811.903 0.962500 495315 26.67 + 2822.143 0.968750 498471 32.00 + 2828.287 0.971875 500496 35.56 + 2832.383 0.975000 502008 40.00 + 2836.479 0.978125 503407 45.71 + 2842.623 0.981250 505446 53.33 + 2848.767 0.984375 506943 64.00 + 2850.815 0.985938 507458 71.11 + 2854.911 0.987500 508382 80.00 + 2859.007 0.989062 509063 91.43 + 2863.103 0.990625 509756 106.67 + 2869.247 0.992188 510518 128.00 + 2873.343 0.992969 510973 142.22 + 2877.439 0.993750 511427 160.00 + 2881.535 0.994531 511913 182.86 + 2883.583 0.995313 512135 213.33 + 2887.679 0.996094 512547 256.00 + 2889.727 0.996484 512837 284.44 + 2891.775 0.996875 513058 320.00 + 2893.823 0.997266 513202 365.71 + 2895.871 0.997656 513342 426.67 + 2899.967 0.998047 513674 512.00 + 2899.967 0.998242 513674 568.89 + 2902.015 0.998437 513763 640.00 + 2906.111 0.998633 513830 731.43 + 2910.207 0.998828 513933 853.33 + 2916.351 0.999023 514054 1024.00 + 2918.399 0.999121 514089 1137.78 + 2920.447 0.999219 514149 1280.00 + 2922.495 0.999316 514194 1462.86 + 2926.591 0.999414 514231 1706.67 + 2930.687 0.999512 514290 2048.00 + 2932.735 0.999561 514324 2275.56 + 2934.783 0.999609 514348 2560.00 + 2938.879 0.999658 514356 2925.71 + 2942.975 0.999707 514410 3413.33 + 2942.975 0.999756 514410 4096.00 + 2945.023 0.999780 514449 4551.11 + 2945.023 0.999805 514449 5120.00 + 2945.023 0.999829 514449 5851.43 + 2947.071 0.999854 514468 6826.67 + 2947.071 0.999878 514468 8192.00 + 2951.167 0.999890 514486 9102.22 + 2951.167 0.999902 514486 10240.00 + 2953.215 0.999915 514514 11702.86 + 2953.215 0.999927 514514 13653.33 + 2953.215 0.999939 514514 16384.00 + 2953.215 0.999945 514514 18204.44 + 2953.215 0.999951 514514 20480.00 + 2953.215 0.999957 514514 23405.71 + 2953.215 0.999963 514514 27306.67 + 2955.263 0.999969 514530 32768.00 + 2955.263 1.000000 514530 inf +#[Mean = 2189.628, StdDeviation = 383.679] +#[Max = 2953.216, Total count = 514530] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1025343 requests in 20.00s, 214.76MB read + Non-2xx or 3xx responses: 464020 +Requests/sec: 51270.81 +Transfer/sec: 10.74MB +------------------------------ + +HTTP Status 404 Count: 464020 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_35000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_35000_wrk.txt new file mode 100644 index 000000000..995c3ebf1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_35000_wrk.txt @@ -0,0 +1,128 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 2.224ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.244ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.259ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.226ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.240ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.250ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.259ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.215ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.70ms 0.88ms 11.31ms 78.75% + Req/Sec 4.61k 406.42 7.00k 72.80% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.58ms + 75.000% 2.11ms + 90.000% 2.61ms + 99.000% 5.07ms + 99.900% 8.41ms + 99.990% 10.01ms + 99.999% 10.81ms +100.000% 11.32ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.078 0.000000 1 1.00 + 0.793 0.100000 34725 1.11 + 1.021 0.200000 69572 1.25 + 1.214 0.300000 104190 1.43 + 1.399 0.400000 138936 1.67 + 1.584 0.500000 173620 2.00 + 1.680 0.550000 191006 2.22 + 1.777 0.600000 208332 2.50 + 1.881 0.650000 225656 2.86 + 1.991 0.700000 243017 3.33 + 2.111 0.750000 260392 4.00 + 2.175 0.775000 269041 4.44 + 2.243 0.800000 277856 5.00 + 2.315 0.825000 286509 5.71 + 2.397 0.850000 295189 6.67 + 2.491 0.875000 303778 8.00 + 2.547 0.887500 308147 8.89 + 2.609 0.900000 312446 10.00 + 2.683 0.912500 316832 11.43 + 2.771 0.925000 321196 13.33 + 2.881 0.937500 325461 16.00 + 2.949 0.943750 327638 17.78 + 3.029 0.950000 329804 20.00 + 3.119 0.956250 331989 22.86 + 3.219 0.962500 334150 26.67 + 3.347 0.968750 336302 32.00 + 3.425 0.971875 337389 35.56 + 3.521 0.975000 338475 40.00 + 3.651 0.978125 339560 45.71 + 3.833 0.981250 340642 53.33 + 4.143 0.984375 341734 64.00 + 4.331 0.985938 342263 71.11 + 4.563 0.987500 342805 80.00 + 4.851 0.989062 343347 91.43 + 5.207 0.990625 343889 106.67 + 5.615 0.992188 344431 128.00 + 5.855 0.992969 344709 142.22 + 6.123 0.993750 344978 160.00 + 6.371 0.994531 345249 182.86 + 6.635 0.995313 345519 213.33 + 6.895 0.996094 345788 256.00 + 7.063 0.996484 345926 284.44 + 7.207 0.996875 346065 320.00 + 7.379 0.997266 346197 365.71 + 7.551 0.997656 346331 426.67 + 7.763 0.998047 346465 512.00 + 7.879 0.998242 346534 568.89 + 8.023 0.998437 346602 640.00 + 8.163 0.998633 346671 731.43 + 8.303 0.998828 346741 853.33 + 8.439 0.999023 346807 1024.00 + 8.535 0.999121 346840 1137.78 + 8.639 0.999219 346875 1280.00 + 8.743 0.999316 346906 1462.86 + 8.863 0.999414 346940 1706.67 + 8.943 0.999512 346974 2048.00 + 9.031 0.999561 346991 2275.56 + 9.135 0.999609 347009 2560.00 + 9.199 0.999658 347025 2925.71 + 9.319 0.999707 347042 3413.33 + 9.495 0.999756 347059 4096.00 + 9.567 0.999780 347067 4551.11 + 9.663 0.999805 347076 5120.00 + 9.767 0.999829 347084 5851.43 + 9.855 0.999854 347093 6826.67 + 9.919 0.999878 347101 8192.00 + 9.967 0.999890 347105 9102.22 + 10.055 0.999902 347110 10240.00 + 10.071 0.999915 347114 11702.86 + 10.151 0.999927 347118 13653.33 + 10.263 0.999939 347122 16384.00 + 10.335 0.999945 347124 18204.44 + 10.359 0.999951 347127 20480.00 + 10.383 0.999957 347129 23405.71 + 10.423 0.999963 347131 27306.67 + 10.495 0.999969 347133 32768.00 + 10.503 0.999973 347134 36408.89 + 10.575 0.999976 347135 40960.00 + 10.607 0.999979 347136 46811.43 + 10.615 0.999982 347137 54613.33 + 10.623 0.999985 347138 65536.00 + 10.751 0.999986 347139 72817.78 + 10.751 0.999988 347139 81920.00 + 10.815 0.999989 347140 93622.86 + 10.815 0.999991 347140 109226.67 + 10.903 0.999992 347141 131072.00 + 10.903 0.999993 347141 145635.56 + 10.903 0.999994 347141 163840.00 + 11.007 0.999995 347142 187245.71 + 11.007 0.999995 347142 218453.33 + 11.007 0.999996 347142 262144.00 + 11.007 0.999997 347142 291271.11 + 11.007 0.999997 347142 327680.00 + 11.319 0.999997 347143 374491.43 + 11.319 1.000000 347143 inf +#[Mean = 1.700, StdDeviation = 0.884] +#[Max = 11.312, Total count = 347143] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 698653 requests in 20.00s, 56.62MB read +Requests/sec: 34934.28 +Transfer/sec: 2.83MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_36000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_36000_wrk.txt new file mode 100644 index 000000000..2db2c6010 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_36000_wrk.txt @@ -0,0 +1,128 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.838ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.861ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.447ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.871ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.862ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.914ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.893ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.983ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.89ms 1.32ms 13.93ms 90.77% + Req/Sec 4.74k 439.39 7.00k 71.11% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.65ms + 75.000% 2.21ms + 90.000% 2.81ms + 99.000% 8.69ms + 99.900% 11.88ms + 99.990% 13.17ms + 99.999% 13.87ms +100.000% 13.94ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.054 0.000000 1 1.00 + 0.829 0.100000 35767 1.11 + 1.067 0.200000 71470 1.25 + 1.271 0.300000 107287 1.43 + 1.460 0.400000 142884 1.67 + 1.652 0.500000 178511 2.00 + 1.753 0.550000 196474 2.22 + 1.858 0.600000 214244 2.50 + 1.970 0.650000 232160 2.86 + 2.087 0.700000 249988 3.33 + 2.215 0.750000 268029 4.00 + 2.283 0.775000 276900 4.44 + 2.355 0.800000 285765 5.00 + 2.437 0.825000 294716 5.71 + 2.531 0.850000 303595 6.67 + 2.649 0.875000 312399 8.00 + 2.725 0.887500 316922 8.89 + 2.813 0.900000 321331 10.00 + 2.923 0.912500 325821 11.43 + 3.059 0.925000 330248 13.33 + 3.227 0.937500 334718 16.00 + 3.337 0.943750 336968 17.78 + 3.475 0.950000 339170 20.00 + 3.693 0.956250 341404 22.86 + 4.089 0.962500 343634 26.67 + 4.755 0.968750 345875 32.00 + 5.195 0.971875 346986 35.56 + 5.667 0.975000 348098 40.00 + 6.215 0.978125 349215 45.71 + 6.819 0.981250 350327 53.33 + 7.495 0.984375 351443 64.00 + 7.823 0.985938 352004 71.11 + 8.159 0.987500 352567 80.00 + 8.487 0.989062 353120 91.43 + 8.839 0.990625 353681 106.67 + 9.199 0.992188 354233 128.00 + 9.415 0.992969 354524 142.22 + 9.607 0.993750 354792 160.00 + 9.815 0.994531 355079 182.86 + 10.047 0.995313 355349 213.33 + 10.287 0.996094 355629 256.00 + 10.415 0.996484 355766 284.44 + 10.575 0.996875 355908 320.00 + 10.735 0.997266 356048 365.71 + 10.935 0.997656 356190 426.67 + 11.111 0.998047 356327 512.00 + 11.231 0.998242 356394 568.89 + 11.399 0.998437 356471 640.00 + 11.543 0.998633 356533 731.43 + 11.735 0.998828 356605 853.33 + 11.911 0.999023 356675 1024.00 + 11.975 0.999121 356708 1137.78 + 12.119 0.999219 356745 1280.00 + 12.231 0.999316 356780 1462.86 + 12.351 0.999414 356813 1706.67 + 12.479 0.999512 356849 2048.00 + 12.527 0.999561 356867 2275.56 + 12.599 0.999609 356884 2560.00 + 12.655 0.999658 356899 2925.71 + 12.751 0.999707 356917 3413.33 + 12.823 0.999756 356935 4096.00 + 12.855 0.999780 356944 4551.11 + 12.919 0.999805 356952 5120.00 + 12.983 0.999829 356960 5851.43 + 13.071 0.999854 356969 6826.67 + 13.119 0.999878 356978 8192.00 + 13.143 0.999890 356982 9102.22 + 13.175 0.999902 356987 10240.00 + 13.239 0.999915 356992 11702.86 + 13.311 0.999927 356995 13653.33 + 13.367 0.999939 357000 16384.00 + 13.399 0.999945 357002 18204.44 + 13.423 0.999951 357004 20480.00 + 13.479 0.999957 357006 23405.71 + 13.511 0.999963 357008 27306.67 + 13.599 0.999969 357011 32768.00 + 13.615 0.999973 357012 36408.89 + 13.631 0.999976 357013 40960.00 + 13.735 0.999979 357014 46811.43 + 13.823 0.999982 357015 54613.33 + 13.871 0.999985 357017 65536.00 + 13.871 0.999986 357017 72817.78 + 13.871 0.999988 357017 81920.00 + 13.879 0.999989 357018 93622.86 + 13.879 0.999991 357018 109226.67 + 13.903 0.999992 357019 131072.00 + 13.903 0.999993 357019 145635.56 + 13.903 0.999994 357019 163840.00 + 13.911 0.999995 357020 187245.71 + 13.911 0.999995 357020 218453.33 + 13.911 0.999996 357020 262144.00 + 13.911 0.999997 357020 291271.11 + 13.911 0.999997 357020 327680.00 + 13.935 0.999997 357021 374491.43 + 13.935 1.000000 357021 inf +#[Mean = 1.886, StdDeviation = 1.321] +#[Max = 13.928, Total count = 357021] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 714250 requests in 20.00s, 57.88MB read +Requests/sec: 35713.44 +Transfer/sec: 2.89MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_37000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_37000_wrk.txt new file mode 100644 index 000000000..7e1831346 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_37000_wrk.txt @@ -0,0 +1,128 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 2.393ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.571ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.805ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.308ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.258ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.279ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.317ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.488ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.11ms 1.64ms 14.60ms 92.53% + Req/Sec 4.88k 467.93 9.00k 73.99% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.78ms + 75.000% 2.38ms + 90.000% 3.14ms + 99.000% 10.77ms + 99.900% 12.94ms + 99.990% 14.03ms + 99.999% 14.45ms +100.000% 14.61ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.081 0.000000 1 1.00 + 0.908 0.100000 36805 1.11 + 1.169 0.200000 73538 1.25 + 1.385 0.300000 110228 1.43 + 1.581 0.400000 146876 1.67 + 1.776 0.500000 183603 2.00 + 1.879 0.550000 201878 2.22 + 1.990 0.600000 220319 2.50 + 2.111 0.650000 238702 2.86 + 2.239 0.700000 257146 3.33 + 2.375 0.750000 275356 4.00 + 2.453 0.775000 284629 4.44 + 2.537 0.800000 293780 5.00 + 2.633 0.825000 302860 5.71 + 2.755 0.850000 312039 6.67 + 2.919 0.875000 321202 8.00 + 3.025 0.887500 325761 8.89 + 3.145 0.900000 330354 10.00 + 3.291 0.912500 334931 11.43 + 3.485 0.925000 339487 13.33 + 3.835 0.937500 344062 16.00 + 4.151 0.943750 346374 17.78 + 4.551 0.950000 348640 20.00 + 5.011 0.956250 350943 22.86 + 5.571 0.962500 353236 26.67 + 6.447 0.968750 355531 32.00 + 6.979 0.971875 356669 35.56 + 7.591 0.975000 357816 40.00 + 8.255 0.978125 358970 45.71 + 8.975 0.981250 360110 53.33 + 9.679 0.984375 361260 64.00 + 9.991 0.985938 361831 71.11 + 10.311 0.987500 362406 80.00 + 10.583 0.989062 362980 91.43 + 10.879 0.990625 363558 106.67 + 11.143 0.992188 364133 128.00 + 11.295 0.992969 364418 142.22 + 11.455 0.993750 364703 160.00 + 11.623 0.994531 364992 182.86 + 11.767 0.995313 365272 213.33 + 11.935 0.996094 365568 256.00 + 12.023 0.996484 365708 284.44 + 12.119 0.996875 365854 320.00 + 12.239 0.997266 365995 365.71 + 12.359 0.997656 366129 426.67 + 12.503 0.998047 366279 512.00 + 12.583 0.998242 366347 568.89 + 12.647 0.998437 366418 640.00 + 12.735 0.998633 366488 731.43 + 12.831 0.998828 366561 853.33 + 12.967 0.999023 366634 1024.00 + 13.039 0.999121 366668 1137.78 + 13.103 0.999219 366703 1280.00 + 13.183 0.999316 366740 1462.86 + 13.255 0.999414 366774 1706.67 + 13.367 0.999512 366812 2048.00 + 13.415 0.999561 366829 2275.56 + 13.471 0.999609 366846 2560.00 + 13.543 0.999658 366865 2925.71 + 13.615 0.999707 366882 3413.33 + 13.711 0.999756 366900 4096.00 + 13.751 0.999780 366910 4551.11 + 13.831 0.999805 366919 5120.00 + 13.879 0.999829 366928 5851.43 + 13.935 0.999854 366936 6826.67 + 13.999 0.999878 366947 8192.00 + 14.015 0.999890 366950 9102.22 + 14.055 0.999902 366954 10240.00 + 14.079 0.999915 366958 11702.86 + 14.127 0.999927 366963 13653.33 + 14.159 0.999939 366967 16384.00 + 14.175 0.999945 366969 18204.44 + 14.247 0.999951 366972 20480.00 + 14.279 0.999957 366974 23405.71 + 14.311 0.999963 366976 27306.67 + 14.351 0.999969 366978 32768.00 + 14.383 0.999973 366979 36408.89 + 14.399 0.999976 366981 40960.00 + 14.415 0.999979 366982 46811.43 + 14.423 0.999982 366983 54613.33 + 14.439 0.999985 366984 65536.00 + 14.439 0.999986 366984 72817.78 + 14.447 0.999988 366985 81920.00 + 14.455 0.999989 366986 93622.86 + 14.455 0.999991 366986 109226.67 + 14.495 0.999992 366987 131072.00 + 14.495 0.999993 366987 145635.56 + 14.495 0.999994 366987 163840.00 + 14.527 0.999995 366988 187245.71 + 14.527 0.999995 366988 218453.33 + 14.527 0.999996 366988 262144.00 + 14.527 0.999997 366988 291271.11 + 14.527 0.999997 366988 327680.00 + 14.607 0.999997 366989 374491.43 + 14.607 1.000000 366989 inf +#[Mean = 2.114, StdDeviation = 1.644] +#[Max = 14.600, Total count = 366989] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 738569 requests in 20.00s, 59.86MB read +Requests/sec: 36930.98 +Transfer/sec: 2.99MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_38000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_38000_wrk.txt new file mode 100644 index 000000000..a747b2bac --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_38000_wrk.txt @@ -0,0 +1,129 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 2.753ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.776ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.554ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.525ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.477ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.720ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.496ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.564ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.42ms 2.60ms 50.43ms 94.83% + Req/Sec 5.01k 561.03 14.33k 75.57% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.94ms + 75.000% 2.63ms + 90.000% 3.68ms + 99.000% 12.31ms + 99.900% 42.33ms + 99.990% 48.13ms + 99.999% 50.14ms +100.000% 50.46ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.078 0.000000 1 1.00 + 0.987 0.100000 37717 1.11 + 1.274 0.200000 75394 1.25 + 1.507 0.300000 113167 1.43 + 1.722 0.400000 150923 1.67 + 1.943 0.500000 188554 2.00 + 2.061 0.550000 207375 2.22 + 2.185 0.600000 226203 2.50 + 2.317 0.650000 245090 2.86 + 2.461 0.700000 264015 3.33 + 2.627 0.750000 282730 4.00 + 2.727 0.775000 292192 4.44 + 2.845 0.800000 301593 5.00 + 2.993 0.825000 311060 5.71 + 3.165 0.850000 320383 6.67 + 3.379 0.875000 329842 8.00 + 3.513 0.887500 334544 8.89 + 3.681 0.900000 339244 10.00 + 3.901 0.912500 343960 11.43 + 4.187 0.925000 348670 13.33 + 4.575 0.937500 353388 16.00 + 4.815 0.943750 355729 17.78 + 5.115 0.950000 358088 20.00 + 5.447 0.956250 360433 22.86 + 5.899 0.962500 362786 26.67 + 6.631 0.968750 365142 32.00 + 7.239 0.971875 366315 35.56 + 8.099 0.975000 367493 40.00 + 9.079 0.978125 368678 45.71 + 9.895 0.981250 369861 53.33 + 10.671 0.984375 371035 64.00 + 11.103 0.985938 371617 71.11 + 11.559 0.987500 372204 80.00 + 12.047 0.989062 372796 91.43 + 12.511 0.990625 373385 106.67 + 13.095 0.992188 373975 128.00 + 13.359 0.992969 374263 142.22 + 13.695 0.993750 374560 160.00 + 14.135 0.994531 374853 182.86 + 14.807 0.995313 375147 213.33 + 16.007 0.996094 375441 256.00 + 18.351 0.996484 375589 284.44 + 21.119 0.996875 375737 320.00 + 23.439 0.997266 375884 365.71 + 25.919 0.997656 376033 426.67 + 29.055 0.998047 376177 512.00 + 32.687 0.998242 376251 568.89 + 36.127 0.998437 376325 640.00 + 39.231 0.998633 376398 731.43 + 40.991 0.998828 376473 853.33 + 42.527 0.999023 376545 1024.00 + 43.295 0.999121 376583 1137.78 + 43.839 0.999219 376620 1280.00 + 44.479 0.999316 376656 1462.86 + 44.927 0.999414 376693 1706.67 + 45.439 0.999512 376732 2048.00 + 45.631 0.999561 376748 2275.56 + 46.047 0.999609 376767 2560.00 + 46.399 0.999658 376785 2925.71 + 46.623 0.999707 376804 3413.33 + 46.975 0.999756 376821 4096.00 + 47.103 0.999780 376831 4551.11 + 47.519 0.999805 376844 5120.00 + 47.647 0.999829 376850 5851.43 + 47.711 0.999854 376858 6826.67 + 47.935 0.999878 376869 8192.00 + 47.967 0.999890 376872 9102.22 + 48.127 0.999902 376877 10240.00 + 48.447 0.999915 376882 11702.86 + 48.607 0.999927 376887 13653.33 + 48.735 0.999939 376891 16384.00 + 48.959 0.999945 376894 18204.44 + 49.055 0.999951 376896 20480.00 + 49.119 0.999957 376897 23405.71 + 49.279 0.999963 376900 27306.67 + 49.375 0.999969 376902 32768.00 + 49.439 0.999973 376903 36408.89 + 49.663 0.999976 376904 40960.00 + 49.727 0.999979 376907 46811.43 + 49.727 0.999982 376907 54613.33 + 49.823 0.999985 376908 65536.00 + 49.823 0.999986 376908 72817.78 + 50.143 0.999988 376909 81920.00 + 50.143 0.999989 376909 93622.86 + 50.271 0.999991 376910 109226.67 + 50.335 0.999992 376911 131072.00 + 50.335 0.999993 376911 145635.56 + 50.335 0.999994 376911 163840.00 + 50.335 0.999995 376911 187245.71 + 50.367 0.999995 376912 218453.33 + 50.367 0.999996 376912 262144.00 + 50.367 0.999997 376912 291271.11 + 50.367 0.999997 376912 327680.00 + 50.367 0.999997 376912 374491.43 + 50.463 0.999998 376913 436906.67 + 50.463 1.000000 376913 inf +#[Mean = 2.420, StdDeviation = 2.601] +#[Max = 50.432, Total count = 376913] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 758519 requests in 20.00s, 61.48MB read +Requests/sec: 37927.38 +Transfer/sec: 3.07MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_48000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_48000_wrk.txt new file mode 100644 index 000000000..a4dc74e88 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_put_48000_wrk.txt @@ -0,0 +1,113 @@ +Running 20s test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 774.145ms, rate sampling interval: 2783ms + Thread calibration: mean lat.: 781.065ms, rate sampling interval: 2799ms + Thread calibration: mean lat.: 870.600ms, rate sampling interval: 3028ms + Thread calibration: mean lat.: 762.832ms, rate sampling interval: 2754ms + Thread calibration: mean lat.: 877.637ms, rate sampling interval: 3045ms + Thread calibration: mean lat.: 865.944ms, rate sampling interval: 3006ms + Thread calibration: mean lat.: 761.215ms, rate sampling interval: 2750ms + Thread calibration: mean lat.: 761.764ms, rate sampling interval: 2750ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.24s 395.70ms 3.04s 57.97% + Req/Sec 5.19k 62.41 5.26k 91.67% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 2.25s + 75.000% 2.59s + 90.000% 2.78s + 99.000% 2.95s + 99.900% 3.02s + 99.990% 3.03s + 99.999% 3.04s +100.000% 3.04s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 1521.663 0.000000 5 1.00 + 1705.983 0.100000 41575 1.11 + 1829.887 0.200000 82744 1.25 + 1958.911 0.300000 124322 1.43 + 2101.247 0.400000 165509 1.67 + 2248.703 0.500000 206815 2.00 + 2314.239 0.550000 227464 2.22 + 2387.967 0.600000 248045 2.50 + 2453.503 0.650000 269399 2.86 + 2508.799 0.700000 289869 3.33 + 2586.623 0.750000 310046 4.00 + 2619.391 0.775000 320691 4.44 + 2648.063 0.800000 330727 5.00 + 2678.783 0.825000 341475 5.71 + 2715.647 0.850000 352007 6.67 + 2746.367 0.875000 362341 8.00 + 2760.703 0.887500 367070 8.89 + 2775.039 0.900000 372227 10.00 + 2791.423 0.912500 377419 11.43 + 2805.759 0.925000 382388 13.33 + 2826.239 0.937500 387542 16.00 + 2842.623 0.943750 390261 17.78 + 2856.959 0.950000 393057 20.00 + 2867.199 0.956250 395299 22.86 + 2879.487 0.962500 397990 26.67 + 2891.775 0.968750 400576 32.00 + 2899.967 0.971875 401886 35.56 + 2908.159 0.975000 403165 40.00 + 2916.351 0.978125 404412 45.71 + 2926.591 0.981250 405820 53.33 + 2934.783 0.984375 406955 64.00 + 2938.879 0.985938 407542 71.11 + 2942.975 0.987500 408179 80.00 + 2947.071 0.989062 408842 91.43 + 2953.215 0.990625 409728 106.67 + 2957.311 0.992188 410150 128.00 + 2961.407 0.992969 410481 142.22 + 2967.551 0.993750 410807 160.00 + 2975.743 0.994531 411134 182.86 + 2981.887 0.995313 411455 213.33 + 2988.031 0.996094 411759 256.00 + 2992.127 0.996484 411932 284.44 + 2996.223 0.996875 412064 320.00 + 3004.415 0.997266 412244 365.71 + 3010.559 0.997656 412412 426.67 + 3014.655 0.998047 412565 512.00 + 3016.703 0.998242 412649 568.89 + 3018.751 0.998437 412745 640.00 + 3020.799 0.998633 412858 731.43 + 3020.799 0.998828 412858 853.33 + 3022.847 0.999023 412952 1024.00 + 3024.895 0.999121 413040 1137.78 + 3024.895 0.999219 413040 1280.00 + 3026.943 0.999316 413136 1462.86 + 3026.943 0.999414 413136 1706.67 + 3028.991 0.999512 413218 2048.00 + 3028.991 0.999561 413218 2275.56 + 3028.991 0.999609 413218 2560.00 + 3028.991 0.999658 413218 2925.71 + 3031.039 0.999707 413270 3413.33 + 3031.039 0.999756 413270 4096.00 + 3031.039 0.999780 413270 4551.11 + 3031.039 0.999805 413270 5120.00 + 3033.087 0.999829 413307 5851.43 + 3033.087 0.999854 413307 6826.67 + 3033.087 0.999878 413307 8192.00 + 3033.087 0.999890 413307 9102.22 + 3033.087 0.999902 413307 10240.00 + 3033.087 0.999915 413307 11702.86 + 3035.135 0.999927 413332 13653.33 + 3035.135 0.999939 413332 16384.00 + 3035.135 0.999945 413332 18204.44 + 3035.135 0.999951 413332 20480.00 + 3035.135 0.999957 413332 23405.71 + 3035.135 0.999963 413332 27306.67 + 3035.135 0.999969 413332 32768.00 + 3035.135 0.999973 413332 36408.89 + 3035.135 0.999976 413332 40960.00 + 3037.183 0.999979 413341 46811.43 + 3037.183 1.000000 413341 inf +#[Mean = 2240.612, StdDeviation = 395.699] +#[Max = 3035.136, Total count = 413341] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 819405 requests in 20.00s, 66.42MB read +Requests/sec: 40973.78 +Transfer/sec: 3.32MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_alloc.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_alloc.html new file mode 100644 index 000000000..c3ed302f5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_alloc.html @@ -0,0 +1,1267 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_cpu.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_cpu.html new file mode 100644 index 000000000..8e1f51fdf --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_cpu.html @@ -0,0 +1,7578 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_lock.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_lock.html new file mode 100644 index 000000000..b2d50e9e8 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_lock.html @@ -0,0 +1,691 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_wrk.txt new file mode 100644 index 000000000..06e20d5b9 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_get_38000_wrk.txt @@ -0,0 +1,158 @@ +Running 5m test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.580ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.580ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.583ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.584ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.583ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.588ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.578ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.586ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.55ms 1.54ms 24.94ms 92.43% + Req/Sec 5.01k 534.32 11.67k 82.42% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.21ms + 75.000% 1.71ms + 90.000% 2.58ms + 99.000% 8.80ms + 99.900% 15.18ms + 99.990% 20.83ms + 99.999% 23.15ms +100.000% 24.96ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.037 0.000000 1 1.00 + 0.483 0.100000 1101753 1.11 + 0.689 0.200000 2203779 1.25 + 0.871 0.300000 3305198 1.43 + 1.043 0.400000 4412348 1.67 + 1.214 0.500000 5512744 2.00 + 1.300 0.550000 6059369 2.22 + 1.388 0.600000 6613413 2.50 + 1.480 0.650000 7164633 2.86 + 1.584 0.700000 7714256 3.33 + 1.709 0.750000 8262552 4.00 + 1.785 0.775000 8540925 4.44 + 1.872 0.800000 8815947 5.00 + 1.976 0.825000 9089973 5.71 + 2.107 0.850000 9365314 6.67 + 2.287 0.875000 9641933 8.00 + 2.411 0.887500 9778609 8.89 + 2.575 0.900000 9915535 10.00 + 2.801 0.912500 10053530 11.43 + 3.109 0.925000 10190771 13.33 + 3.513 0.937500 10328148 16.00 + 3.761 0.943750 10397384 17.78 + 4.045 0.950000 10466256 20.00 + 4.367 0.956250 10535286 22.86 + 4.747 0.962500 10603840 26.67 + 5.223 0.968750 10672588 32.00 + 5.527 0.971875 10707141 35.56 + 5.887 0.975000 10741244 40.00 + 6.327 0.978125 10775686 45.71 + 6.843 0.981250 10810304 53.33 + 7.427 0.984375 10844657 64.00 + 7.755 0.985938 10861838 71.11 + 8.119 0.987500 10879025 80.00 + 8.527 0.989062 10896173 91.43 + 8.991 0.990625 10913385 106.67 + 9.559 0.992188 10930778 128.00 + 9.879 0.992969 10939290 142.22 + 10.239 0.993750 10947791 160.00 + 10.647 0.994531 10956521 182.86 + 11.103 0.995313 10965089 213.33 + 11.607 0.996094 10973624 256.00 + 11.903 0.996484 10978015 284.44 + 12.215 0.996875 10982241 320.00 + 12.575 0.997266 10986594 365.71 + 12.983 0.997656 10990896 426.67 + 13.455 0.998047 10995185 512.00 + 13.727 0.998242 10997293 568.89 + 14.047 0.998437 10999436 640.00 + 14.415 0.998633 11001630 731.43 + 14.807 0.998828 11003766 853.33 + 15.231 0.999023 11005899 1024.00 + 15.471 0.999121 11006988 1137.78 + 15.727 0.999219 11008044 1280.00 + 16.023 0.999316 11009120 1462.86 + 16.367 0.999414 11010212 1706.67 + 16.767 0.999512 11011280 2048.00 + 17.007 0.999561 11011816 2275.56 + 17.295 0.999609 11012342 2560.00 + 17.679 0.999658 11012897 2925.71 + 18.127 0.999707 11013418 3413.33 + 18.655 0.999756 11013970 4096.00 + 18.911 0.999780 11014229 4551.11 + 19.199 0.999805 11014497 5120.00 + 19.583 0.999829 11014774 5851.43 + 20.015 0.999854 11015032 6826.67 + 20.463 0.999878 11015306 8192.00 + 20.655 0.999890 11015435 9102.22 + 20.879 0.999902 11015575 10240.00 + 21.071 0.999915 11015710 11702.86 + 21.311 0.999927 11015841 13653.33 + 21.615 0.999939 11015977 16384.00 + 21.759 0.999945 11016044 18204.44 + 21.935 0.999951 11016108 20480.00 + 22.095 0.999957 11016175 23405.71 + 22.287 0.999963 11016244 27306.67 + 22.431 0.999969 11016312 32768.00 + 22.527 0.999973 11016349 36408.89 + 22.607 0.999976 11016377 40960.00 + 22.687 0.999979 11016412 46811.43 + 22.799 0.999982 11016448 54613.33 + 22.911 0.999985 11016477 65536.00 + 22.975 0.999986 11016496 72817.78 + 23.055 0.999988 11016513 81920.00 + 23.119 0.999989 11016530 93622.86 + 23.167 0.999991 11016545 109226.67 + 23.231 0.999992 11016562 131072.00 + 23.279 0.999993 11016570 145635.56 + 23.343 0.999994 11016580 163840.00 + 23.375 0.999995 11016587 187245.71 + 23.439 0.999995 11016595 218453.33 + 23.519 0.999996 11016603 262144.00 + 23.583 0.999997 11016611 291271.11 + 23.599 0.999997 11016612 327680.00 + 23.663 0.999997 11016617 374491.43 + 23.695 0.999998 11016620 436906.67 + 23.759 0.999998 11016624 524288.00 + 23.791 0.999998 11016627 582542.22 + 23.823 0.999998 11016630 655360.00 + 23.903 0.999999 11016631 748982.86 + 23.951 0.999999 11016633 873813.33 + 24.047 0.999999 11016636 1048576.00 + 24.047 0.999999 11016636 1165084.44 + 24.079 0.999999 11016637 1310720.00 + 24.111 0.999999 11016638 1497965.71 + 24.175 0.999999 11016640 1747626.67 + 24.175 1.000000 11016640 2097152.00 + 24.207 1.000000 11016641 2330168.89 + 24.207 1.000000 11016641 2621440.00 + 24.463 1.000000 11016642 2995931.43 + 24.463 1.000000 11016642 3495253.33 + 24.591 1.000000 11016643 4194304.00 + 24.591 1.000000 11016643 4660337.78 + 24.591 1.000000 11016643 5242880.00 + 24.735 1.000000 11016644 5991862.86 + 24.735 1.000000 11016644 6990506.67 + 24.735 1.000000 11016644 8388608.00 + 24.735 1.000000 11016644 9320675.55 + 24.735 1.000000 11016644 10485760.00 + 24.959 1.000000 11016645 11983725.71 + 24.959 1.000000 11016645 inf +#[Mean = 1.546, StdDeviation = 1.542] +#[Max = 24.944, Total count = 11016645] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 11398279 requests in 5.00m, 2.33GB read + Non-2xx or 3xx responses: 5167527 +Requests/sec: 37994.39 +Transfer/sec: 7.96MB +------------------------------ + +HTTP Status 404 Count: 5167527 +HTTP Status 503 Count: 0 diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_alloc.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_alloc.html new file mode 100644 index 000000000..d43fd348b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_alloc.html @@ -0,0 +1,1226 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_cpu.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_cpu.html new file mode 100644 index 000000000..c66248d31 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_cpu.html @@ -0,0 +1,7066 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_lock.html b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_lock.html new file mode 100644 index 000000000..ed4b9beff --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_lock.html @@ -0,0 +1,695 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_wrk.txt b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_wrk.txt new file mode 100644 index 000000000..1b0dacfbe --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/html/stage3/cluster_4_stable_put_36000_wrk.txt @@ -0,0 +1,152 @@ +Running 5m test @ http://localhost:8080 + 8 threads and 128 connections + Thread calibration: mean lat.: 1.806ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.753ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.775ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.729ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.748ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.818ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.874ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.732ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.16ms 2.12ms 97.86ms 91.45% + Req/Sec 4.75k 628.79 18.33k 78.12% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.70ms + 75.000% 2.41ms + 90.000% 3.88ms + 99.000% 9.20ms + 99.900% 22.08ms + 99.990% 61.15ms + 99.999% 93.69ms +100.000% 97.92ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.048 0.000000 1 1.00 + 0.806 0.100000 1045061 1.11 + 1.050 0.200000 2089364 1.25 + 1.266 0.300000 3132926 1.43 + 1.478 0.400000 4175161 1.67 + 1.700 0.500000 5221801 2.00 + 1.818 0.550000 5741069 2.22 + 1.946 0.600000 6266172 2.50 + 2.085 0.650000 6789914 2.86 + 2.235 0.700000 7306894 3.33 + 2.411 0.750000 7829930 4.00 + 2.517 0.775000 8089213 4.44 + 2.649 0.800000 8350679 5.00 + 2.823 0.825000 8611455 5.71 + 3.065 0.850000 8871946 6.67 + 3.385 0.875000 9132957 8.00 + 3.603 0.887500 9263861 8.89 + 3.885 0.900000 9393935 10.00 + 4.227 0.912500 9525192 11.43 + 4.611 0.925000 9655406 13.33 + 5.043 0.937500 9785111 16.00 + 5.283 0.943750 9850808 17.78 + 5.539 0.950000 9915554 20.00 + 5.823 0.956250 9980819 22.86 + 6.143 0.962500 10046387 26.67 + 6.507 0.968750 10111401 32.00 + 6.719 0.971875 10143728 35.56 + 6.963 0.975000 10176626 40.00 + 7.243 0.978125 10208992 45.71 + 7.583 0.981250 10241732 53.33 + 8.007 0.984375 10274064 64.00 + 8.271 0.985938 10290407 71.11 + 8.583 0.987500 10306821 80.00 + 8.951 0.989062 10323190 91.43 + 9.391 0.990625 10339340 106.67 + 9.951 0.992188 10355760 128.00 + 10.295 0.992969 10363843 142.22 + 10.687 0.993750 10371952 160.00 + 11.159 0.994531 10380052 182.86 + 11.751 0.995313 10388178 213.33 + 12.551 0.996094 10396381 256.00 + 13.039 0.996484 10400430 284.44 + 13.623 0.996875 10404514 320.00 + 14.279 0.997266 10408591 365.71 + 15.119 0.997656 10412640 426.67 + 16.287 0.998047 10416719 512.00 + 17.119 0.998242 10418782 568.89 + 18.063 0.998437 10420795 640.00 + 19.183 0.998633 10422836 731.43 + 20.527 0.998828 10424872 853.33 + 22.303 0.999023 10426901 1024.00 + 23.311 0.999121 10427935 1137.78 + 24.623 0.999219 10428950 1280.00 + 26.671 0.999316 10429961 1462.86 + 29.679 0.999414 10430975 1706.67 + 32.735 0.999512 10431998 2048.00 + 34.175 0.999561 10432505 2275.56 + 35.839 0.999609 10433015 2560.00 + 37.887 0.999658 10433526 2925.71 + 41.503 0.999707 10434033 3413.33 + 47.519 0.999756 10434542 4096.00 + 50.815 0.999780 10434801 4551.11 + 53.407 0.999805 10435052 5120.00 + 55.615 0.999829 10435307 5851.43 + 57.439 0.999854 10435563 6826.67 + 59.231 0.999878 10435816 8192.00 + 60.159 0.999890 10435944 9102.22 + 61.471 0.999902 10436074 10240.00 + 62.879 0.999915 10436199 11702.86 + 64.319 0.999927 10436326 13653.33 + 66.367 0.999939 10436453 16384.00 + 68.479 0.999945 10436517 18204.44 + 73.855 0.999951 10436581 20480.00 + 78.399 0.999957 10436649 23405.71 + 81.279 0.999963 10436708 27306.67 + 86.719 0.999969 10436772 32768.00 + 88.959 0.999973 10436805 36408.89 + 89.855 0.999976 10436838 40960.00 + 90.431 0.999979 10436868 46811.43 + 91.007 0.999982 10436900 54613.33 + 92.031 0.999985 10436932 65536.00 + 92.607 0.999986 10436948 72817.78 + 92.927 0.999988 10436964 81920.00 + 93.503 0.999989 10436981 93622.86 + 94.079 0.999991 10436995 109226.67 + 94.783 0.999992 10437011 131072.00 + 94.975 0.999993 10437019 145635.56 + 95.231 0.999994 10437030 163840.00 + 95.359 0.999995 10437035 187245.71 + 95.679 0.999995 10437045 218453.33 + 95.871 0.999996 10437052 262144.00 + 95.935 0.999997 10437055 291271.11 + 96.127 0.999997 10437060 327680.00 + 96.255 0.999997 10437066 374491.43 + 96.319 0.999998 10437067 436906.67 + 96.447 0.999998 10437072 524288.00 + 96.639 0.999998 10437074 582542.22 + 96.703 0.999998 10437075 655360.00 + 96.895 0.999999 10437077 748982.86 + 96.959 0.999999 10437079 873813.33 + 97.087 0.999999 10437082 1048576.00 + 97.087 0.999999 10437082 1165084.44 + 97.215 0.999999 10437085 1310720.00 + 97.215 0.999999 10437085 1497965.71 + 97.215 0.999999 10437085 1747626.67 + 97.343 1.000000 10437087 2097152.00 + 97.343 1.000000 10437087 2330168.89 + 97.343 1.000000 10437087 2621440.00 + 97.343 1.000000 10437087 2995931.43 + 97.471 1.000000 10437088 3495253.33 + 97.471 1.000000 10437088 4194304.00 + 97.471 1.000000 10437088 4660337.78 + 97.535 1.000000 10437089 5242880.00 + 97.535 1.000000 10437089 5991862.86 + 97.535 1.000000 10437089 6990506.67 + 97.535 1.000000 10437089 8388608.00 + 97.535 1.000000 10437089 9320675.55 + 97.919 1.000000 10437090 10485760.00 + 97.919 1.000000 10437090 inf +#[Mean = 2.163, StdDeviation = 2.117] +#[Max = 97.856, Total count = 10437090] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 10798612 requests in 5.00m, 0.85GB read +Requests/sec: 35995.47 +Transfer/sec: 2.92MB diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/imgs/stage3_balancer.png b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/imgs/stage3_balancer.png new file mode 100644 index 000000000..e619a5afc Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/imgs/stage3_balancer.png differ diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/stage3.md b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/stage3.md index 793421806..e810f2080 100644 --- a/src/main/java/ru/vk/itmo/test/kovalevigor/reports/stage3.md +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/reports/stage3.md @@ -29,6 +29,8 @@ При этом, наверное, нужно было бы увеличить ограничение по памяти в зависимости от размера кластера до `128 * n` (где `n` - это количество нод). Однако, было решено просто уменьшить порог flush'а, чтобы все было ок +**GET** запросы проходят на базе суммарным объемом в 1GB после заполнения wrk при помощи [put_request.lua](scripts%2Flua%2Fput_request.lua) + ## Посмотрим, что меняется в зависимости от размера кластера *Т.к. rps снизился, то у нас меньше данных, следовательно в GET запросах больше 404, но мне не кажется это критичным из-за отсутсвия оптимизаций для отсутствующих записей.* @@ -48,7 +50,7 @@ #### GET -Остановились на 36000, 5min: +Остановились на 36000, 2min: [alloc](html/stage3/cluster_2_get_stable_40000_alloc.html) [cpu](html/stage3/cluster_2_get_stable_40000_cpu.html) [lock](html/stage3/cluster_2_get_stable_40000_lock.html) @@ -75,8 +77,7 @@ [35000](html/stage3/cluster_4_get_35000_wrk.txt) [36000](html/stage3/cluster_4_get_36000_wrk.txt) [38000](html/stage3/cluster_4_get_38000_wrk.txt) - [40000](html/stage3/ - cluster_4_get_40000_wrk.txt) + [40000](html/stage3/cluster_4_get_40000_wrk.txt) [80000](html/stage3/cluster_4_get_80000_wrk.txt) Остановились на 38000, 5min: @@ -115,3 +116,9 @@ Как я понимаю, в следующих заданиях мы будем решать эту проблему при помощи асинхронных запросов +### Проверка алгоритма балансировки + +В качестве алгоритма распределения ключей между серверами был выбран `consistent hashing` + +Эмпирические наблюдения показывают, что данные распределеяются между нодами равномерно +![stage3_balancer.png](imgs%2Fstage3_balancer.png) \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/server/strategy/decorators/ServerExecutorStrategyDecorator.java b/src/main/java/ru/vk/itmo/test/kovalevigor/server/strategy/decorators/ServerExecutorStrategyDecorator.java index c08fdfb05..b3ab3fce7 100644 --- a/src/main/java/ru/vk/itmo/test/kovalevigor/server/strategy/decorators/ServerExecutorStrategyDecorator.java +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/server/strategy/decorators/ServerExecutorStrategyDecorator.java @@ -25,6 +25,8 @@ public class ServerExecutorStrategyDecorator extends ServerStrategyDecorator implements RejectedExecutionHandler { private final ThreadPoolExecutor mainExecutor; private final Map executors; + private final long keepAliveTime; + private static final int SUB_EXECUTORS_POOL_SIZE = 1; public ServerExecutorStrategyDecorator( ServerStrategy serverStrategy, @@ -32,17 +34,18 @@ public ServerExecutorStrategyDecorator( long keepAliveTime, int queueCapacity ) { super(serverStrategy); - executors = new HashMap<>(); + this.executors = new HashMap<>(); + this.keepAliveTime = keepAliveTime; int realCorePoolSize = corePoolSize == 1 ? 1 : corePoolSize / 2; - mainExecutor = new ThreadPoolExecutor( + this.mainExecutor = new ThreadPoolExecutor( realCorePoolSize, maximumPoolSize - realCorePoolSize, - keepAliveTime, + this.keepAliveTime, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(queueCapacity * 10), + new LinkedBlockingQueue<>(queueCapacity * realCorePoolSize), new CustomPolicy() ); - mainExecutor.prestartAllCoreThreads(); + this.mainExecutor.prestartAllCoreThreads(); } private class CustomPolicy implements RejectedExecutionHandler { @@ -82,12 +85,12 @@ public void start(ServerFull httpServer) { ThreadPoolExecutor[] threadPoolExecutors = new ThreadPoolExecutor[mainExecutor.getCorePoolSize()]; for (int i = 0; i < threadPoolExecutors.length; i++) { threadPoolExecutors[i] = new ThreadPoolExecutor( - 1, - 1, - 100, + SUB_EXECUTORS_POOL_SIZE, + SUB_EXECUTORS_POOL_SIZE, + this.keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<>( - mainExecutor.getQueue().remainingCapacity() / 10 + mainExecutor.getQueue().remainingCapacity() / mainExecutor.getCorePoolSize() ), this ); diff --git a/src/main/java/ru/vk/itmo/test/kovalevigor/server/util/ServerUtil.java b/src/main/java/ru/vk/itmo/test/kovalevigor/server/util/ServerUtil.java index c265f94a2..0037f0da3 100644 --- a/src/main/java/ru/vk/itmo/test/kovalevigor/server/util/ServerUtil.java +++ b/src/main/java/ru/vk/itmo/test/kovalevigor/server/util/ServerUtil.java @@ -12,6 +12,8 @@ public final class ServerUtil { + public static final TimeUnit SHUTDOWN_TIMEOUT_TIME_UNIT = TimeUnit.SECONDS; + public static final int SHUTDOWN_TIMEOUT = 60; public static final Set GOOD_STATUSES = Set.of( 200, 201, 202, 404 ); @@ -42,10 +44,10 @@ public static void shutdownAndAwaitTermination(ExecutorService pool) { pool.shutdown(); // Disable new tasks from being submitted try { // Wait a while for existing tasks to terminate - if (!pool.awaitTermination(60, TimeUnit.SECONDS)) { + if (!pool.awaitTermination(SHUTDOWN_TIMEOUT, SHUTDOWN_TIMEOUT_TIME_UNIT)) { pool.shutdownNow(); // Cancel currently executing tasks // Wait a while for tasks to respond to being cancelled - if (!pool.awaitTermination(60, TimeUnit.SECONDS)) { + if (!pool.awaitTermination(SHUTDOWN_TIMEOUT, SHUTDOWN_TIMEOUT_TIME_UNIT)) { log.severe("Pool did not terminate"); } } diff --git a/src/main/java/ru/vk/itmo/test/reference/HandleResult.java b/src/main/java/ru/vk/itmo/test/reference/HandleResult.java new file mode 100644 index 000000000..8055e91e7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/HandleResult.java @@ -0,0 +1,14 @@ +package ru.vk.itmo.test.reference; + +public record HandleResult(int status, byte[] data, long timestamp) { + + public HandleResult(int status, byte[] data, long timestamp) { + this.status = status; + this.data = data; + this.timestamp = timestamp; + } + + public HandleResult(int status, byte[] data) { + this(status, data, 0); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/MergeHandleResult.java b/src/main/java/ru/vk/itmo/test/reference/MergeHandleResult.java new file mode 100644 index 000000000..64a1b95aa --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/MergeHandleResult.java @@ -0,0 +1,85 @@ +package ru.vk.itmo.test.reference; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import one.nio.http.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +public class MergeHandleResult { + private static final Logger log = LoggerFactory.getLogger(MergeHandleResult.class); + private final HandleResult[] handleResults; + private final AtomicInteger successCount = new AtomicInteger(); + private final AtomicInteger count = new AtomicInteger(); + private final int ack; + private final int from; + private final ReferenceHttpSession session; + private final ExecutorService executorMerge; + + public MergeHandleResult(ReferenceHttpSession session, ExecutorService executorMerge, int size, int ack) { + this.session = session; + this.handleResults = new HandleResult[size]; + this.ack = ack; + this.from = size; + this.executorMerge = executorMerge; + } + + + public void add(int index, CompletableFuture handleResult) { + merge(handleResult, index); + } + + //FIXME + @CanIgnoreReturnValue + private CompletableFuture merge(CompletableFuture handleResult, int index) { + return handleResult .whenCompleteAsync((handleResult1, throwable) -> { + if (throwable != null) { + if (throwable instanceof IOException e) { + log.info("I/O exception", e); + } else if (throwable instanceof Exception e) { + log.info("Exception in competeAsync for merging", e); + } else { + // todo error + } + } + if (handleResult1 != null) { + if (handleResult1.status() == HttpURLConnection.HTTP_OK + || handleResult1.status() == HttpURLConnection.HTTP_CREATED + || handleResult1.status() == HttpURLConnection.HTTP_ACCEPTED + || handleResult1.status() == HttpURLConnection.HTTP_NOT_FOUND) { + handleResults[index] = handleResult1; + int get = successCount.incrementAndGet(); + if (get == ack) { + sendResult(); + return; + } + } + } + int currentCount = count.incrementAndGet(); + if (currentCount == from && successCount.get() < ack) { + session.sendResponseOrClose(new Response(Response.GATEWAY_TIMEOUT, ("ack/from:" + ack + "/" + from).getBytes(StandardCharsets.UTF_8))); + } + }, executorMerge); + } + + + private void sendResult() { + HandleResult mergedResult = new HandleResult(HttpURLConnection.HTTP_GATEWAY_TIMEOUT, null); + for (HandleResult handleResult : handleResults) { + if (handleResult != null) { + if (mergedResult.timestamp() <= handleResult.timestamp()) { + mergedResult = handleResult; + } + } + } + + session.sendResponseOrClose(new Response(String.valueOf(mergedResult.status()), mergedResult.data())); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/ReferenceHttpSession.java b/src/main/java/ru/vk/itmo/test/reference/ReferenceHttpSession.java new file mode 100644 index 000000000..df22088af --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/ReferenceHttpSession.java @@ -0,0 +1,33 @@ +package ru.vk.itmo.test.reference; + +import one.nio.http.HttpServer; +import one.nio.http.HttpSession; +import one.nio.http.Response; +import one.nio.net.Socket; + +import java.io.IOException; + +public class ReferenceHttpSession extends HttpSession { + public ReferenceHttpSession(Socket socket, HttpServer server) { + super(socket, server); + } + + public void sendResponseOrClose(Response response) { + try { + sendResponse(response); + } catch (IOException e) { + log.error("Exception while sending close connection", e); + scheduleClose(); + } + } + + public void sendError(Throwable e) { + log.error("Exception during handleRequest", e); + try { + sendResponse(new Response(Response.INTERNAL_ERROR, Response.EMPTY)); + } catch (IOException ex) { + log.error("Exception while sending close connection", e); + scheduleClose(); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/ReferenceServer.java b/src/main/java/ru/vk/itmo/test/reference/ReferenceServer.java index 12bb05175..cb0d54874 100644 --- a/src/main/java/ru/vk/itmo/test/reference/ReferenceServer.java +++ b/src/main/java/ru/vk/itmo/test/reference/ReferenceServer.java @@ -1,55 +1,63 @@ package ru.vk.itmo.test.reference; import one.nio.async.CustomThreadFactory; -import one.nio.http.HttpServer; -import one.nio.http.HttpServerConfig; -import one.nio.http.HttpSession; -import one.nio.http.Request; -import one.nio.http.Response; -import one.nio.http.Path; +import one.nio.http.*; +import one.nio.net.Socket; import one.nio.server.AcceptorConfig; +import one.nio.server.RejectedSessionException; import one.nio.util.Hash; import one.nio.util.Utf8; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import ru.vk.itmo.ServiceConfig; -import ru.vk.itmo.dao.BaseEntry; -import ru.vk.itmo.dao.Dao; -import ru.vk.itmo.dao.Entry; +import ru.vk.itmo.test.reference.dao2.ReferenceBaseEntry; +import ru.vk.itmo.test.reference.dao2.ReferenceDao; import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.net.HttpURLConnection; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; -import java.time.temporal.ChronoUnit; -import java.util.concurrent.*; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; public class ReferenceServer extends HttpServer { + private static final String HEADER_REMOTE = "X-nodep"; + private static final String HEADER_REMOTE_ONE_NIO_HEADER = HEADER_REMOTE + ": da"; + private static final String HEADER_TIMESTAMP = "X-rmfp"; + private static final String HEADER_TIMESTAMP_ONE_NIO_HEADER = HEADER_TIMESTAMP + ": "; private static final Logger log = LoggerFactory.getLogger(ReferenceServer.class); private final static int THREADS = Runtime.getRuntime().availableProcessors(); - private final ExecutorService executorLocal = Executors.newFixedThreadPool(THREADS / 2, new CustomThreadFactory("local-work")); - private final ExecutorService executorRemote = Executors.newFixedThreadPool(THREADS / 2, new CustomThreadFactory("remote-work")); - private final Dao> dao; + private final ExecutorService executorLocal = Executors.newFixedThreadPool(THREADS, new CustomThreadFactory("local-work")); + private final ExecutorService executorWork = Executors.newFixedThreadPool(THREADS, new CustomThreadFactory("work-thread")); + private final ExecutorService clientExecutor = Executors.newFixedThreadPool(THREADS, new CustomThreadFactory("client-executor")); + private final ExecutorService responseExecutor = Executors.newFixedThreadPool(THREADS, new CustomThreadFactory("response-executor")); + + private final ExecutorService executorMerge = Executors.newFixedThreadPool(THREADS); + ExecutorService executorService; + private final ReferenceDao dao; private final ServiceConfig config; private final HttpClient httpClient; public ReferenceServer(ServiceConfig config, - Dao> dao) throws IOException { + ReferenceDao dao) throws IOException { super(createServerConfigWithPort(config.selfPort())); this.dao = dao; this.config = config; + executorService = Executors.newSingleThreadExecutor(); + this.httpClient = HttpClient.newBuilder() - .executor(Executors.newFixedThreadPool(THREADS)) + .executor(clientExecutor) .connectTimeout(Duration.ofMillis(500)) .version(HttpClient.Version.HTTP_1_1) .build(); @@ -62,6 +70,7 @@ private static HttpServerConfig createServerConfigWithPort(int port) { acceptorConfig.reusePort = true; // acceptorConfig.threads = Runtime.getRuntime().availableProcessors() / 2; serverConfig.selectors = Runtime.getRuntime().availableProcessors() / 2; +// serverConfig.keepAlive = 60_000; serverConfig.acceptors = new AcceptorConfig[]{acceptorConfig}; serverConfig.closeSessions = true; @@ -69,66 +78,112 @@ private static HttpServerConfig createServerConfigWithPort(int port) { } @Override - public void handleRequest(Request request, HttpSession session) throws IOException { - if (!"/v0/entity".equals(request.getPath())) { - session.sendError(Response.BAD_REQUEST, null); - return; + public void handleRequest(Request request, HttpSession sessionI) throws IOException { + if (!(sessionI instanceof ReferenceHttpSession session)) { + throw new IllegalArgumentException("this method support only ReferenceHttpSession"); } - +// if (config.selfPort() == 8100) { +// return; +// } String id = request.getParameter("id="); if (id == null || id.isBlank()) { session.sendError(Response.BAD_REQUEST, null); return; } + if (request.getHeader(HEADER_REMOTE_ONE_NIO_HEADER) != null) { - String executorNode = getNodeByEntityId(id); - if (executorNode.equals(config.selfUrl())) { - handleAsync(session, executorLocal, () -> local(request, session, id)); - } else { - handleAsync(session, executorRemote, () -> remote(request, session, executorNode)); - } - } + executorLocal.execute(new Runnable() { + @Override + public void run() { + try { + HandleResult local = localInternal(request, id); - private void remote(Request request, HttpSession session, String executorNode) throws IOException { + Response response = new Response(String.valueOf(local.status()), local.data()); + response.addHeader(HEADER_TIMESTAMP_ONE_NIO_HEADER + local.timestamp()); + session.sendResponseOrClose(response); + } catch (Exception e) { + session.sendError(e); + } + } + }); - try { - Response response = invokeRemote(executorNode, request); - session.sendResponse(response); - } catch (IOException e) { - log.info("I/O exception while calling remote node", e); - session.sendResponse(new Response(Response.INTERNAL_ERROR, Response.EMPTY)); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.info("Thread interrupted"); - session.sendResponse(new Response(Response.SERVICE_UNAVAILABLE, Response.EMPTY)); + return; } + executorWork.execute(new Runnable() { + @Override + public void run() { + try { + + + if (!"/v0/entity".equals(request.getPath())) { + session.sendError(Response.BAD_REQUEST, null); + return; + } + + if (request.getMethod() != Request.METHOD_GET + && request.getMethod() != Request.METHOD_DELETE + && request.getMethod() != Request.METHOD_PUT) { + session.sendError(Response.METHOD_NOT_ALLOWED, null); + return; + } + + + int ack = getInt(request, "ack=", config.clusterUrls().size() / 2 + 1); + int from = getInt(request, "from=", config.clusterUrls().size()); + + if (from <= 0 || from > config.clusterUrls().size() || ack > from || ack <= 0) { + session.sendError(Response.BAD_REQUEST, "Illegal argument ack/from " + ack + "/" + from); + return; + } + + + int[] indexes = getIndexes(id, from); //todo ??? + MergeHandleResult mergeHandleResult = new MergeHandleResult(session, executorMerge, indexes.length, ack); + for (int i = 0; i < indexes.length; i++) { + int index = indexes[i]; + String executorNode = config.clusterUrls().get(index); + if (executorNode.equals(config.selfUrl())) { + handle(i, mergeHandleResult, () -> local(request, id)); + } else { + handle(i, mergeHandleResult, () -> remote(request, executorNode)); + } + } + } catch (Exception e) { + session.sendError(e); + } + } + + }); + } + + private int getInt(Request request, String param, int defaultValue) throws IOException { + int ack; + String ackStr = request.getParameter(param); + if (ackStr == null || ackStr.isBlank()) { + ack = defaultValue; + } else { + try { + ack = Integer.parseInt(ackStr); + } catch (Exception e) { + // todo ваша ошибка + throw new IllegalArgumentException("parse error"); + } + } + return ack; } - private void local(Request request, HttpSession session, String id) throws IOException { - Response response = invokeLocal(request, id); - session.sendResponse(response); + private CompletableFuture remote(Request request, String executorNode) { + return invokeRemote(executorNode, request); } - private void handleAsync(HttpSession session, ExecutorService executor, ERunnable runnable) throws IOException { + private void handle(int index, MergeHandleResult mergeHandleResult, ERunnable runnable) { try { - executor.execute(() -> { - try { - runnable.run(); - } catch (Exception e) { - log.error("Exception during handleRequest", e); - try { - session.sendError(Response.INTERNAL_ERROR, null); - } catch (IOException ex) { - log.error("Exception while sending close connection", e); - session.scheduleClose(); - } - } - }); - } catch (RejectedExecutionException e) { - log.warn("Workers pool queue overflow", e); - session.sendError(Response.SERVICE_UNAVAILABLE, null); + CompletableFuture handleResult = runnable.run(); + mergeHandleResult.add(index, handleResult); + } catch (Exception e) { + mergeHandleResult.add(index, CompletableFuture.failedFuture(e)); } } @@ -137,12 +192,7 @@ public void handleDefault(Request request, HttpSession session) throws IOExcepti session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); } - @Path("/v0/status") - public Response status() { - return Response.ok("OK"); - } - - private Response invokeRemote(String executorNode, Request request) throws IOException, InterruptedException { + private CompletableFuture invokeRemote(String executorNode, Request request) { HttpRequest httpRequest = HttpRequest.newBuilder(URI.create(executorNode + request.getURI())) .method( request.getMethodName(), @@ -150,63 +200,108 @@ private Response invokeRemote(String executorNode, Request request) throws IOExc ? HttpRequest.BodyPublishers.noBody() : HttpRequest.BodyPublishers.ofByteArray(request.getBody()) ) + .header(HEADER_REMOTE, "da") .timeout(Duration.ofMillis(500)) .build(); - HttpResponse httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofByteArray()); - return new Response(Integer.toString(httpResponse.statusCode()), httpResponse.body()); + + CompletableFuture> future = httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofByteArray()); + return future.thenApplyAsync(httpResponse -> { + Optional string = httpResponse.headers().firstValue(HEADER_TIMESTAMP); + long timestamp; + if (string.isPresent()) { + try { + timestamp = Long.parseLong(string.get()); + } catch (Exception e) { + log.error("todo "); + timestamp = 0; + } + } else { + timestamp = 0; + } + return new HandleResult(httpResponse.statusCode(), httpResponse.body(), timestamp); + }, responseExecutor); + } - private Response invokeLocal(Request request, String id) { + private CompletableFuture local(Request request, String id) { + return CompletableFuture.supplyAsync(() -> localInternal(request, id), executorLocal); + } + + private HandleResult localInternal(Request request, String id) { + long currentTimeMillis = System.currentTimeMillis(); switch (request.getMethod()) { case Request.METHOD_GET -> { MemorySegment key = MemorySegment.ofArray(Utf8.toBytes(id)); - Entry entry = dao.get(key); + ReferenceBaseEntry entry = dao.get(key); if (entry == null) { - return new Response(Response.NOT_FOUND, Response.EMPTY); + return new HandleResult(HttpURLConnection.HTTP_NOT_FOUND, Response.EMPTY); + } + if (entry.value() == null) { + return new HandleResult(HttpURLConnection.HTTP_NOT_FOUND, Response.EMPTY, entry.timestamp()); } - return Response.ok(entry.value().toArray(ValueLayout.JAVA_BYTE)); + return new HandleResult(HttpURLConnection.HTTP_OK, entry.value().toArray(ValueLayout.JAVA_BYTE), entry.timestamp()); } case Request.METHOD_PUT -> { MemorySegment key = MemorySegment.ofArray(Utf8.toBytes(id)); MemorySegment value = MemorySegment.ofArray(request.getBody()); - dao.upsert(new BaseEntry<>(key, value)); - return new Response(Response.CREATED, Response.EMPTY); + dao.upsert(new ReferenceBaseEntry<>(key, value, currentTimeMillis)); + return new HandleResult(HttpURLConnection.HTTP_CREATED, Response.EMPTY); } case Request.METHOD_DELETE -> { MemorySegment key = MemorySegment.ofArray(Utf8.toBytes(id)); - dao.upsert(new BaseEntry<>(key, null)); - return new Response(Response.ACCEPTED, Response.EMPTY); + dao.upsert(new ReferenceBaseEntry<>(key, null, currentTimeMillis)); + return new HandleResult(HttpURLConnection.HTTP_ACCEPTED, Response.EMPTY); } default -> { - return new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); + return new HandleResult(HttpURLConnection.HTTP_BAD_METHOD, Response.EMPTY); } } } + // count <= config.clusterUrls().size() + private int[] getIndexes(String id, int count) { + assert count < 5; + + int[] result = new int[count]; + int[] maxHashs = new int[count]; - private String getNodeByEntityId(String id) { - int nodeId = 0; - int maxHash = Hash.murmur3(config.clusterUrls().getFirst() + id); - for (int i = 1; i < config.clusterUrls().size(); i++) { + for (int i = 0; i < count; i++) { String url = config.clusterUrls().get(i); - int result = Hash.murmur3(url + id); - if (maxHash < result) { - maxHash = result; - nodeId = i; + int hash = Hash.murmur3(url + id); + result[i] = i; + maxHashs[i] = hash; + } + + for (int i = count; i < config.clusterUrls().size(); i++) { + String url = config.clusterUrls().get(i); + int hash = Hash.murmur3(url + id); + for (int j = 0; j < maxHashs.length; j++) { + int maxHash = maxHashs[j]; + if (maxHash < hash) { + maxHashs[j] = hash; + result[j] = i; + break; + } } } - return config.clusterUrls().get(nodeId); + return result; } + //todo naming private interface ERunnable { - void run() throws Exception; + CompletableFuture run() throws Exception; } @Override public synchronized void stop() { super.stop(); ReferenceService.shutdownAndAwaitTermination(executorLocal); - ReferenceService.shutdownAndAwaitTermination(executorRemote); + ReferenceService.shutdownAndAwaitTermination(executorWork); + } + + @Override + public HttpSession createSession(Socket socket) throws RejectedSessionException { + return new ReferenceHttpSession(socket, this); } } \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/reference/ReferenceService.java b/src/main/java/ru/vk/itmo/test/reference/ReferenceService.java index 31da7695e..039ad7c61 100644 --- a/src/main/java/ru/vk/itmo/test/reference/ReferenceService.java +++ b/src/main/java/ru/vk/itmo/test/reference/ReferenceService.java @@ -1,16 +1,12 @@ package ru.vk.itmo.test.reference; -import one.nio.async.CustomThreadFactory; import ru.vk.itmo.Service; import ru.vk.itmo.ServiceConfig; import ru.vk.itmo.dao.Config; -import ru.vk.itmo.dao.Dao; -import ru.vk.itmo.dao.Entry; import ru.vk.itmo.test.ServiceFactory; -import ru.vk.itmo.test.reference.dao.ReferenceDao; +import ru.vk.itmo.test.reference.dao2.ReferenceDao; import java.io.IOException; -import java.lang.foreign.MemorySegment; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -18,14 +14,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.*; public class ReferenceService implements Service { @@ -35,9 +24,10 @@ public class ReferenceService implements Service { private final ServiceConfig config; - private Dao> dao; + private ReferenceDao dao; private ReferenceServer server; private boolean stopped; + public ReferenceService(ServiceConfig config) { this.config = config; } @@ -81,7 +71,7 @@ public static void shutdownAndAwaitTermination(ExecutorService pool) { } } - @ServiceFactory(stage = 3) + @ServiceFactory(stage = 5) public static class Factory implements ServiceFactory.Factory { @Override @@ -106,10 +96,7 @@ public static void main(String[] args) throws IOException { String url = entry.getValue(); Path path = Paths.get("tmp/db/" + port); Files.createDirectories(path); - ServiceConfig serviceConfig = new ServiceConfig(port, - url, - clusterUrls, - path); + ServiceConfig serviceConfig = new ServiceConfig(port, url, clusterUrls, path); clusterConfs.add(serviceConfig); } diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/ByteArraySegment.java b/src/main/java/ru/vk/itmo/test/reference/dao2/ByteArraySegment.java new file mode 100644 index 000000000..1a69233eb --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/ByteArraySegment.java @@ -0,0 +1,48 @@ +package ru.vk.itmo.test.reference.dao2; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.nio.ByteBuffer; + +/** + * Growable buffer with {@link ByteBuffer} and {@link MemorySegment} interface. + * + * @author incubos + */ +final class ByteArraySegment { + private byte[] array; + private MemorySegment segment; + + ByteArraySegment(final int capacity) { + this.array = new byte[capacity]; + this.segment = MemorySegment.ofArray(array); + } + + void withArray(final ArrayConsumer consumer) throws IOException { + consumer.process(array); + } + + MemorySegment segment() { + return segment; + } + + void ensureCapacity(final long size) { + if (size > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Too big!"); + } + + final int capacity = (int) size; + if (array.length >= capacity) { + return; + } + + // Grow to the nearest bigger power of 2 + final int newSize = Integer.highestOneBit(capacity) << 1; + array = new byte[newSize]; + segment = MemorySegment.ofArray(array); + } + + interface ArrayConsumer { + void process(byte[] array) throws IOException; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/LiveFilteringIterator.java b/src/main/java/ru/vk/itmo/test/reference/dao2/LiveFilteringIterator.java new file mode 100644 index 000000000..aaa952122 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/LiveFilteringIterator.java @@ -0,0 +1,52 @@ +package ru.vk.itmo.test.reference.dao2; + +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Filters non tombstone {@link Entry}s. + * + * @author incubos + */ +final class LiveFilteringIterator implements Iterator> { + private final Iterator> delegate; + private ReferenceBaseEntry next; + + LiveFilteringIterator(final Iterator> delegate) { + this.delegate = delegate; + skipTombstones(); + } + + private void skipTombstones() { + while (delegate.hasNext()) { + final ReferenceBaseEntry entry = delegate.next(); + if (entry.value() != null) { + this.next = entry; + break; + } + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public ReferenceBaseEntry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Consume + final ReferenceBaseEntry result = next; + next = null; + + skipTombstones(); + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/MemTable.java b/src/main/java/ru/vk/itmo/test/reference/dao2/MemTable.java new file mode 100644 index 000000000..ce032950a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/MemTable.java @@ -0,0 +1,49 @@ +package ru.vk.itmo.test.reference.dao2; + +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Memory table. + * + * @author incubos + */ +final class MemTable { + private final NavigableMap> map = + new ConcurrentSkipListMap<>( + MemorySegmentComparator.INSTANCE); + + boolean isEmpty() { + return map.isEmpty(); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + if (from == null && to == null) { + // All + return map.values().iterator(); + } else if (from == null) { + // Head + return map.headMap(to).values().iterator(); + } else if (to == null) { + // Tail + return map.tailMap(from).values().iterator(); + } else { + // Slice + return map.subMap(from, to).values().iterator(); + } + } + + ReferenceBaseEntry get(final MemorySegment key) { + return map.get(key); + } + + ReferenceBaseEntry upsert(final ReferenceBaseEntry entry) { + return map.put(entry.key(), entry); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/test/reference/dao2/MemorySegmentComparator.java new file mode 100644 index 000000000..b65534aca --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/MemorySegmentComparator.java @@ -0,0 +1,89 @@ +package ru.vk.itmo.test.reference.dao2; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +/** + * Compares {@link MemorySegment}s. + * + * @author incubos + */ +final class MemorySegmentComparator implements Comparator { + static final Comparator INSTANCE = + new MemorySegmentComparator(); + + private MemorySegmentComparator() { + // Singleton + } + + @Override + public int compare( + final MemorySegment left, + final MemorySegment right) { + final long mismatch = left.mismatch(right); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == left.byteSize()) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == right.byteSize()) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + left.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch), + right.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch)); + } + + static int compare( + final MemorySegment srcSegment, + final long srcFromOffset, + final long srcLength, + final MemorySegment dstSegment, + final long dstFromOffset, + final long dstLength) { + final long mismatch = + MemorySegment.mismatch( + srcSegment, + srcFromOffset, + srcFromOffset + srcLength, + dstSegment, + dstFromOffset, + dstFromOffset + dstLength); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == srcLength) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == dstLength) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + srcSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + srcFromOffset + mismatch), + dstSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + dstFromOffset + mismatch)); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/MergingEntryIterator.java b/src/main/java/ru/vk/itmo/test/reference/dao2/MergingEntryIterator.java new file mode 100644 index 000000000..31118ff94 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/MergingEntryIterator.java @@ -0,0 +1,70 @@ +package ru.vk.itmo.test.reference.dao2; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +/** + * Merges entry {@link Iterator}s. + * + * @author incubos + */ +final class MergingEntryIterator implements Iterator> { + private final Queue iterators; + + MergingEntryIterator(final List iterators) { + assert iterators.stream().allMatch(WeightedPeekingEntryIterator::hasNext); + + this.iterators = new PriorityQueue<>(iterators); + } + + @Override + public boolean hasNext() { + return !iterators.isEmpty(); + } + + @Override + public ReferenceBaseEntry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final WeightedPeekingEntryIterator top = iterators.remove(); + final ReferenceBaseEntry result = top.next(); + + if (top.hasNext()) { + // Not exhausted + iterators.add(top); + } + + // Remove older versions of the key + while (true) { + final WeightedPeekingEntryIterator iterator = iterators.peek(); + if (iterator == null) { + // Nothing left + break; + } + + // Skip entries with the same key + final ReferenceBaseEntry entry = iterator.peek(); + if (MemorySegmentComparator.INSTANCE.compare(result.key(), entry.key()) != 0) { + // Reached another key + break; + } + + // Drop + iterators.remove(); + // Skip + iterator.next(); + if (iterator.hasNext()) { + // Not exhausted + iterators.add(iterator); + } + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/ReferenceBaseEntry.java b/src/main/java/ru/vk/itmo/test/reference/dao2/ReferenceBaseEntry.java new file mode 100644 index 000000000..c9f0ec898 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/ReferenceBaseEntry.java @@ -0,0 +1,10 @@ +package ru.vk.itmo.test.reference.dao2; + +import ru.vk.itmo.dao.Entry; + +public record ReferenceBaseEntry(D key, D value, long timestamp) implements Entry { + @Override + public String toString() { + return "{" + key + ":" + value + "," + timestamp + "}"; + } +} \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/ReferenceDao.java b/src/main/java/ru/vk/itmo/test/reference/dao2/ReferenceDao.java new file mode 100644 index 000000000..baab8c3b0 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/ReferenceDao.java @@ -0,0 +1,292 @@ +package ru.vk.itmo.test.reference.dao2; + +import ru.vk.itmo.dao.Config; +import ru.vk.itmo.dao.Dao; +import ru.vk.itmo.dao.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Reference implementation of {@link Dao}. + * + * @author incubos + */ +public class ReferenceDao implements Dao> { + private final Config config; + private final Arena arena; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + // Guarded by lock + private volatile TableSet tableSet; + + private final ExecutorService flusher = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("flusher"); + return result; + }); + private final ExecutorService compactor = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("compactor"); + return result; + }); + + private final AtomicBoolean closed = new AtomicBoolean(); + + public ReferenceDao(final Config config) throws IOException { + this.config = config; + this.arena = Arena.ofShared(); + + // First complete promotion of compacted SSTables + SSTables.promote( + config.basePath(), + 0, + 1); + + this.tableSet = + TableSet.from( + SSTables.discover( + arena, + config.basePath())); + } + + @Override + public Iterator> get( + final MemorySegment from, + final MemorySegment to) { + return new LiveFilteringIterator( + tableSet.get( + from, + to)); + } + + @Override + public ReferenceBaseEntry get(final MemorySegment key) { + // Without lock, just snapshot of table set + return tableSet.get(key); + } + + @Override + public void upsert(final ReferenceBaseEntry entry) { + final boolean autoFlush; + lock.readLock().lock(); + try { + if (tableSet.memTableSize.get() > config.flushThresholdBytes() + && tableSet.flushingTable != null) { + throw new IllegalStateException("Can't keep up with flushing!"); + } + + // Upsert + final Entry previous = tableSet.upsert(entry); + + // Update size estimate + final long size = tableSet.memTableSize.addAndGet(sizeOf(entry) - sizeOf(previous)); + autoFlush = size > config.flushThresholdBytes(); + } finally { + lock.readLock().unlock(); + } + + if (autoFlush) { + initiateFlush(true); + } + } + + private static long sizeOf(final Entry entry) { + if (entry == null) { + return 0L; + } + + if (entry.value() == null) { + return entry.key().byteSize(); + } + + return entry.key().byteSize() + entry.value().byteSize(); + } + + private void initiateFlush(final boolean auto) { + flusher.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + if (this.tableSet.memTable.isEmpty()) { + // Nothing to flush + return; + } + + if (auto && this.tableSet.memTableSize.get() < config.flushThresholdBytes()) { + // Not enough data to flush + return; + } + + // Switch memTable to flushing + currentTableSet = this.tableSet.flushing(); + this.tableSet = currentTableSet; + } finally { + lock.writeLock().unlock(); + } + + // Write + final int sequence = currentTableSet.nextSequence(); + try { + new SSTableWriter() + .write( + config.basePath(), + sequence, + currentTableSet.flushingTable.get(null, null)); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-1); + return; + } + + // Open + final SSTable flushed; + try { + flushed = SSTables.open( + arena, + config.basePath(), + sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-2); + return; + } + + // Switch + lock.writeLock().lock(); + try { + this.tableSet = this.tableSet.flushed(flushed); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void flush() throws IOException { + initiateFlush(false); + } + + @Override + public void compact() throws IOException { + compactor.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + currentTableSet = this.tableSet; + if (currentTableSet.ssTables.size() < 2) { + // Nothing to compact + return; + } + } finally { + lock.writeLock().unlock(); + } + + // Compact to 0 + try { + new SSTableWriter() + .write( + config.basePath(), + 0, + new LiveFilteringIterator( + currentTableSet.allSSTableEntries())); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-3); + } + + // Open 0 + final SSTable compacted; + try { + compacted = + SSTables.open( + arena, + config.basePath(), + 0); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-4); + return; + } + + // Replace old SSTables with compacted one to + // keep serving requests + final Set replaced = new HashSet<>(currentTableSet.ssTables); + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + replaced, + compacted); + } finally { + lock.writeLock().unlock(); + } + + // Remove compacted SSTables starting from the oldest ones. + // If we crash, 0 contains all the data, and + // it will be promoted on reopen. + for (final SSTable ssTable : currentTableSet.ssTables.reversed()) { + try { + SSTables.remove( + config.basePath(), + ssTable.sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-5); + } + } + + // Promote zero to one (possibly replacing) + try { + SSTables.promote( + config.basePath(), + 0, + 1); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-6); + } + + // Replace promoted SSTable + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + Collections.singleton(compacted), + compacted.withSequence(1)); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void close() throws IOException { + if (closed.getAndSet(true)) { + // Already closed + return; + } + + // Maybe flush + flush(); + + // Stop all the threads + flusher.close(); + compactor.close(); + + // Close arena + arena.close(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/SSTable.java b/src/main/java/ru/vk/itmo/test/reference/dao2/SSTable.java new file mode 100644 index 000000000..880f09bd3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/SSTable.java @@ -0,0 +1,208 @@ +package ru.vk.itmo.test.reference.dao2; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Persistent SSTable in data file and index file. + * + * @author incubos + * @see SSTables + */ +final class SSTable { + final int sequence; + + private final MemorySegment index; + private final MemorySegment data; + private final long size; + + SSTable( + final int sequence, + final MemorySegment index, + final MemorySegment data) { + this.sequence = sequence; + this.index = index; + this.data = data; + this.size = index.byteSize() / Long.BYTES; + } + + SSTable withSequence(final int sequence) { + return new SSTable( + sequence, + index, + data); + } + + /** + * Returns index of the entry if found; otherwise, (-(insertion point) - 1). + * The insertion point is defined as the point at which the key would be inserted: + * the index of the first element greater than the key, + * or size if all keys are less than the specified key. + * Note that this guarantees that the return value will be >= 0 + * if and only if the key is found. + */ + private long entryBinarySearch(final MemorySegment key) { + long low = 0L; + long high = size - 1; + + while (low <= high) { + final long mid = (low + high) >>> 1; + final long midEntryOffset = entryOffset(mid); + final long midKeyLength = getLength(midEntryOffset); + final int compare = + MemorySegmentComparator.compare( + data, + midEntryOffset + Long.BYTES, // Position at key + midKeyLength, + key, + 0L, + key.byteSize()); + + if (compare < 0) { + low = mid + 1; + } else if (compare > 0) { + high = mid - 1; + } else { + return mid; + } + } + + return -(low + 1); + } + + private long entryOffset(final long entry) { + return index.get( + ValueLayout.OfLong.JAVA_LONG, + entry * Long.BYTES); + } + + private long getLength(final long offset) { + return data.get( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + offset); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + assert from == null || to == null || MemorySegmentComparator.INSTANCE.compare(from, to) <= 0; + + // Slice of SSTable in absolute offsets + final long fromOffset; + final long toOffset; + + // Left offset bound + if (from == null) { + // Start from the beginning + fromOffset = 0L; + } else { + final long fromEntry = entryBinarySearch(from); + if (fromEntry >= 0L) { + fromOffset = entryOffset(fromEntry); + } else if (-fromEntry - 1 == size) { + // No relevant data + return Collections.emptyIterator(); + } else { + // Greater but existing key found + fromOffset = entryOffset(-fromEntry - 1); + } + } + + // Right offset bound + if (to == null) { + // Up to the end + toOffset = data.byteSize(); + } else { + final long toEntry = entryBinarySearch(to); + if (toEntry >= 0L) { + toOffset = entryOffset(toEntry); + } else if (-toEntry - 1 == size) { + // Up to the end + toOffset = data.byteSize(); + } else { + // Greater but existing key found + toOffset = entryOffset(-toEntry - 1); + } + } + + return new SliceIterator(fromOffset, toOffset); + } + + ReferenceBaseEntry get(final MemorySegment key) { + final long entry = entryBinarySearch(key); + if (entry < 0) { + return null; + } + + // Skip key (will reuse the argument) + long offset = entryOffset(entry); + offset += Long.BYTES + key.byteSize(); + // Extract timestamp + long timestamp = data.get(ValueLayout.OfLong.JAVA_LONG_UNALIGNED, offset); + offset += Long.BYTES; + // Extract value length + final long valueLength = getLength(offset); + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new ReferenceBaseEntry<>(key, null, timestamp); + } else { + // Get value + offset += Long.BYTES; + final MemorySegment value = data.asSlice(offset, valueLength); + return new ReferenceBaseEntry<>(key, value, timestamp); + } + } + + private final class SliceIterator implements Iterator> { + private long offset; + private final long toOffset; + + private SliceIterator( + final long offset, + final long toOffset) { + this.offset = offset; + this.toOffset = toOffset; + } + + @Override + public boolean hasNext() { + return offset < toOffset; + } + + @Override + public ReferenceBaseEntry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Read key length + final long keyLength = getLength(offset); + offset += Long.BYTES; + + // Read key + final MemorySegment key = data.asSlice(offset, keyLength); + offset += keyLength; + + // Read timestamp + long timestamp = data.get(ValueLayout.OfLong.JAVA_LONG_UNALIGNED, offset); + offset += Long.BYTES; + + // Read value length + final long valueLength = getLength(offset); + offset += Long.BYTES; + + // Read value + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new ReferenceBaseEntry<>(key, null, timestamp); + } else { + final MemorySegment value = data.asSlice(offset, valueLength); + offset += valueLength; + return new ReferenceBaseEntry<>(key, value, timestamp); + } + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/SSTableWriter.java b/src/main/java/ru/vk/itmo/test/reference/dao2/SSTableWriter.java new file mode 100644 index 000000000..c397e208f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/SSTableWriter.java @@ -0,0 +1,169 @@ +package ru.vk.itmo.test.reference.dao2; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Iterator; + +/** + * Writes {@link ReferenceBaseEntry} {@link Iterator} to SSTable on disk. + * + *

Index file {@code .index} contains {@code long} offsets to entries in data file: + * {@code [offset0, offset1, ...]} + * + *

Data file {@code .data} contains serialized entries: + * {@code } + * + *

Tombstones are encoded as {@code valueLength} {@code -1} and no subsequent value. + * + * @author incubos + */ +final class SSTableWriter { + private static final int BUFFER_SIZE = 64 * 1024; + + // Reusable buffers to eliminate allocations. + // But excessive memory copying is still there :( + // Long cell + private final ByteArraySegment longBuffer = new ByteArraySegment(Long.BYTES); + // Growable blob cell + private final ByteArraySegment blobBuffer = new ByteArraySegment(512); + + void write( + final Path baseDir, + final int sequence, + final Iterator> entries) throws IOException { + // Write to temporary files + final Path tempIndexName = SSTables.tempIndexName(baseDir, sequence); + final Path tempDataName = SSTables.tempDataName(baseDir, sequence); + + // Delete temporary files to eliminate tails + Files.deleteIfExists(tempIndexName); + Files.deleteIfExists(tempDataName); + + // Iterate in a single pass! + // Will write through FileChannel despite extra memory copying and + // no buffering (which may be implemented later). + // Looking forward to MemorySegment facilities in FileChannel! + try (OutputStream index = + new BufferedOutputStream( + new FileOutputStream( + tempIndexName.toFile()), + BUFFER_SIZE); + OutputStream data = + new BufferedOutputStream( + new FileOutputStream( + tempDataName.toFile()), + BUFFER_SIZE)) { + long entryOffset = 0L; + + // Iterate and serialize + while (entries.hasNext()) { + // First write offset to the entry + writeLong(entryOffset, index); + + // Then write the entry + final ReferenceBaseEntry entry = entries.next(); + entryOffset += writeEntry(entry, data); + } + } + + // Publish files atomically + // FIRST index, LAST data + final Path indexName = + SSTables.indexName( + baseDir, + sequence); + Files.move( + tempIndexName, + indexName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + final Path dataName = + SSTables.dataName( + baseDir, + sequence); + Files.move( + tempDataName, + dataName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + + private void writeLong( + final long value, + final OutputStream os) throws IOException { + longBuffer.segment().set( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + 0, + value); + longBuffer.withArray(os::write); + } + + private void writeSegment( + final MemorySegment value, + final OutputStream os) throws IOException { + final long size = value.byteSize(); + blobBuffer.ensureCapacity(size); + MemorySegment.copy( + value, + 0L, + blobBuffer.segment(), + 0L, + size); + blobBuffer.withArray(array -> + os.write( + array, + 0, + (int) size)); + } + + /** + * Writes {@link ReferenceBaseEntry} to {@link FileChannel}. + * + * @return written bytes + */ + private long writeEntry( + final ReferenceBaseEntry entry, + final OutputStream os) throws IOException { + final MemorySegment key = entry.key(); + final MemorySegment value = entry.value(); + final long timestamp = entry.timestamp(); + long result = 0L; + + // Key size + writeLong(key.byteSize(), os); + result += Long.BYTES; + + // Key + writeSegment(key, os); + result += key.byteSize(); + + // timestamp + writeLong(timestamp, os); + result += Long.BYTES; + + // Value size and possibly value + if (value == null) { + // Tombstone + writeLong(SSTables.TOMBSTONE_VALUE_LENGTH, os); + result += Long.BYTES; + } else { + // Value length + writeLong(value.byteSize(), os); + result += Long.BYTES; + + // Value + writeSegment(value, os); + result += value.byteSize(); + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/SSTables.java b/src/main/java/ru/vk/itmo/test/reference/dao2/SSTables.java new file mode 100644 index 000000000..0695833d0 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/SSTables.java @@ -0,0 +1,162 @@ +package ru.vk.itmo.test.reference.dao2; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +/** + * Provides {@link SSTable} management facilities: dumping and discovery. + * + * @author incubos + */ +final class SSTables { + public static final String INDEX_SUFFIX = ".index"; + public static final String DATA_SUFFIX = ".data"; + public static final long TOMBSTONE_VALUE_LENGTH = -1L; + + private static final String TEMP_SUFFIX = ".tmp"; + + /** + * Can't instantiate. + */ + private SSTables() { + // Only static methods + } + + static Path indexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX); + } + + static Path dataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX); + } + + static Path tempIndexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX + TEMP_SUFFIX); + } + + static Path tempDataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX + TEMP_SUFFIX); + } + + /** + * Returns {@link List} of {@link SSTable}s from freshest to oldest. + */ + static List discover( + final Arena arena, + final Path baseDir) throws IOException { + if (!Files.exists(baseDir)) { + return Collections.emptyList(); + } + + final List result = new ArrayList<>(); + try (Stream files = Files.list(baseDir)) { + files.forEach(file -> { + final String fileName = file.getFileName().toString(); + if (!fileName.endsWith(DATA_SUFFIX)) { + // Skip non data + return; + } + + final int sequence = + // .data -> N + Integer.parseInt( + fileName.substring( + 0, + fileName.length() - DATA_SUFFIX.length())); + + try { + result.add(open(arena, baseDir, sequence)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + // Sort from freshest to oldest + result.sort((o1, o2) -> Integer.compare(o2.sequence, o1.sequence)); + + return Collections.unmodifiableList(result); + } + + static SSTable open( + final Arena arena, + final Path baseDir, + final int sequence) throws IOException { + final MemorySegment index = + mapReadOnly( + arena, + indexName(baseDir, sequence)); + final MemorySegment data = + mapReadOnly( + arena, + dataName(baseDir, sequence)); + + return new SSTable( + sequence, + index, + data); + } + + private static MemorySegment mapReadOnly( + final Arena arena, + final Path file) throws IOException { + try (FileChannel channel = + FileChannel.open( + file, + StandardOpenOption.READ)) { + return channel.map( + FileChannel.MapMode.READ_ONLY, + 0L, + Files.size(file), + arena); + } + } + + static void remove( + final Path baseDir, + final int sequence) throws IOException { + // First delete data file to make SSTable invisible + Files.delete(dataName(baseDir, sequence)); + Files.delete(indexName(baseDir, sequence)); + } + + static void promote( + final Path baseDir, + final int from, + final int to) throws IOException { + // Build to progress to the same outcome + if (Files.exists(indexName(baseDir, from))) { + Files.move( + indexName(baseDir, from), + indexName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + if (Files.exists(dataName(baseDir, from))) { + Files.move( + dataName(baseDir, from), + dataName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/TableSet.java b/src/main/java/ru/vk/itmo/test/reference/dao2/TableSet.java new file mode 100644 index 000000000..fe1a0819c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/TableSet.java @@ -0,0 +1,203 @@ +package ru.vk.itmo.test.reference.dao2; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Data set in various tables. + * + * @author incubos + */ +final class TableSet { + final MemTable memTable; + final AtomicLong memTableSize; + // null or read-only + final MemTable flushingTable; + // From freshest to oldest + final List ssTables; + + private TableSet( + final MemTable memTable, + final AtomicLong memTableSize, + final MemTable flushingTable, + final List ssTables) { + this.memTable = memTable; + this.memTableSize = memTableSize; + this.flushingTable = flushingTable; + this.ssTables = ssTables; + } + + static TableSet from(final List ssTables) { + return new TableSet( + new MemTable(), + new AtomicLong(), + null, + ssTables); + } + + int nextSequence() { + return ssTables.stream() + .mapToInt(t -> t.sequence) + .max() + .orElse(0) + 1; + } + + TableSet flushing() { + if (memTable.isEmpty()) { + throw new IllegalStateException("Nothing to flush"); + } + + if (flushingTable != null) { + throw new IllegalStateException("Already flushing"); + } + + return new TableSet( + new MemTable(), + new AtomicLong(), + memTable, + ssTables); + } + + TableSet flushed(final SSTable flushed) { + final List newSSTables = new ArrayList<>(ssTables.size() + 1); + newSSTables.add(flushed); + newSSTables.addAll(ssTables); + return new TableSet( + memTable, + memTableSize, + null, + newSSTables); + } + + TableSet compacted( + final Set replaced, + final SSTable with) { + final List newSsTables = new ArrayList<>(this.ssTables.size() + 1); + + // Keep not replaced SSTables + for (final SSTable ssTable : this.ssTables) { + if (!replaced.contains(ssTable)) { + newSsTables.add(ssTable); + } + } + + // Logically the oldest one + newSsTables.add(with); + + return new TableSet( + memTable, + memTableSize, + flushingTable, + newSsTables); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + final List iterators = + new ArrayList<>(2 + ssTables.size()); + + // MemTable goes first + final Iterator> memTableIterator = + memTable.get(from, to); + if (memTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE, + memTableIterator)); + } + + // Then goes flushing + if (flushingTable != null) { + final Iterator> flushingIterator = + flushingTable.get(from, to); + if (flushingIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE + 1, + flushingIterator)); + } + } + + // Then go all the SSTables + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(from, to); + if (ssTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + } + + return switch (iterators.size()) { + case 0 -> Collections.emptyIterator(); + case 1 -> iterators.get(0); + default -> new MergingEntryIterator(iterators); + }; + } + + ReferenceBaseEntry get(final MemorySegment key) { + // Slightly optimized version not to pollute the heap + + // First check MemTable + ReferenceBaseEntry result = memTable.get(key); + if (result != null) { + // Transform tombstone + return result; + } + + // Then check flushing + if (flushingTable != null) { + result = flushingTable.get(key); + if (result != null) { + // Transform tombstone + return result; + } + } + + // At last check SSTables from freshest to oldest + for (final SSTable ssTable : ssTables) { + result = ssTable.get(key); + if (result != null) { + // Transform tombstone + return result; + } + } + + // Nothing found + return null; + } + +// private static ReferenceBaseEntry swallowTombstone(final ReferenceBaseEntry entry) { +// return entry.value() == null ? null : entry; +// } + + ReferenceBaseEntry upsert(final ReferenceBaseEntry entry) { + return memTable.upsert(entry); + } + + Iterator> allSSTableEntries() { + final List iterators = + new ArrayList<>(ssTables.size()); + + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(null, null); + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + + return new MergingEntryIterator(iterators); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reference/dao2/WeightedPeekingEntryIterator.java b/src/main/java/ru/vk/itmo/test/reference/dao2/WeightedPeekingEntryIterator.java new file mode 100644 index 000000000..b425106b5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reference/dao2/WeightedPeekingEntryIterator.java @@ -0,0 +1,65 @@ +package ru.vk.itmo.test.reference.dao2; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Peeking {@link Iterator} wrapper. + * + * @author incubos + */ +final class WeightedPeekingEntryIterator + implements Iterator>, + Comparable { + private final int weight; + private final Iterator> delegate; + private ReferenceBaseEntry next; + + WeightedPeekingEntryIterator( + final int weight, + final Iterator> delegate) { + this.weight = weight; + this.delegate = delegate; + this.next = delegate.hasNext() ? delegate.next() : null; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public ReferenceBaseEntry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final ReferenceBaseEntry result = next; + next = delegate.hasNext() ? delegate.next() : null; + return result; + } + + ReferenceBaseEntry peek() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + return next; + } + + @Override + public int compareTo(final WeightedPeekingEntryIterator other) { + // First compare keys + int result = + MemorySegmentComparator.INSTANCE.compare( + peek().key(), + other.peek().key()); + if (result != 0) { + return result; + } + + // Then compare weights if keys are equal + return Integer.compare(weight, other.weight); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/HttpServerImpl.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/HttpServerImpl.java index 7b84fb636..524fa9593 100644 --- a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/HttpServerImpl.java +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/HttpServerImpl.java @@ -7,35 +7,61 @@ import one.nio.http.Param; import one.nio.http.Path; import one.nio.http.Request; -import one.nio.http.RequestMethod; import one.nio.http.Response; import one.nio.server.AcceptorConfig; +import one.nio.util.Utf8; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.vk.itmo.ServiceConfig; -import ru.vk.itmo.dao.BaseEntry; -import ru.vk.itmo.dao.Dao; -import ru.vk.itmo.dao.Entry; +import ru.vk.itmo.test.reshetnikovaleksei.dao.BaseEntry; +import ru.vk.itmo.test.reshetnikovaleksei.dao.Dao; +import ru.vk.itmo.test.reshetnikovaleksei.dao.Entry; import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeoutException; public class HttpServerImpl extends HttpServer { private static final Logger LOGGER = LoggerFactory.getLogger(HttpServerImpl.class); + private static final String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; + + public static final String TIMESTAMP_HEADER_NAME = "X-Request-Timestamp"; + public static final String HTTP_TIMESTAMP_HEADER_NAME = "x-request-timestamp"; + public static final String REDIRECTED_REQUEST_HEADER_NAME = "X-Redirected-Request"; private final Dao> dao; private final ExecutorService executorService; + private final RequestRouter requestRouter; + private final String selfUrl; + private final int clusterSize; public HttpServerImpl(ServiceConfig config, Dao> dao, - ExecutorService executorService) throws IOException { + ExecutorService executorService, + RequestRouter requestRouter) throws IOException { super(createConfig(config)); this.dao = dao; this.executorService = executorService; + this.requestRouter = requestRouter; + this.selfUrl = config.selfUrl(); + this.clusterSize = config.clusterUrls().size(); + } + + @Override + public void handleDefault(Request request, HttpSession session) throws IOException { + if (request.getMethod() == Request.METHOD_GET) { + session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); + } else { + session.sendResponse(new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY)); + } } @Override @@ -70,65 +96,117 @@ public void handleRequest(Request request, HttpSession session) throws IOExcepti } @Path("/v0/entity") - @RequestMethod(Request.METHOD_GET) - public Response get(@Param(value = "id", required = true) String id) { - try { - if (id.isEmpty()) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); - } + public Response entity(Request request, + @Param(value = "id") String id, + @Param(value = "from") Integer from, + @Param(value = "ack") Integer ack) { + if (id == null || id.isBlank()) { + return new Response(Response.BAD_REQUEST, Response.EMPTY); + } + + if (from == null) { + from = clusterSize; + } + + if (ack == null) { + ack = (from + 1) / 2; + } - Entry result = dao.get(parseToMemorySegment(id)); - if (result == null) { - return new Response(Response.NOT_FOUND, Response.EMPTY); + if (from < 0 || from > clusterSize || from < ack || ack <= 0) { + return new Response(Response.BAD_REQUEST, Response.EMPTY); + } + + if (request.getHeader(REDIRECTED_REQUEST_HEADER_NAME) == null) { + List not5xxResponses = getNot5xxResponses( + sendRequestsAndGetResponses( + request, id, requestRouter.getNodesByEntityId(id, from) + ) + ); + + if (not5xxResponses.size() >= ack) { + if (request.getMethod() == Request.METHOD_GET) { + not5xxResponses.sort(Comparator.comparingLong(r -> + Long.parseLong(r.getHeader(TIMESTAMP_HEADER_NAME + ": ")))); + return not5xxResponses.getLast(); + } else { + return not5xxResponses.getFirst(); + } } - return Response.ok(result.value().toArray(ValueLayout.JAVA_BYTE)); - } catch (Exception e) { - return new Response(Response.INTERNAL_ERROR, Response.EMPTY); + return new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY); } + + return invokeLocal(request, id); } - @Path("/v0/entity") - @RequestMethod(Request.METHOD_PUT) - public Response put(@Param(value = "id", required = true) String id, Request request) { - try { - if (id.isEmpty() || request.getBody() == null) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); - } + private Response invokeLocal(Request request, String id) { + switch (request.getMethod()) { + case Request.METHOD_GET -> { + MemorySegment key = MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)); + Entry entry = dao.get(key); + if (entry == null || entry.value() == null) { + Response response = new Response(Response.NOT_FOUND, Response.EMPTY); + response.addHeader(TIMESTAMP_HEADER_NAME + ": " + (entry != null ? entry.timestamp() : 0)); + + return response; + } - dao.upsert(new BaseEntry<>(parseToMemorySegment(id), MemorySegment.ofArray(request.getBody()))); - return new Response(Response.CREATED, Response.EMPTY); - } catch (Exception e) { - return new Response(Response.INTERNAL_ERROR, Response.EMPTY); + Response response = Response.ok(entry.value().toArray(ValueLayout.JAVA_BYTE)); + response.addHeader(TIMESTAMP_HEADER_NAME + ": " + entry.timestamp()); + return response; + } + case Request.METHOD_PUT -> { + MemorySegment key = MemorySegment.ofArray(Utf8.toBytes(id)); + MemorySegment value = MemorySegment.ofArray(request.getBody()); + dao.upsert(new BaseEntry<>(key, value, System.currentTimeMillis())); + return new Response(Response.CREATED, Response.EMPTY); + } + case Request.METHOD_DELETE -> { + MemorySegment key = MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)); + dao.upsert(new BaseEntry<>(key, null, System.currentTimeMillis())); + return new Response(Response.ACCEPTED, Response.EMPTY); + } + default -> { + return new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); + } } } - @Path("/v0/entity") - @RequestMethod(Request.METHOD_DELETE) - public Response delete(@Param(value = "id", required = true) String id) { - try { - if (id.isEmpty()) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); + private List sendRequestsAndGetResponses(Request request, String id, List nodes) { + List responses = new ArrayList<>(); + for (String node : nodes) { + if (node.equals(selfUrl)) { + responses.add(invokeLocal(request, id)); + continue; } - dao.upsert(new BaseEntry<>(parseToMemorySegment(id), null)); - return new Response(Response.ACCEPTED, Response.EMPTY); - } catch (Exception e) { - return new Response(Response.INTERNAL_ERROR, Response.EMPTY); + try { + responses.add(requestRouter.redirect(node, request, id)); + } catch (TimeoutException e) { + LOGGER.error("timeout while invoking remote node"); + responses.add(new Response(Response.REQUEST_TIMEOUT, Response.EMPTY)); + } catch (ExecutionException | IOException e) { + LOGGER.error("I/O exception while calling remote node"); + responses.add(new Response(Response.INTERNAL_ERROR, Response.EMPTY)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.error("Thread interrupted"); + responses.add(new Response(Response.SERVICE_UNAVAILABLE, Response.EMPTY)); + } } + + return responses; } - @Override - public void handleDefault(Request request, HttpSession session) throws IOException { - if (request.getMethod() == Request.METHOD_GET) { - session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); - } else { - session.sendResponse(new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY)); + private List getNot5xxResponses(List responses) { + List not5xxResponses = new ArrayList<>(); + for (Response response : responses) { + if (response.getStatus() < 500) { + not5xxResponses.add(response); + } } - } - private MemorySegment parseToMemorySegment(String input) { - return MemorySegment.ofArray(input.getBytes(StandardCharsets.UTF_8)); + return not5xxResponses; } private void processIOException(Request request, HttpSession session, IOException e) { diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/RequestRouter.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/RequestRouter.java new file mode 100644 index 000000000..4f2896442 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/RequestRouter.java @@ -0,0 +1,86 @@ +package ru.vk.itmo.test.reshetnikovaleksei; + +import one.nio.http.Request; +import one.nio.http.Response; +import one.nio.util.Hash; +import ru.vk.itmo.ServiceConfig; + +import java.io.Closeable; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static ru.vk.itmo.test.reshetnikovaleksei.HttpServerImpl.HTTP_TIMESTAMP_HEADER_NAME; +import static ru.vk.itmo.test.reshetnikovaleksei.HttpServerImpl.REDIRECTED_REQUEST_HEADER_NAME; +import static ru.vk.itmo.test.reshetnikovaleksei.HttpServerImpl.TIMESTAMP_HEADER_NAME; + +public class RequestRouter implements Closeable { + private static final String REQUEST_PARAMS = "/v0/entity?id="; + + private final List nodes; + private final HttpClient httpClient; + + public RequestRouter(ServiceConfig config) { + this.nodes = config.clusterUrls(); + this.httpClient = HttpClient.newHttpClient(); + } + + public Response redirect(String node, Request request, String entryId) + throws IOException, InterruptedException, ExecutionException, TimeoutException { + HttpRequest redirectRequest = HttpRequest.newBuilder(URI.create(node + REQUEST_PARAMS + entryId)) + .method(request.getMethodName(), + request.getBody() == null + ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofByteArray(request.getBody())) + .header(REDIRECTED_REQUEST_HEADER_NAME, "true") + .build(); + + HttpResponse httpResponse = httpClient + .sendAsync(redirectRequest, HttpResponse.BodyHandlers.ofByteArray()) + .get(500, TimeUnit.MILLISECONDS); + + Response response = new Response(Integer.toString(httpResponse.statusCode()), httpResponse.body()); + long timestamp = httpResponse.headers().firstValueAsLong(HTTP_TIMESTAMP_HEADER_NAME).orElse(0); + response.addHeader(TIMESTAMP_HEADER_NAME + ": " + timestamp); + + return response; + } + + public List getNodesByEntityId(String id, int from) { + int partition = getNodeIdByEntityId(id); + + List result = new ArrayList<>(); + for (int i = 0; i < from; i++) { + int idx = (partition + i) % nodes.size(); + result.add(nodes.get(idx)); + } + + return result; + } + + private int getNodeIdByEntityId(String id) { + int nodeId = 0; + int maxHash = Hash.murmur3(nodes.getFirst() + id); + for (int i = 1; i < nodes.size(); i++) { + String url = nodes.get(i); + int result = Hash.murmur3(url + id); + if (maxHash < result) { + maxHash = result; + nodeId = i; + } + } + return nodeId; + } + + @Override + public void close() { + httpClient.close(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/ServerRunnerApp.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/ServerRunnerApp.java index 584f1082f..c9e4efaaf 100644 --- a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/ServerRunnerApp.java +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/ServerRunnerApp.java @@ -1,28 +1,57 @@ package ru.vk.itmo.test.reshetnikovaleksei; import ru.vk.itmo.ServiceConfig; -import ru.vk.itmo.dao.Config; -import ru.vk.itmo.dao.Dao; -import ru.vk.itmo.dao.Entry; -import ru.vk.itmo.test.reference.dao.ReferenceDao; import java.io.IOException; -import java.lang.foreign.MemorySegment; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; public final class ServerRunnerApp { + private static final int NODES_COUNT = 3; + private static final String LOCALHOST_PREFIX = "http://localhost:"; + private static final String PATH_PREFIX = "/Users/alreshetnikov/data_%s/"; + private ServerRunnerApp() { } - public static void main(String[] args) throws IOException { - Path path = Path.of("/Users/alreshetnikov/IdeaProjects/2024-highload-dht/data"); + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException, TimeoutException { + Map nodes = generateNodes(); + + List configs = new ArrayList<>(); + for (Map.Entry entry : nodes.entrySet()) { + Path path = Path.of(PATH_PREFIX.formatted(entry.getKey())); + Files.createDirectories(path); + + ServiceConfig serviceConfig = new ServiceConfig( + entry.getKey(), entry.getValue(), new ArrayList<>(nodes.values()), path); + configs.add(serviceConfig); + + } + + for (ServiceConfig config : configs) { + ServiceImpl service = new ServiceImpl(config); + service.start().get(1, TimeUnit.SECONDS); + } + } + + private static Map generateNodes() { + Map result = new HashMap<>(); - ServiceConfig serviceConfig = new ServiceConfig(8080, "http:localhost", List.of("http:localhost"), path); - Dao> dao = new ReferenceDao(new Config(path, 1024 * 1024)); - HttpServerImpl server = new HttpServerImpl(serviceConfig, dao, ExecutorServiceFactory.createExecutorService()); + int nodePort = 8080; + for (int i = 0; i < NODES_COUNT; i++) { + result.put(nodePort, LOCALHOST_PREFIX + nodePort); + nodePort += 10; + } - server.start(); + return result; } } diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/ServiceImpl.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/ServiceImpl.java index 81b326686..6cf685a68 100644 --- a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/ServiceImpl.java +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/ServiceImpl.java @@ -3,10 +3,10 @@ import ru.vk.itmo.Service; import ru.vk.itmo.ServiceConfig; import ru.vk.itmo.dao.Config; -import ru.vk.itmo.dao.Dao; -import ru.vk.itmo.dao.Entry; import ru.vk.itmo.test.ServiceFactory; -import ru.vk.itmo.test.reference.dao.ReferenceDao; +import ru.vk.itmo.test.reshetnikovaleksei.dao.Dao; +import ru.vk.itmo.test.reshetnikovaleksei.dao.Entry; +import ru.vk.itmo.test.reshetnikovaleksei.dao.ReferenceDao; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -23,6 +23,8 @@ public class ServiceImpl implements Service { private Dao> dao; private HttpServerImpl server; private ExecutorService executorService; + private RequestRouter requestRouter; + private boolean stopped; public ServiceImpl(ServiceConfig serviceConfig) { this.serviceConfig = serviceConfig; @@ -30,21 +32,27 @@ public ServiceImpl(ServiceConfig serviceConfig) { } @Override - public CompletableFuture start() throws IOException { + public synchronized CompletableFuture start() throws IOException { dao = new ReferenceDao(daoConfig); executorService = ExecutorServiceFactory.createExecutorService(); - server = new HttpServerImpl(serviceConfig, dao, executorService); + requestRouter = new RequestRouter(serviceConfig); + server = new HttpServerImpl(serviceConfig, dao, executorService, requestRouter); server.start(); + stopped = false; return CompletableFuture.completedFuture(null); } @Override - public CompletableFuture stop() throws IOException { + public synchronized CompletableFuture stop() throws IOException { + if (stopped) { + return CompletableFuture.completedFuture(null); + } + internalStop(); return CompletableFuture.completedFuture(null); } - @ServiceFactory(stage = 2) + @ServiceFactory(stage = 4) public static class Factory implements ServiceFactory.Factory { @Override @@ -59,6 +67,7 @@ private static Config createDaoConfig(ServiceConfig serviceConfig) { private void internalStop() throws IOException { server.stop(); + requestRouter.close(); dao.close(); executorService.shutdown(); @@ -70,5 +79,7 @@ private void internalStop() throws IOException { executorService.shutdownNow(); Thread.currentThread().interrupt(); } + + stopped = true; } } diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/BaseEntry.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/BaseEntry.java new file mode 100644 index 000000000..0fd253000 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/BaseEntry.java @@ -0,0 +1,8 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +public record BaseEntry(D key, D value, long timestamp) implements Entry { + @Override + public String toString() { + return "{" + key + ":" + value + ":" + timestamp + "}"; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/ByteArraySegment.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/ByteArraySegment.java new file mode 100644 index 000000000..b835c6272 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/ByteArraySegment.java @@ -0,0 +1,48 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.nio.ByteBuffer; + +/** + * Growable buffer with {@link ByteBuffer} and {@link MemorySegment} interface. + * + * @author incubos + */ +final class ByteArraySegment { + private byte[] array; + private MemorySegment segment; + + ByteArraySegment(final int capacity) { + this.array = new byte[capacity]; + this.segment = MemorySegment.ofArray(array); + } + + void withArray(final ArrayConsumer consumer) throws IOException { + consumer.process(array); + } + + MemorySegment segment() { + return segment; + } + + void ensureCapacity(final long size) { + if (size > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Too big!"); + } + + final int capacity = (int) size; + if (array.length >= capacity) { + return; + } + + // Grow to the nearest bigger power of 2 + final int newSize = Integer.highestOneBit(capacity) << 1; + array = new byte[newSize]; + segment = MemorySegment.ofArray(array); + } + + interface ArrayConsumer { + void process(byte[] array) throws IOException; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/Dao.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/Dao.java new file mode 100644 index 000000000..ea5aa5835 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/Dao.java @@ -0,0 +1,89 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import ru.vk.itmo.dao.Entry; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; + +public interface Dao> extends Closeable { + /** + * Returns ordered iterator of entries with keys between from (inclusive) and to (exclusive). + * @param from lower bound of range (inclusive) + * @param to upper bound of range (exclusive) + * @return entries [from;to) + */ + Iterator get(D from, D to); + + /** + * Returns entry by key. Note: default implementation is far from optimal. + * @param key entry`s key + * @return entry + */ + default E get(D key) { + Iterator iterator = get(key, null); + if (!iterator.hasNext()) { + return null; + } + + E next = iterator.next(); + if (next.key().equals(key)) { + return next; + } + return null; + } + + /** + * Returns ordered iterator of all entries with keys from (inclusive). + * @param from lower bound of range (inclusive) + * @return entries with key >= from + */ + default Iterator allFrom(D from) { + return get(from, null); + } + + /** + * Returns ordered iterator of all entries with keys < to. + * @param to upper bound of range (exclusive) + * @return entries with key < to + */ + default Iterator allTo(D to) { + return get(null, to); + } + + /** + * Returns ordered iterator of all entries. + * @return all entries + */ + default Iterator all() { + return get(null, null); + } + + /** + * Inserts of replaces entry. + * @param entry element to upsert + */ + void upsert(E entry); + + /** + * Persists data (no-op by default). + */ + default void flush() throws IOException { + // Do nothing + } + + /** + * Compacts data (no-op by default). + */ + default void compact() throws IOException { + // Do nothing + } + + /* + * Releases Dao (calls flush by default). + */ + @Override + default void close() throws IOException { + flush(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/Entry.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/Entry.java new file mode 100644 index 000000000..473de839b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/Entry.java @@ -0,0 +1,5 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +public interface Entry extends ru.vk.itmo.dao.Entry { + long timestamp(); +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/LiveFilteringIterator.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/LiveFilteringIterator.java new file mode 100644 index 000000000..4a46cf497 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/LiveFilteringIterator.java @@ -0,0 +1,50 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Filters non tombstone {@link Entry}s. + * + * @author incubos + */ +final class LiveFilteringIterator implements Iterator> { + private final Iterator> delegate; + private Entry next; + + LiveFilteringIterator(final Iterator> delegate) { + this.delegate = delegate; + skipTombstones(); + } + + private void skipTombstones() { + while (delegate.hasNext()) { + final Entry entry = delegate.next(); + if (entry.value() != null) { + this.next = entry; + break; + } + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Consume + final Entry result = next; + next = null; + + skipTombstones(); + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MemTable.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MemTable.java new file mode 100644 index 000000000..d7434eb21 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MemTable.java @@ -0,0 +1,47 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Memory table. + * + * @author incubos + */ +final class MemTable { + private final NavigableMap> map = + new ConcurrentSkipListMap<>( + MemorySegmentComparator.INSTANCE); + + boolean isEmpty() { + return map.isEmpty(); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + if (from == null && to == null) { + // All + return map.values().iterator(); + } else if (from == null) { + // Head + return map.headMap(to).values().iterator(); + } else if (to == null) { + // Tail + return map.tailMap(from).values().iterator(); + } else { + // Slice + return map.subMap(from, to).values().iterator(); + } + } + + Entry get(final MemorySegment key) { + return map.get(key); + } + + Entry upsert(final Entry entry) { + return map.put(entry.key(), entry); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MemorySegmentComparator.java new file mode 100644 index 000000000..87e4a0d24 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MemorySegmentComparator.java @@ -0,0 +1,89 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +/** + * Compares {@link MemorySegment}s. + * + * @author incubos + */ +final class MemorySegmentComparator implements Comparator { + static final Comparator INSTANCE = + new MemorySegmentComparator(); + + private MemorySegmentComparator() { + // Singleton + } + + @Override + public int compare( + final MemorySegment left, + final MemorySegment right) { + final long mismatch = left.mismatch(right); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == left.byteSize()) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == right.byteSize()) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + left.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch), + right.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch)); + } + + static int compare( + final MemorySegment srcSegment, + final long srcFromOffset, + final long srcLength, + final MemorySegment dstSegment, + final long dstFromOffset, + final long dstLength) { + final long mismatch = + MemorySegment.mismatch( + srcSegment, + srcFromOffset, + srcFromOffset + srcLength, + dstSegment, + dstFromOffset, + dstFromOffset + dstLength); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == srcLength) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == dstLength) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + srcSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + srcFromOffset + mismatch), + dstSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + dstFromOffset + mismatch)); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MergingEntryIterator.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MergingEntryIterator.java new file mode 100644 index 000000000..1965f8403 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/MergingEntryIterator.java @@ -0,0 +1,70 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +/** + * Merges entry {@link Iterator}s. + * + * @author incubos + */ +final class MergingEntryIterator implements Iterator> { + private final Queue iterators; + + MergingEntryIterator(final List iterators) { + assert iterators.stream().allMatch(WeightedPeekingEntryIterator::hasNext); + + this.iterators = new PriorityQueue<>(iterators); + } + + @Override + public boolean hasNext() { + return !iterators.isEmpty(); + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final WeightedPeekingEntryIterator top = iterators.remove(); + final Entry result = top.next(); + + if (top.hasNext()) { + // Not exhausted + iterators.add(top); + } + + // Remove older versions of the key + while (true) { + final WeightedPeekingEntryIterator iterator = iterators.peek(); + if (iterator == null) { + // Nothing left + break; + } + + // Skip entries with the same key + final Entry entry = iterator.peek(); + if (MemorySegmentComparator.INSTANCE.compare(result.key(), entry.key()) != 0) { + // Reached another key + break; + } + + // Drop + iterators.remove(); + // Skip + iterator.next(); + if (iterator.hasNext()) { + // Not exhausted + iterators.add(iterator); + } + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/ReferenceDao.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/ReferenceDao.java new file mode 100644 index 000000000..40e5d8c91 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/ReferenceDao.java @@ -0,0 +1,288 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import ru.vk.itmo.dao.Config; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Reference implementation of {@link Dao}. + * + * @author incubos + */ +public class ReferenceDao implements Dao> { + private final Config config; + private final Arena arena; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + // Guarded by lock + private volatile TableSet tableSet; + + private final ExecutorService flusher = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("flusher"); + return result; + }); + private final ExecutorService compactor = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("compactor"); + return result; + }); + + private final AtomicBoolean closed = new AtomicBoolean(); + + public ReferenceDao(final Config config) throws IOException { + this.config = config; + this.arena = Arena.ofShared(); + + // First complete promotion of compacted SSTables + SSTables.promote( + config.basePath(), + 0, + 1); + + this.tableSet = + TableSet.from( + SSTables.discover( + arena, + config.basePath())); + } + + @Override + public Iterator> get( + final MemorySegment from, + final MemorySegment to) { + return tableSet.get( + from, + to); + } + + @Override + public Entry get(final MemorySegment key) { + // Without lock, just snapshot of table set + return tableSet.get(key); + } + + @Override + public void upsert(final Entry entry) { + final boolean autoFlush; + lock.readLock().lock(); + try { + if (tableSet.memTableSize.get() > config.flushThresholdBytes() + && tableSet.flushingTable != null) { + throw new IllegalStateException("Can't keep up with flushing!"); + } + + // Upsert + final Entry previous = tableSet.upsert(entry); + + // Update size estimate + final long size = tableSet.memTableSize.addAndGet(sizeOf(entry) - sizeOf(previous)); + autoFlush = size > config.flushThresholdBytes(); + } finally { + lock.readLock().unlock(); + } + + if (autoFlush) { + initiateFlush(true); + } + } + + private static long sizeOf(final Entry entry) { + if (entry == null) { + return 0L; + } + + if (entry.value() == null) { + return entry.key().byteSize() + Long.BYTES; + } + + return entry.key().byteSize() + entry.value().byteSize() + Long.BYTES; + } + + private void initiateFlush(final boolean auto) { + flusher.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + if (this.tableSet.memTable.isEmpty()) { + // Nothing to flush + return; + } + + if (auto && this.tableSet.memTableSize.get() < config.flushThresholdBytes()) { + // Not enough data to flush + return; + } + + // Switch memTable to flushing + currentTableSet = this.tableSet.flushing(); + this.tableSet = currentTableSet; + } finally { + lock.writeLock().unlock(); + } + + // Write + final int sequence = currentTableSet.nextSequence(); + try { + new SSTableWriter() + .write( + config.basePath(), + sequence, + currentTableSet.flushingTable.get(null, null)); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-1); + return; + } + + // Open + final SSTable flushed; + try { + flushed = SSTables.open( + arena, + config.basePath(), + sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-2); + return; + } + + // Switch + lock.writeLock().lock(); + try { + this.tableSet = this.tableSet.flushed(flushed); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void flush() throws IOException { + initiateFlush(false); + } + + @Override + public void compact() throws IOException { + compactor.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + currentTableSet = this.tableSet; + if (currentTableSet.ssTables.size() < 2) { + // Nothing to compact + return; + } + } finally { + lock.writeLock().unlock(); + } + + // Compact to 0 + try { + new SSTableWriter() + .write( + config.basePath(), + 0, + currentTableSet.allSSTableEntries()); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-3); + } + + // Open 0 + final SSTable compacted; + try { + compacted = + SSTables.open( + arena, + config.basePath(), + 0); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-4); + return; + } + + // Replace old SSTables with compacted one to + // keep serving requests + final Set replaced = new HashSet<>(currentTableSet.ssTables); + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + replaced, + compacted); + } finally { + lock.writeLock().unlock(); + } + + // Remove compacted SSTables starting from the oldest ones. + // If we crash, 0 contains all the data, and + // it will be promoted on reopen. + for (final SSTable ssTable : currentTableSet.ssTables.reversed()) { + try { + SSTables.remove( + config.basePath(), + ssTable.sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-5); + } + } + + // Promote zero to one (possibly replacing) + try { + SSTables.promote( + config.basePath(), + 0, + 1); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-6); + } + + // Replace promoted SSTable + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + Collections.singleton(compacted), + compacted.withSequence(1)); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void close() throws IOException { + if (closed.getAndSet(true)) { + // Already closed + return; + } + + // Maybe flush + flush(); + + // Stop all the threads + flusher.close(); + compactor.close(); + + // Close arena + arena.close(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTable.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTable.java new file mode 100644 index 000000000..069893fe2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTable.java @@ -0,0 +1,211 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Persistent SSTable in data file and index file. + * + * @author incubos + * @see SSTables + */ +final class SSTable { + final int sequence; + + private final MemorySegment index; + private final MemorySegment data; + private final long size; + + SSTable( + final int sequence, + final MemorySegment index, + final MemorySegment data) { + this.sequence = sequence; + this.index = index; + this.data = data; + this.size = index.byteSize() / Long.BYTES; + } + + SSTable withSequence(final int sequence) { + return new SSTable( + sequence, + index, + data); + } + + /** + * Returns index of the entry if found; otherwise, (-(insertion point) - 1). + * The insertion point is defined as the point at which the key would be inserted: + * the index of the first element greater than the key, + * or size if all keys are less than the specified key. + * Note that this guarantees that the return value will be >= 0 + * if and only if the key is found. + */ + private long entryBinarySearch(final MemorySegment key) { + long low = 0L; + long high = size - 1; + + while (low <= high) { + final long mid = (low + high) >>> 1; + final long midEntryOffset = entryOffset(mid); + final long midKeyLength = getValue(midEntryOffset); + final int compare = + MemorySegmentComparator.compare( + data, + midEntryOffset + Long.BYTES, // Position at key + midKeyLength, + key, + 0L, + key.byteSize()); + + if (compare < 0) { + low = mid + 1; + } else if (compare > 0) { + high = mid - 1; + } else { + return mid; + } + } + + return -(low + 1); + } + + private long entryOffset(final long entry) { + return index.get( + ValueLayout.OfLong.JAVA_LONG, + entry * Long.BYTES); + } + + private long getValue(final long offset) { + return data.get( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + offset); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + assert from == null || to == null || MemorySegmentComparator.INSTANCE.compare(from, to) <= 0; + + // Slice of SSTable in absolute offsets + final long fromOffset; + final long toOffset; + + // Left offset bound + if (from == null) { + // Start from the beginning + fromOffset = 0L; + } else { + final long fromEntry = entryBinarySearch(from); + if (fromEntry >= 0L) { + fromOffset = entryOffset(fromEntry); + } else if (-fromEntry - 1 == size) { + // No relevant data + return Collections.emptyIterator(); + } else { + // Greater but existing key found + fromOffset = entryOffset(-fromEntry - 1); + } + } + + // Right offset bound + if (to == null) { + // Up to the end + toOffset = data.byteSize(); + } else { + final long toEntry = entryBinarySearch(to); + if (toEntry >= 0L) { + toOffset = entryOffset(toEntry); + } else if (-toEntry - 1 == size) { + // Up to the end + toOffset = data.byteSize(); + } else { + // Greater but existing key found + toOffset = entryOffset(-toEntry - 1); + } + } + + return new SliceIterator(fromOffset, toOffset); + } + + Entry get(final MemorySegment key) { + final long entry = entryBinarySearch(key); + if (entry < 0) { + return null; + } + + // Skip key (will reuse the argument) + long offset = entryOffset(entry); + offset += Long.BYTES + key.byteSize(); + // Extract value length + final long valueLength = getValue(offset); + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + offset += Long.BYTES; + final long timestamp = getValue(offset); + return new BaseEntry<>(key, null, timestamp); + } else { + // Get value + offset += Long.BYTES; + final MemorySegment value = data.asSlice(offset, valueLength); + + offset += valueLength; + final long timestamp = getValue(offset); + return new BaseEntry<>(key, value, timestamp); + } + } + + private final class SliceIterator implements Iterator> { + private long offset; + private final long toOffset; + + private SliceIterator( + final long offset, + final long toOffset) { + this.offset = offset; + this.toOffset = toOffset; + } + + @Override + public boolean hasNext() { + return offset < toOffset; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Read key length + final long keyLength = getValue(offset); + offset += Long.BYTES; + + // Read key + final MemorySegment key = data.asSlice(offset, keyLength); + offset += keyLength; + + // Read value length + final long valueLength = getValue(offset); + offset += Long.BYTES; + + // Read value + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + final long timestamp = getValue(offset); + offset += Long.BYTES; + return new BaseEntry<>(key, null, timestamp); + } else { + final MemorySegment value = data.asSlice(offset, valueLength); + offset += valueLength; + + final long timestamp = getValue(offset); + offset += Long.BYTES; + return new BaseEntry<>(key, value, timestamp); + } + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTableWriter.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTableWriter.java new file mode 100644 index 000000000..da926eb06 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTableWriter.java @@ -0,0 +1,171 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Iterator; + +/** + * Writes {@link Entry} {@link Iterator} to SSTable on disk. + * + *

Index file {@code .index} contains {@code long} offsets to entries in data file: + * {@code [offset0, offset1, ...]} + * + *

Data file {@code .data} contains serialized entries: + * {@code } + * + *

Tombstones are encoded as {@code valueLength} {@code -1} and no subsequent value. + * + * @author incubos + */ +final class SSTableWriter { + private static final int BUFFER_SIZE = 64 * 1024; + + // Reusable buffers to eliminate allocations. + // But excessive memory copying is still there :( + // Long cell + private final ByteArraySegment longBuffer = new ByteArraySegment(Long.BYTES); + // Growable blob cell + private final ByteArraySegment blobBuffer = new ByteArraySegment(512); + + void write( + final Path baseDir, + final int sequence, + final Iterator> entries) throws IOException { + // Write to temporary files + final Path tempIndexName = SSTables.tempIndexName(baseDir, sequence); + final Path tempDataName = SSTables.tempDataName(baseDir, sequence); + + // Delete temporary files to eliminate tails + Files.deleteIfExists(tempIndexName); + Files.deleteIfExists(tempDataName); + + // Iterate in a single pass! + // Will write through FileChannel despite extra memory copying and + // no buffering (which may be implemented later). + // Looking forward to MemorySegment facilities in FileChannel! + try (OutputStream index = + new BufferedOutputStream( + new FileOutputStream( + tempIndexName.toFile()), + BUFFER_SIZE); + OutputStream data = + new BufferedOutputStream( + new FileOutputStream( + tempDataName.toFile()), + BUFFER_SIZE)) { + long entryOffset = 0L; + + // Iterate and serialize + while (entries.hasNext()) { + // First write offset to the entry + writeLong(entryOffset, index); + + // Then write the entry + final Entry entry = entries.next(); + entryOffset += writeEntry(entry, data); + } + } + + // Publish files atomically + // FIRST index, LAST data + final Path indexName = + SSTables.indexName( + baseDir, + sequence); + Files.move( + tempIndexName, + indexName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + final Path dataName = + SSTables.dataName( + baseDir, + sequence); + Files.move( + tempDataName, + dataName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + + private void writeLong( + final long value, + final OutputStream os) throws IOException { + longBuffer.segment().set( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + 0, + value); + longBuffer.withArray(os::write); + } + + private void writeSegment( + final MemorySegment value, + final OutputStream os) throws IOException { + final long size = value.byteSize(); + blobBuffer.ensureCapacity(size); + MemorySegment.copy( + value, + 0L, + blobBuffer.segment(), + 0L, + size); + blobBuffer.withArray(array -> + os.write( + array, + 0, + (int) size)); + } + + /** + * Writes {@link Entry} to {@link FileChannel}. + * + * @return written bytes + */ + private long writeEntry( + final Entry entry, + final OutputStream os) throws IOException { + final MemorySegment key = entry.key(); + final MemorySegment value = entry.value(); + final long timestamp = entry.timestamp(); + long result = 0L; + + // Key size + writeLong(key.byteSize(), os); + result += Long.BYTES; + + // Key + writeSegment(key, os); + result += key.byteSize(); + + // Value size and possibly value + if (value == null) { + // Tombstone + writeLong(SSTables.TOMBSTONE_VALUE_LENGTH, os); + result += Long.BYTES; + + writeLong(timestamp, os); + result += Long.BYTES; + } else { + // Value length + writeLong(value.byteSize(), os); + result += Long.BYTES; + + // Value + writeSegment(value, os); + result += value.byteSize(); + + writeLong(timestamp, os); + result += Long.BYTES; + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTables.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTables.java new file mode 100644 index 000000000..216fdc3d9 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/SSTables.java @@ -0,0 +1,162 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +/** + * Provides {@link SSTable} management facilities: dumping and discovery. + * + * @author incubos + */ +final class SSTables { + public static final String INDEX_SUFFIX = ".index"; + public static final String DATA_SUFFIX = ".data"; + public static final long TOMBSTONE_VALUE_LENGTH = -1L; + + private static final String TEMP_SUFFIX = ".tmp"; + + /** + * Can't instantiate. + */ + private SSTables() { + // Only static methods + } + + static Path indexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX); + } + + static Path dataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX); + } + + static Path tempIndexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX + TEMP_SUFFIX); + } + + static Path tempDataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX + TEMP_SUFFIX); + } + + /** + * Returns {@link List} of {@link SSTable}s from freshest to oldest. + */ + static List discover( + final Arena arena, + final Path baseDir) throws IOException { + if (!Files.exists(baseDir)) { + return Collections.emptyList(); + } + + final List result = new ArrayList<>(); + try (Stream files = Files.list(baseDir)) { + files.forEach(file -> { + final String fileName = file.getFileName().toString(); + if (!fileName.endsWith(DATA_SUFFIX)) { + // Skip non data + return; + } + + final int sequence = + // .data -> N + Integer.parseInt( + fileName.substring( + 0, + fileName.length() - DATA_SUFFIX.length())); + + try { + result.add(open(arena, baseDir, sequence)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + // Sort from freshest to oldest + result.sort((o1, o2) -> Integer.compare(o2.sequence, o1.sequence)); + + return Collections.unmodifiableList(result); + } + + static SSTable open( + final Arena arena, + final Path baseDir, + final int sequence) throws IOException { + final MemorySegment index = + mapReadOnly( + arena, + indexName(baseDir, sequence)); + final MemorySegment data = + mapReadOnly( + arena, + dataName(baseDir, sequence)); + + return new SSTable( + sequence, + index, + data); + } + + private static MemorySegment mapReadOnly( + final Arena arena, + final Path file) throws IOException { + try (FileChannel channel = + FileChannel.open( + file, + StandardOpenOption.READ)) { + return channel.map( + FileChannel.MapMode.READ_ONLY, + 0L, + Files.size(file), + arena); + } + } + + static void remove( + final Path baseDir, + final int sequence) throws IOException { + // First delete data file to make SSTable invisible + Files.delete(dataName(baseDir, sequence)); + Files.delete(indexName(baseDir, sequence)); + } + + static void promote( + final Path baseDir, + final int from, + final int to) throws IOException { + // Build to progress to the same outcome + if (Files.exists(indexName(baseDir, from))) { + Files.move( + indexName(baseDir, from), + indexName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + if (Files.exists(dataName(baseDir, from))) { + Files.move( + dataName(baseDir, from), + dataName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/TableSet.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/TableSet.java new file mode 100644 index 000000000..a055da821 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/TableSet.java @@ -0,0 +1,198 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Data set in various tables. + * + * @author incubos + */ +final class TableSet { + final MemTable memTable; + final AtomicLong memTableSize; + // null or read-only + final MemTable flushingTable; + // From freshest to oldest + final List ssTables; + + private TableSet( + final MemTable memTable, + final AtomicLong memTableSize, + final MemTable flushingTable, + final List ssTables) { + this.memTable = memTable; + this.memTableSize = memTableSize; + this.flushingTable = flushingTable; + this.ssTables = ssTables; + } + + static TableSet from(final List ssTables) { + return new TableSet( + new MemTable(), + new AtomicLong(), + null, + ssTables); + } + + int nextSequence() { + return ssTables.stream() + .mapToInt(t -> t.sequence) + .max() + .orElse(0) + 1; + } + + TableSet flushing() { + if (memTable.isEmpty()) { + throw new IllegalStateException("Nothing to flush"); + } + + if (flushingTable != null) { + throw new IllegalStateException("Already flushing"); + } + + return new TableSet( + new MemTable(), + new AtomicLong(), + memTable, + ssTables); + } + + TableSet flushed(final SSTable flushed) { + final List newSSTables = new ArrayList<>(ssTables.size() + 1); + newSSTables.add(flushed); + newSSTables.addAll(ssTables); + return new TableSet( + memTable, + memTableSize, + null, + newSSTables); + } + + TableSet compacted( + final Set replaced, + final SSTable with) { + final List newSsTables = new ArrayList<>(this.ssTables.size() + 1); + + // Keep not replaced SSTables + for (final SSTable ssTable : this.ssTables) { + if (!replaced.contains(ssTable)) { + newSsTables.add(ssTable); + } + } + + // Logically the oldest one + newSsTables.add(with); + + return new TableSet( + memTable, + memTableSize, + flushingTable, + newSsTables); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + final List iterators = + new ArrayList<>(2 + ssTables.size()); + + // MemTable goes first + final Iterator> memTableIterator = + memTable.get(from, to); + if (memTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE, + memTableIterator)); + } + + // Then goes flushing + if (flushingTable != null) { + final Iterator> flushingIterator = + flushingTable.get(from, to); + if (flushingIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE + 1, + flushingIterator)); + } + } + + // Then go all the SSTables + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(from, to); + if (ssTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + } + + return switch (iterators.size()) { + case 0 -> Collections.emptyIterator(); + case 1 -> iterators.get(0); + default -> new MergingEntryIterator(iterators); + }; + } + + Entry get(final MemorySegment key) { + // Slightly optimized version not to pollute the heap + + // First check MemTable + Entry result = memTable.get(key); + if (result != null) { + // Transform tombstone + return result; + } + + // Then check flushing + if (flushingTable != null) { + result = flushingTable.get(key); + if (result != null) { + // Transform tombstone + return result; + } + } + + // At last check SSTables from freshest to oldest + for (final SSTable ssTable : ssTables) { + result = ssTable.get(key); + if (result != null) { + return result; + } + } + + // Nothing found + return null; + } + + Entry upsert(final Entry entry) { + return memTable.upsert(entry); + } + + Iterator> allSSTableEntries() { + final List iterators = + new ArrayList<>(ssTables.size()); + + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(null, null); + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + + return new MergingEntryIterator(iterators); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/WeightedPeekingEntryIterator.java b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/WeightedPeekingEntryIterator.java new file mode 100644 index 000000000..70550939e --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/dao/WeightedPeekingEntryIterator.java @@ -0,0 +1,65 @@ +package ru.vk.itmo.test.reshetnikovaleksei.dao; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Peeking {@link Iterator} wrapper. + * + * @author incubos + */ +final class WeightedPeekingEntryIterator + implements Iterator>, + Comparable { + private final int weight; + private final Iterator> delegate; + private Entry next; + + WeightedPeekingEntryIterator( + final int weight, + final Iterator> delegate) { + this.weight = weight; + this.delegate = delegate; + this.next = delegate.hasNext() ? delegate.next() : null; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final Entry result = next; + next = delegate.hasNext() ? delegate.next() : null; + return result; + } + + Entry peek() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + return next; + } + + @Override + public int compareTo(final WeightedPeekingEntryIterator other) { + // First compare keys + int result = + MemorySegmentComparator.INSTANCE.compare( + peek().key(), + other.peek().key()); + if (result != 0) { + return result; + } + + // Then compare weights if keys are equal + return Integer.compare(weight, other.weight); + } +} diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/get_alloc.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/get_alloc.html new file mode 100644 index 000000000..7599d67bb --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/get_alloc.html @@ -0,0 +1,3193 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/get_cpu.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/get_cpu.html new file mode 100644 index 000000000..5c37b5349 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/get_cpu.html @@ -0,0 +1,3189 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/put_alloc.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/put_alloc.html new file mode 100644 index 000000000..459ddb4f0 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/put_alloc.html @@ -0,0 +1,3147 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/put_cpu.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/put_cpu.html new file mode 100644 index 000000000..cb30e8cab --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/put_cpu.html @@ -0,0 +1,4189 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/stage3.md b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/stage3.md new file mode 100644 index 000000000..f7b457842 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage3/stage3.md @@ -0,0 +1,57 @@ +# Stage 3 + +## PUT TEST + +```shell +./wrk -d 30 -t 8 -c 64 -R 70000 -L -s src/main/java/ru/vk/itmo/test/reshetnikovaleksei/lua/put.lua http://localhost:8080 + + Latency | было | стало + 50.000% | 2.01ms | 11.99s + 75.000% | 149.12ms | 15.50s + 90.000% | 283.90ms | 24.87s + 99.000% | 365.05ms | 25.05s + 99.900% | 371.97ms | 25.10s + 99.990% | 374.78ms | 26.08s + 99.999% | 376.58ms | 26.10s +100.000% | 377.09ms | 26.10s +``` + +[Allocation profile](put_alloc.html) +[CPU profile](put_cpu.html) + +В целом доли такие же, как и в предыдущем этапе, за исключением появления походов через `HttpClient`, которые занимает значительную +долю ресурсов в рамках аллокаций и cpu + +- 46.32% аллокаций занимает `HttpClient` (перенаправление запроса), 2% аллокаций занимает обратботка запроса в текущем инстансе +- 12.92% cpu занимает `HtppClient`, 0.67% cpu занимает обработать запрос в текущем инстансе + +## GET TEST + +```shell +./wrk -d 30 -t 8 -c 64 -R 75000 -L -s src/main/java/ru/vk/itmo/test/reshetnikovaleksei/lua/get.lua http://localhost:8080 + + Latency | было | стало + 50.000% | 1.64ms | 16.72s + 75.000% | 2.57ms | 17.71s + 90.000% | 14.61ms | 22.00s + 99.000% | 142.21ms | 25.71s + 99.900% | 157.95ms | 25.76s + 99.990% | 166.65ms | 27.25s + 99.999% | 169.47ms | 27.26s +100.000% | 170.11ms | 27.28s +``` + +[Allocation profile](get_alloc.html) +[CPU profile](get_cpu.html) + +Для `GET` операций картина аналогичная с `PUT` операциями. Посмотрим более подробно на числа: + +- 53.32% аллокаций занимает `HttpClient` (перенаправление запроса), 1.93% аллокаций занимает обратботка запроса в текущем инстансе +- 15.11% cpu занимает `HtppClient`, 0.94% cpu занимает обработать запрос в текущем инстансе + +## Выводы + +В сравнении со вторым этапем производительность сильно снизилась. Как видно из профилей, сетевое взаимодействие между нодами +занимает значительное количество ресурсов, из-за чего уменьшается пропускная способность и увеличивается нагруженность сервера. +Относительно затрат на сетевое взаимодействие, оптимизации бизнес логики приложения не будут давать какого-либо +значительного результата diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_alloc.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_alloc.html new file mode 100644 index 000000000..c80ab8b64 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_alloc.html @@ -0,0 +1,2975 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_cpu.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_cpu.html new file mode 100644 index 000000000..e5f196e78 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_cpu.html @@ -0,0 +1,2719 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_lock.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_lock.html new file mode 100644 index 000000000..aeec4abbc --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/get_lock.html @@ -0,0 +1,839 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_alloc.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_alloc.html new file mode 100644 index 000000000..4a99a964f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_alloc.html @@ -0,0 +1,3026 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_cpu.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_cpu.html new file mode 100644 index 000000000..e76386669 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_cpu.html @@ -0,0 +1,3584 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_lock.html b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_lock.html new file mode 100644 index 000000000..27728ace2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/put_lock.html @@ -0,0 +1,785 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/stage4.md b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/stage4.md new file mode 100644 index 000000000..a615717ae --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/reshetnikovaleksei/reports/stage4/stage4.md @@ -0,0 +1,89 @@ +# Stage 4 + +from=3, ack=2 (по дефолту) + +запускаю 3 инстанса в разных потоках + +## PUT TEST + +#### Сравнение со stage 3 + +```shell +./wrk -d 30 -t 4 -c 32 -R 15000 -L -s src/main/java/ru/vk/itmo/test/reshetnikovaleksei/lua/put.lua http://localhost:8080 + + Latency | стало | было + 50.000% | 6.29s | 1.37ms + 75.000% | 7.17s | 1.96ms + 90.000% | 7.56s | 3.83ms + 99.000% | 7.82s | 464.13ms + 99.900% | 7.86s | 503.04ms + 99.990% | 7.88s | 505.86ms + 99.999% | 7.89s | 507.65ms +100.000% | 7.89s | 508.16ms +``` + +В среднем производительность уменьшилась примерно в 15 раз, что в целом очевидно, так как запрос теперь рассылается +по нескольким нодам, и теперь нужно ждать ответа от всех нод: проксирование происходит сразу по всем узлам. + +#### Профилирование + +[Allocation profile](put_alloc.html) + +- 89% аллокаций на `ThreadPoolExecutor.runWorker` - отправка и ожидание запросов от соседних нод происходят здесь + - 19.73% аллокаций уходят на метод `sendRequestAndGetResponses`, который отправляет запросы в другие ноды и ждет ответы + - 0.74% аллокаций уходят на `invokeLocal` - локальная обработка запроса + - 1.45% аллокаций на отправку ответа + - остальное из `ThreadPoolExecutor.runWorker` - обработка ответов из соседних нод +- 4% аллокаций на `SelectorManager.run` +- 6.89% на `SelectorThread.run` + +[CPU profile](put_cpu.html) + +- 65.29% спу на `ThreadPoolExecutor.runWorker` + - 10.67% спу на `sendRequestAndGetResponses` + - 0.37% спу на `invokeLocal` + - 3.87% спу на отправку ответа + - остальное из `ThreadPoolExecutor.runWorker` - обработка ответов из соседних нод +- 6.98% спу на `ForkJoinWorkerThread.run` +- 7.6% аллокаций на `SelectorManager.run` +- 14.43% на `SelectorThread.run` + +[Lock profile](put_lock.html) + +- 5.61% локов на `SelectorManager.run` +- 94.23% локов на `ThreadPoolExecutor.runWorker` + +## GET TEST + +#### Сравнение с stage3 + +```shell +./wrk -d 30 -t 4 -c 32 -R 15000 -L -s src/main/java/ru/vk/itmo/test/reshetnikovaleksei/lua/get.lua http://localhost:8080 + + Latency | стало | было + 50.000% | 436.99ms | 1.00ms + 75.000% | 640.00ms | 1.36ms + 90.000% | 729.09ms | 1.64ms + 99.000% | 997.38ms | 2.12ms + 99.900% | 1.02s | 2.45ms + 99.990% | 1.03s | 2.79ms + 99.999% | 1.04s | 3.25ms +100.000% | 1.04s | 3.51ms +``` + +Аналогично с `PUT` наблюдается сильная деградация производительности по той же причине + +#### Профилирование + +[Allocation profile](get_alloc.html) + +[CPU profile](get_cpu.html) + +[Lock profile](get_lock.html) + +По числам при `GET` примерно такая же картина + +## Вывод + +Репликация очень сильно деградировала производительность, но за это сервис стал более надежным за счет того, что у нас +теперь есть возможность указывать минимальное кол-во нод, которые должны отправить положительный ответ diff --git a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/reports/hw5/report.md b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/reports/hw5/report.md new file mode 100644 index 000000000..8e257aea0 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/reports/hw5/report.md @@ -0,0 +1,73 @@ +# Отчет HW4 + +Тестирование wrk проводилось в 4 потока для 16 соединений с `ack=2&from=3` + +## PUT + +Для `put` запросов точкой разладки было 16к rps (база изначально была пустой) + +| percentage | latency | +|-------------| ----| +| 50.000% | 1.46ms | +| 75.000% | 1.99ms | +| 90.000% | 2.72ms | +| 99.000% | 4.49ms | +| 99.900% | 6.49ms | +| 99.990% | 7.92ms | +| 99.999% | 8.66ms | +| 100.000% | 8.86ms | + +[график](https://disk.yandex.ru/i/Ayg7t1u3g8AoKg) + +[профилирование cpu](https://disk.yandex.ru/d/KCmrwW5bOi9-gg) + +[профилирование alloc](https://disk.yandex.ru/d/3YvQmU6zERjAOQ) + +[профилирование lock](https://disk.yandex.ru/d/rNXn4J7LL4kZVw) + +## GET + +При `get` запросах точкой разладки оказалось 17k rpc (база была заполнена на 800 мегабайт на ноду, суммарно 960 тысяч значений) + +| percentage | latency | +| ---------- | ------- | +| 50.000% | 1.47ms | +| 75.000% | 2.03ms | +| 90.000% | 2.67ms | +| 99.000% | 4.13ms | +| 99.900% | 6.29ms | +| 99.990% | 14.29ms | +| 99.999% | 17.30ms | +| 100.000% | 17.66ms | + +[график](https://disk.yandex.ru/i/oLhmEo2s_s74-g) + +[профилирование cpu](https://disk.yandex.ru/d/qGeFGb321YH2RQ) + +[профилирование alloc](https://disk.yandex.ru/d/fRJGPqU8zZldVA) + +[профилирование lock](https://disk.yandex.ru/d/-2dAsxyVSQVMeQ) + + +Что изменилось? Если посмотреть на предыдущее профилирование [put](https://disk.yandex.ru/d/Evdm1hsgjZziwA), то можно заметить много отличий. + +Исполнение метода, который исполняет логику самой базы занимало 0,89% всего cpu, а теперь это значение 0,4% +При этом само задание воркера исполнялось 35%, а стало исполняться 4%. + +При этом суммарно вызовы park на потоках около 35% процента, когда как до этого было около 15 процентов. + +Где-то проскочили вызовы `ForkJoinPool`, поэтому, в том числе, ожидание зависело и от этого пула потоков. + +Если сравнивать [предыдущий alloc](https://disk.yandex.ru/d/gE_920JbDpDPHQ), то если раньше редиректы занимали почти все аллокации на put (60 процентов всех обращений к TLAB), то теперь они занимают менее 20 процентов. + +Если посмотреть почти все аллокации касаются записи и чтения пересланных запросов. Весь этот код выполняется внутри `CopletableFuture` и `HttpClient` + +С get c аллокациями ситуация такая же с точностью до процентов. + + +## sync + +Написал синхронную версию. Все забросы отправляются в неблокирующем режиме, но поток worker ждет пока ответят все. +Также использовалась `CountDownLatch`. + +В итоге получилась [картинка](https://disk.yandex.ru/i/pzXkxq6ePzFlAA) (Запускалось в тех же условиях, но в другое время, поэтому абсолютные числа отличаются от верхних) \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/DaoHttpServer.java b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/DaoHttpServer.java index 2584d5428..18c5d6b9f 100644 --- a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/DaoHttpServer.java +++ b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/DaoHttpServer.java @@ -1,6 +1,5 @@ package ru.vk.itmo.test.smirnovdmitrii.server; -import one.nio.http.HttpException; import one.nio.http.HttpServer; import one.nio.http.HttpSession; import one.nio.http.Request; @@ -8,34 +7,48 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.vk.itmo.dao.Dao; +import ru.vk.itmo.test.smirnovdmitrii.application.properties.DhtValue; import ru.vk.itmo.test.smirnovdmitrii.dao.TimeEntry; import java.io.IOException; import java.io.UncheckedIOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; +import java.net.HttpURLConnection; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.TimeUnit; +import static java.net.HttpURLConnection.HTTP_ACCEPTED; +import static java.net.HttpURLConnection.HTTP_BAD_METHOD; +import static java.net.HttpURLConnection.HTTP_CREATED; +import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static java.net.HttpURLConnection.HTTP_OK; import static one.nio.http.Request.METHOD_DELETE; import static one.nio.http.Request.METHOD_GET; import static one.nio.http.Request.METHOD_PUT; public class DaoHttpServer extends HttpServer { + + @DhtValue("server.use.async:true") + private static boolean useAsync; + @DhtValue("server.redirect.pool.size:30") + private static int REDIRECT_POOL_SIZE; + private static final String REQUEST_PATH = "/v0/entity"; private static final String SERVER_STOP_PATH = "/stop"; private static final byte[] INVALID_KEY_MESSAGE = "invalid id".getBytes(StandardCharsets.UTF_8); - private static final String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; private static final Logger logger = LoggerFactory.getLogger(DaoHttpServer.class); - private static final String TIMESTAMP_HEADER_NAME = "X-KEY-TS: "; private final ExecutorService workerPool; + private final ExecutorService redirectPool; private final Dao> dao; private final Balancer balancer; private final RedirectService redirectService; @@ -49,7 +62,8 @@ public DaoHttpServer( super(config); this.dao = dao; this.balancer = new Balancer(config.clusterUrls); - this.redirectService = new RedirectService(config.selfUrl, config.clusterUrls); + this.redirectPool = Executors.newFixedThreadPool(REDIRECT_POOL_SIZE); + this.redirectService = new RedirectService(redirectPool); this.selfUrl = config.selfUrl; if (config.useWorkers && config.useVirtualThreads) { this.workerPool = Executors.newVirtualThreadPerTaskExecutor(); @@ -73,34 +87,34 @@ public DaoHttpServer( this.useWorkers = config.useWorkers; } - public Response get(final MemorySegment key) { + public ProcessResult get(final MemorySegment key) { final TimeEntry entry = dao.get(key); if (entry == null) { - return new Response(Response.NOT_FOUND, Response.EMPTY); + return new ProcessResult(HTTP_NOT_FOUND, Response.EMPTY); } else if (entry.value() == null) { - final Response response = new Response(Response.NOT_FOUND, Response.EMPTY); - response.addHeader(TIMESTAMP_HEADER_NAME + entry.millis()); - return response; + return new ProcessResult(HTTP_NOT_FOUND, Response.EMPTY, epochMillisNow()); } - final Response response = Response.ok(entry.value().toArray(ValueLayout.JAVA_BYTE)); - response.addHeader(TIMESTAMP_HEADER_NAME + entry.millis()); - return response; + return new ProcessResult( + HTTP_OK, + entry.value().toArray(ValueLayout.JAVA_BYTE), + epochMillisNow() + ); } - public Response put( + public ProcessResult put( final MemorySegment key, final Request request ) { final MemorySegment value = MemorySegment.ofArray(request.getBody()); final TimeEntry entry = new TimeEntry<>(epochMillisNow(), key, value); dao.upsert(entry); - return new Response(Response.CREATED, Response.EMPTY); + return new ProcessResult(HTTP_CREATED, Response.EMPTY); } - public Response delete(final MemorySegment key) { + public ProcessResult delete(final MemorySegment key) { final TimeEntry entry = new TimeEntry<>(epochMillisNow(), key, null); dao.upsert(entry); - return new Response(Response.ACCEPTED, Response.EMPTY); + return new ProcessResult(HTTP_ACCEPTED, Response.EMPTY); } @Override @@ -128,184 +142,154 @@ private void handleRequestTask(final Request request, final HttpSession session) stop(); return; } - final int method = request.getMethod(); final String id = request.getParameter("id="); - final Response validationFailedResponse = validateResponse(path, id, method); - if (validationFailedResponse != null) { - session.sendResponse(validationFailedResponse); + final ProcessResultHandler handler = processHandler(session, request, id, path); + if (handler.isClosed()) { return; } - final String redirectHeader = request.getHeader(RedirectService.REDIRECT_HEADER_NAME); + final String redirectHeader = request.getHeader(Utils.REDIRECT_ONE_NIO_HEADER_NAME); if (redirectHeader != null && redirectHeader.equals("true")) { - final byte[] keyBytes = id.getBytes(StandardCharsets.UTF_8); - final MemorySegment key = MemorySegment.ofArray(keyBytes); - session.sendResponse(processRequest(request, method, key)); + handler.sendResult(processRequest(request, handler.method(), id)); return; } - processRequestAck(request, session, id, method); + processRequestAck(request, id, handler); } catch (final IOException e) { logger.error("IOException in send response."); + } catch (final InterruptedException e) { + logger.error("Interrupted while processing tasks."); + Thread.currentThread().interrupt(); } } private void processRequestAck( final Request request, - final HttpSession session, final String id, - final int method - ) throws IOException { - final String ackParam = request.getParameter("ack="); - final String fromParam = request.getParameter("from="); - final int from; - if (fromParam == null) { - from = balancer.clusterSize(); + final ProcessResultHandler handler + ) throws IOException, InterruptedException { + final String[] nodeUrls = balancer.getNodeUrls(id, handler.from()); + if (useAsync) { + processRequestAckAsync(request, id, handler, nodeUrls); } else { - try { - from = Integer.parseInt(fromParam); - } catch (final NumberFormatException e) { - session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); - return; - } + processRequestAckSync(request, id, handler, nodeUrls); } - final int ack; - if (ackParam == null) { - ack = balancer.quorum(from); - } else { - try { - ack = Integer.parseInt(ackParam); - } catch (final NumberFormatException e) { - session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); - return; - } - } - if (ack < 1 || from < 1 || ack > from) { - session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); - return; - } - final String[] nodeUrls = balancer.getNodeUrls(id, from); - final Response response; - if (method == METHOD_GET) { - response = processGetRequest(request, nodeUrls, id, ack); - } else { - response = processNotGetRequest(request, nodeUrls, id, ack); - } - session.sendResponse(response); } - private Response processNotGetRequest( + private void processRequestAckSync( final Request request, - final String[] nodeUrls, final String id, - final int ack - ) { - int success = 0; - Response response = null; - for (final String url : nodeUrls) { - final Response curResponse = processRequestWithUrl(request, url, id); - final int status = curResponse.getStatus(); - if (200 <= status && status < 300) { - success++; - response = curResponse; + final ProcessResultHandler handler, + final String[] nodeUrls + ) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(nodeUrls.length); + for (final String url: nodeUrls) { + if (url.equals(selfUrl)) { + redirectPool.execute(() -> { + try { + handler.add(processRequest(request, handler.method(), id)); + } finally { + latch.countDown(); + } + }); + } else { + redirectPool.execute(() -> { + try { + redirectService.redirectSync(url, request, handler); + } catch (final Exception e) { + handler.add(new ProcessResult(HttpURLConnection.HTTP_GATEWAY_TIMEOUT, Response.EMPTY)); + } finally { + latch.countDown(); + } + }); } } - if (success >= ack) { - return response; + latch.await(); + if (handler.isOutcomeSuccess()) { + handler.sendResult(handler.outcome.get()); } else { - return new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY); + handler.sendNotEnoughReplicas(); } } - private Response processGetRequest( + private void processRequestAckAsync( final Request request, - final String[] nodeUrls, final String id, - final int ack + final ProcessResultHandler handler, + final String[] nodeUrls ) { - int success = 0; - Response response = null; - long responseTs = -1; + boolean isLocal = false; for (final String url : nodeUrls) { - final Response curResponse = processRequestWithUrl(request, url, id); - final int status = curResponse.getStatus(); - if (200 <= status && status < 500) { - success++; - final String headerTs = curResponse.getHeader(TIMESTAMP_HEADER_NAME); - final long responseMillis = headerTs == null ? 0 : Long.parseLong(headerTs); - if (responseTs < responseMillis) { - response = curResponse; - responseTs = responseMillis; - } + if (url.equals(selfUrl)) { + isLocal = true; + } else { + redirectService.redirectAsync(url, request, handler); } } - if (success >= ack) { - return response; - } else { - return new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY); - } - } - - private Response processRequestWithUrl( - final Request request, - final String url, - final String id - ) { - final Response response; - if (url.equals(selfUrl)) { - final byte[] keyBytes = id.getBytes(StandardCharsets.UTF_8); - final MemorySegment key = MemorySegment.ofArray(keyBytes); - response = processRequest(request, request.getMethod(), key); - } else { - response = shardResponse(request, url); - } - return response; - } - - private Response shardResponse( - final Request request, - final String nodeUrl - ) { - try { - return redirectService.redirect(nodeUrl, request); - } catch (final IOException e) { - logger.error("IOException in get response from node {}", nodeUrl, e); - } catch (final HttpException e) { - logger.error("HttpException in sending to node {}", nodeUrl, e); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - logger.error("Timeout in sending to node {}", nodeUrl, e); + if (isLocal) { + handler.add(processRequest(request, handler.method(), id)); } - return new Response(Response.BAD_GATEWAY, Response.EMPTY); } - private Response processRequest(final Request request, final int method, final MemorySegment key) { - Response response; + private ProcessResult processRequest(final Request request, final int method, final String id) { + final MemorySegment key = MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)); + ProcessResult response; try { response = switch (method) { case METHOD_GET -> get(key); case METHOD_DELETE -> delete(key); case METHOD_PUT -> put(key, request); - default -> new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); + default -> new ProcessResult(HTTP_BAD_METHOD, Response.EMPTY, -1); }; } catch (final Exception e) { logger.error("Exception while handling request", e); - response = new Response(Response.INTERNAL_ERROR, Response.EMPTY); + response = new ProcessResult(HTTP_INTERNAL_ERROR, Response.EMPTY, -1); } return response; } - public Response validateResponse( - final String path, + public ProcessResultHandler processHandler( + final HttpSession session, + final Request request, final String id, - final int method + final String path ) { - if (!path.equals(REQUEST_PATH)) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); + final int method = request.getMethod(); + final String ackParam = request.getParameter("ack="); + final String fromParam = request.getParameter("from="); + final int from; + if (fromParam == null) { + from = balancer.clusterSize(); + } else { + try { + from = Integer.parseInt(fromParam); + } catch (final NumberFormatException e) { + final ProcessResultHandler execHandler = new ProcessResultHandler(session, method, -1, -1); + execHandler.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); + return execHandler; + } + } + final int ack; + if (ackParam == null) { + ack = balancer.quorum(from); + } else { + try { + ack = Integer.parseInt(ackParam); + } catch (final NumberFormatException e) { + final ProcessResultHandler execHandler = new ProcessResultHandler(session, method, -1, -1); + execHandler.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); + return execHandler; + } + } + final ProcessResultHandler handler = new ProcessResultHandler(session, method, ack, from); + if (ack < 1 || from < 1 || ack > from) { + handler.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); + } else if (!path.equals(REQUEST_PATH)) { + handler.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); } else if (isInvalidKey(id)) { - return new Response(Response.BAD_REQUEST, INVALID_KEY_MESSAGE); + handler.sendResponse(new Response(Response.BAD_REQUEST, INVALID_KEY_MESSAGE)); } else if (METHOD_GET != method && METHOD_DELETE != method && METHOD_PUT != method) { - return new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); + handler.sendResponse(new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY)); } - return null; + return handler; } public boolean isInvalidKey(final String key) { @@ -324,6 +308,9 @@ public synchronized void stop() { if (workerPool != null) { gracefulShutdown(workerPool); } + if (redirectPool != null) { + gracefulShutdown(redirectPool); + } try { dao.close(); } catch (final IOException e) { diff --git a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ProcessResult.java b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ProcessResult.java new file mode 100644 index 000000000..1940cf9ad --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ProcessResult.java @@ -0,0 +1,29 @@ +package ru.vk.itmo.test.smirnovdmitrii.server; + +public class ProcessResult { + private final int status; + private final byte[] data; + private final long timestamp; + + public ProcessResult(int status, byte[] data, long timestamp) { + this.status = status; + this.data = data; + this.timestamp = timestamp; + } + + public ProcessResult(int status, byte[] data) { + this(status, data, -1); + } + + public int status() { + return status; + } + + public byte[] data() { + return data; + } + + public long timestamp() { + return timestamp; + } +} diff --git a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ProcessResultHandler.java b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ProcessResultHandler.java new file mode 100644 index 000000000..eaeb2d0de --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ProcessResultHandler.java @@ -0,0 +1,148 @@ +package ru.vk.itmo.test.smirnovdmitrii.server; + +import one.nio.http.HttpSession; +import one.nio.http.Request; +import one.nio.http.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ru.vk.itmo.test.smirnovdmitrii.application.properties.DhtValue; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class ProcessResultHandler { + + @DhtValue("server.use.async:true") + private static boolean useAsync; + private static final Logger logger = LoggerFactory.getLogger(ProcessResultHandler.class); + private static final String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; + private final AtomicInteger successes = new AtomicInteger(0); + private final AtomicInteger fails = new AtomicInteger(0); + private final AtomicBoolean isClosed = new AtomicBoolean(false); + public final AtomicReference outcome = new AtomicReference<>(null); + private final HttpSession session; + private final int method; + private final int ack; + private final int from; + + public ProcessResultHandler( + final HttpSession session, + final int method, + final int ack, + final int from + ) { + this.session = session; + this.method = method; + this.ack = ack; + this.from = from; + } + + public ProcessResultHandler( + final HttpSession session, + final int method + ) { + this(session, method, 1, 1); + } + + public boolean isOutcomeSuccess() { + return successes.get() >= ack; + } + + public int method() { + return method; + } + + public int ack() { + return ack; + } + + public int from() { + return from; + } + + public boolean isClosed() { + return isClosed.get(); + } + + public void add(final ProcessResult result) { + add(result, null); + } + + public void add(final ProcessResult result, final Throwable throwable) { + if (throwable != null) { + logger.error("Exception while processing ack's", throwable); + } + if (isClosed.get()) { + return; + } + + Response response; + if (throwable == null && success(result)) { + if (successes.incrementAndGet() >= ack) { + response = new Response(String.valueOf(result.status()), result.data()); + } else { + return; + } + } else { + if (fails.incrementAndGet() > from - ack) { + response = new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY); + } else { + return; + } + } + if (useAsync) { + sendResponse(response); + } + } + + private boolean success(final ProcessResult result) { + final int status = result.status(); + if (this.method == Request.METHOD_GET) { + final boolean isSuccess = status == HttpURLConnection.HTTP_OK || status == HttpURLConnection.HTTP_NOT_FOUND; + while (true) { + final ProcessResult curResult = outcome.get(); + if (curResult != null && curResult.timestamp() > result.timestamp()) { + break; + } + if (outcome.compareAndSet(curResult, result)) { + break; + } + } + return isSuccess; + } else { + if (status == HttpURLConnection.HTTP_CREATED || status == HttpURLConnection.HTTP_ACCEPTED) { + outcome.set(result); + return true; + } + return false; + } + } + + public void sendResponse(final Response response) { + if (isClosed.compareAndSet(false, true)) { + try { + session.sendResponse(response); + } catch (final IOException e) { + logger.error("Error while sending response", e); + try { + session.sendResponse(new Response(Response.INTERNAL_ERROR, Response.EMPTY)); + } catch (final IOException ignored) { + // do nothing + } + } + } + } + + public void sendNotEnoughReplicas() { + sendResponse(new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY)); + } + + public void sendResult(final ProcessResult result) { + final Response response = new Response(String.valueOf(result.status()), result.data()); + response.addHeader(Utils.TIMESTAMP_HEADER_NAME + ":" + result.timestamp()); + sendResponse(response); + } +} diff --git a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/RedirectService.java b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/RedirectService.java index e82368b05..9d867b373 100644 --- a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/RedirectService.java +++ b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/RedirectService.java @@ -1,67 +1,88 @@ package ru.vk.itmo.test.smirnovdmitrii.server; -import one.nio.http.HttpClient; -import one.nio.http.HttpException; import one.nio.http.Request; -import one.nio.http.Response; -import one.nio.net.ConnectionString; -import one.nio.pool.PoolException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.vk.itmo.test.smirnovdmitrii.application.properties.DhtValue; import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.OptionalLong; +import java.util.concurrent.Executor; public class RedirectService { @DhtValue("cluster.redirect.timeout:100") private static int REDIRECT_TIMEOUT; - private final Map clients = new HashMap<>(); + @DhtValue("server.connection.timeout.millis:100") + private static int CONNECTION_TIMEOUT; + private final HttpClient client; private static final Logger logger = LoggerFactory.getLogger(RedirectService.class); - public static final String REDIRECT_HEADER_NAME = "X-REPLICATION-REQUEST:"; - public static final String REDIRECT_HEADER = REDIRECT_HEADER_NAME + " true"; - public RedirectService(final String selfUrl, List clusterUrls) { - for (final String clusterUrl: clusterUrls) { - if (clusterUrl.equals(selfUrl)) { - continue; - } - clients.put(clusterUrl, new HttpClient(new ConnectionString(clusterUrl))); + public RedirectService(final Executor asyncExecutor) { + this.client = HttpClient.newBuilder() + .executor(asyncExecutor) + .connectTimeout(Duration.ofMillis(CONNECTION_TIMEOUT)) + .version(HttpClient.Version.HTTP_1_1) + .build(); + } + + @SuppressWarnings("FutureReturnValueIgnored") + public void redirectAsync( + final String url, + final Request request, + final ProcessResultHandler handler + ) { + logger.trace("sending redirect to node {}", request.getURI()); + final HttpRequest redirectRequest = createRequest(url, request); + client.sendAsync(redirectRequest, HttpResponse.BodyHandlers.ofByteArray()) + .thenApply(this::convert) + .whenComplete(handler::add); + } + + public void redirectSync( + final String url, + final Request request, + final ProcessResultHandler handler + ) throws IOException, InterruptedException { + logger.trace("sending redirect to node {}", request.getURI()); + final HttpRequest redirectRequest = createRequest(url, request); + handler.add(convert(client.send(redirectRequest, HttpResponse.BodyHandlers.ofByteArray()))); + } + + private ProcessResult convert(final HttpResponse response) { + final HttpHeaders httpHeaders = response.headers(); + final OptionalLong value = httpHeaders.firstValueAsLong(Utils.TIMESTAMP_HEADER_NAME); + if (value.isEmpty()) { + return new ProcessResult(response.statusCode(), response.body(), -1); } + return new ProcessResult(response.statusCode(), response.body(), value.getAsLong()); } - public Response redirect( + private HttpRequest createRequest( final String url, final Request request - ) throws HttpException, IOException, InterruptedException { - final HttpClient client = clients.get(url); - try { - logger.trace("sending redirect to node {}", url); - final Request redirectRequest = client.createRequest( - request.getMethod(), - request.getURI() - ); - final byte[] body = request.getBody(); - if (body != null) { - redirectRequest.setBody(body); - } - final String[] headers = request.getHeaders(); - for (int i = 0; i < request.getHeaderCount(); i++) { - redirectRequest.addHeader(headers[i]); - } - redirectRequest.addHeader(REDIRECT_HEADER); - return client.invoke(redirectRequest, REDIRECT_TIMEOUT); - } catch (PoolException e) { - return new Response(Response.BAD_GATEWAY, Response.EMPTY); - } + ) { + final byte[] body = request.getBody(); + return HttpRequest.newBuilder() + .uri(URI.create(url + request.getURI())) + .method( + request.getMethodName(), + body == null + ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofByteArray(request.getBody())) + .header(Utils.REDIRECT_HEADER_NAME, "true") + .timeout(Duration.ofMillis(REDIRECT_TIMEOUT)) + .build(); } public void close() { - for (final Map.Entry entry: clients.entrySet()) { - entry.getValue().close(); - } + client.close(); } + } diff --git a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ServiceFactoryImpl.java b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ServiceFactoryImpl.java index 4942a2dd3..4eb84ecc0 100644 --- a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ServiceFactoryImpl.java +++ b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/ServiceFactoryImpl.java @@ -7,7 +7,7 @@ import java.io.IOException; import java.io.UncheckedIOException; -@ServiceFactory(stage = 4) +@ServiceFactory(stage = 5) public class ServiceFactoryImpl implements ServiceFactory.Factory { @Override public Service create(ServiceConfig config) { diff --git a/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/Utils.java b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/Utils.java new file mode 100644 index 000000000..129d0c02d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/smirnovdmitrii/server/Utils.java @@ -0,0 +1,16 @@ +package ru.vk.itmo.test.smirnovdmitrii.server; + +import java.time.Instant; + +public final class Utils { + public static final String TIMESTAMP_HEADER_NAME = "X-KEY-TS"; + public static final String REDIRECT_HEADER_NAME = "X-REPLICATION-REQUEST"; + public static final String REDIRECT_ONE_NIO_HEADER_NAME = REDIRECT_HEADER_NAME + ":"; + + private Utils() { + } + + public static long currentMillis() { + return Instant.now().toEpochMilli(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyFactory.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyFactory.java new file mode 100644 index 000000000..c03d3dec1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyFactory.java @@ -0,0 +1,14 @@ +package ru.vk.itmo.test.tarazanovmaxim; + +import ru.vk.itmo.Service; +import ru.vk.itmo.ServiceConfig; +import ru.vk.itmo.test.ServiceFactory; + +@ServiceFactory(stage = 3) +public class MyFactory implements ServiceFactory.Factory { + + @Override + public Service create(ServiceConfig config) { + return new MyService(config); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyServer.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyServer.java new file mode 100644 index 000000000..d08c75b03 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyServer.java @@ -0,0 +1,246 @@ +package ru.vk.itmo.test.tarazanovmaxim; + +import one.nio.http.HttpClient; +import one.nio.http.HttpServer; +import one.nio.http.HttpServerConfig; +import one.nio.http.HttpSession; +import one.nio.http.Param; +import one.nio.http.Path; +import one.nio.http.Request; +import one.nio.http.RequestMethod; +import one.nio.http.Response; +import one.nio.net.ConnectionString; +import one.nio.server.AcceptorConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ru.vk.itmo.ServiceConfig; +import ru.vk.itmo.dao.BaseEntry; +import ru.vk.itmo.dao.Config; +import ru.vk.itmo.dao.Entry; +import ru.vk.itmo.test.tarazanovmaxim.dao.ReferenceDao; +import ru.vk.itmo.test.tarazanovmaxim.hash.ConsistentHashing; +import ru.vk.itmo.test.tarazanovmaxim.hash.Hasher; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class MyServer extends HttpServer { + + private static final long FLUSH_THRESHOLD_BYTES = 1 << 20; + private static final long REQUEST_TTL = TimeUnit.SECONDS.toNanos(1000); + private static final String PATH = "/v0/entity"; + private static final Logger logger = LoggerFactory.getLogger(MyServer.class); + private final ReferenceDao dao; + private final ExecutorService executorService; + private final Map httpClients = new HashMap<>(); + private final Map shardKeyMap = new HashMap<>(); + private final ConsistentHashing shards; + private final String selfUrl; + + public MyServer(ServiceConfig config) throws IOException { + super(createServerConfig(config)); + selfUrl = config.selfUrl(); + shards = new ConsistentHashing(); + dao = new ReferenceDao(new Config(config.workingDir(), FLUSH_THRESHOLD_BYTES)); + + int corePoolSize = Runtime.getRuntime().availableProcessors() / 2 + 1; + long keepAliveTime = 0L; + int queueCapacity = 100; + + executorService = new ThreadPoolExecutor( + corePoolSize, + corePoolSize, + keepAliveTime, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(queueCapacity), + new ThreadPoolExecutor.AbortPolicy() + ); + + int nodeCount = 1; + HashSet nodeSet = new HashSet<>(nodeCount); + Hasher hasher = new Hasher(); + for (String url : config.clusterUrls()) { + for (int i = 0; i < nodeCount; ++i) { + nodeSet.add(hasher.digest(url.getBytes(StandardCharsets.UTF_8))); + } + shards.addShard(url, nodeSet); + + httpClients.put(url, new HttpClient(new ConnectionString(url))); + } + } + + private static HttpServerConfig createServerConfig(ServiceConfig serviceConfig) { + HttpServerConfig httpServerConfig = new HttpServerConfig(); + AcceptorConfig acceptorConfig = new AcceptorConfig(); + + acceptorConfig.port = serviceConfig.selfPort(); + acceptorConfig.reusePort = true; + + httpServerConfig.acceptors = new AcceptorConfig[]{acceptorConfig}; + httpServerConfig.closeSessions = true; + + return httpServerConfig; + } + + private static MemorySegment toMemorySegment(String string) { + return MemorySegment.ofArray(string.getBytes(StandardCharsets.UTF_8)); + } + + public void close() throws IOException { + executorService.shutdown(); + executorService.shutdownNow(); + + for (HttpClient httpClient : httpClients.values()) { + httpClient.close(); + } + dao.close(); + } + + private boolean onAnotherServer(final String id) { + return !shards.getShardByKey(id).equals(selfUrl); + } + + private Response shardLookup(final String id, final Request request) { + Response response; + Request redirect = new Request(request); + String shard = shardKeyMap.computeIfAbsent(id, key -> { + return shards.getShardByKey(id); + }); + try { + response = httpClients.get(shard).invoke(redirect, 500); + } catch (Exception e) { + response = new Response(Response.BAD_GATEWAY, Response.EMPTY); + } + return new Response(response.getHeaders()[0], response.getBody()); + } + + @Path(PATH) + @RequestMethod(Request.METHOD_GET) + public final Response get(@Param(value = "id", required = true) String id, Request request) { + MemorySegment key = + (id == null || id.isEmpty()) + ? null + : toMemorySegment(id); + + if (key == null) { + return new Response(Response.BAD_REQUEST, Response.EMPTY); + } + + if (onAnotherServer(id)) { + return shardLookup(id, request); + } + + Entry entry = dao.get(key); + + if (entry == null) { + return new Response(Response.NOT_FOUND, Response.EMPTY); + } + + return Response.ok(entry.value().toArray(ValueLayout.JAVA_BYTE)); + } + + @Path(PATH) + @RequestMethod(Request.METHOD_PUT) + public final Response put(@Param(value = "id", required = true) String id, Request request) { + MemorySegment key = + (id == null || id.isEmpty()) + ? null + : toMemorySegment(id); + + if (key == null) { + return new Response(Response.BAD_REQUEST, Response.EMPTY); + } + + if (onAnotherServer(id)) { + return shardLookup(id, request); + } + + Entry entry = new BaseEntry<>( + key, + MemorySegment.ofArray(request.getBody()) + ); + + dao.upsert(entry); + + return new Response(Response.CREATED, Response.EMPTY); + } + + @Path(PATH) + @RequestMethod(Request.METHOD_DELETE) + public final Response delete(@Param(value = "id", required = true) String id, Request request) { + MemorySegment key = + (id == null || id.isEmpty()) + ? null + : toMemorySegment(id); + + if (key == null) { + return new Response(Response.BAD_REQUEST, Response.EMPTY); + } + + if (onAnotherServer(id)) { + return shardLookup(id, request); + } + + dao.upsert(new BaseEntry<>(key, null)); + + return new Response(Response.ACCEPTED, Response.EMPTY); + } + + @Path(PATH) + public Response otherMethod() { + return new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); + } + + @Override + public void handleDefault(Request request, HttpSession session) { + Response response = new Response(Response.BAD_REQUEST, Response.EMPTY); + sendResponse(response, session); + } + + @Override + public void handleRequest(Request request, HttpSession session) { + try { + long startTime = System.nanoTime(); + executorService.execute(() -> { + if (System.nanoTime() > startTime + REQUEST_TTL) { + sendResponse(new Response(Response.REQUEST_TIMEOUT, Response.EMPTY), session); + return; + } + + try { + super.handleRequest(request, session); + } catch (Exception e) { + logger.error("IOException in handleRequest->executorService.execute()"); + sendResponse( + new Response( + e.getClass() == IOException.class ? Response.INTERNAL_ERROR : Response.BAD_REQUEST, + Response.EMPTY + ), + session + ); + } + }); + } catch (RejectedExecutionException e) { + logger.error("RejectedExecutionException in handleRequest: " + request + session); + sendResponse(new Response("429 Too Many Requests", Response.EMPTY), session); + } + } + + public void sendResponse(Response response, HttpSession session) { + try { + session.sendResponse(response); + } catch (IOException e) { + logger.error("IOException in sendResponse: " + response + session); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyService.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyService.java new file mode 100644 index 000000000..69efb9e58 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/MyService.java @@ -0,0 +1,31 @@ +package ru.vk.itmo.test.tarazanovmaxim; + +import ru.vk.itmo.Service; +import ru.vk.itmo.ServiceConfig; + +import java.io.IOException; +import java.util.concurrent.CompletableFuture; + +public class MyService implements Service { + private MyServer server; + + private final ServiceConfig config; + + public MyService(ServiceConfig config) { + this.config = config; + } + + @Override + public CompletableFuture start() throws IOException { + server = new MyServer(config); + server.start(); + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture stop() throws IOException { + server.stop(); + server.close(); + return CompletableFuture.completedFuture(null); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/ServerMain.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/ServerMain.java new file mode 100644 index 000000000..dd0a1db98 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/ServerMain.java @@ -0,0 +1,45 @@ +package ru.vk.itmo.test.tarazanovmaxim; + +import ru.vk.itmo.ServiceConfig; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + +public final class ServerMain { + + private ServerMain() { + } + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException, URISyntaxException { + List services = new ArrayList<>(); + List urls = List.of( + "http://localhost:8080", + "http://localhost:8081", + "http://localhost:8082" + ); + + for (String url : urls) { + int port = new URI(url).getPort(); + services.add( + new MyService( + new ServiceConfig( + port, + "http://localhost:" + port, + urls, + Files.createTempDirectory("dao" + Integer.toString(port)) + ) + ) + ); + } + + for (MyService service : services) { + service.start().get(); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/ByteArraySegment.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/ByteArraySegment.java new file mode 100644 index 000000000..41b5a19d5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/ByteArraySegment.java @@ -0,0 +1,48 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.nio.ByteBuffer; + +/** + * Growable buffer with {@link ByteBuffer} and {@link MemorySegment} interface. + * + * @author incubos + */ +final class ByteArraySegment { + private byte[] array; + private MemorySegment segment; + + ByteArraySegment(final int capacity) { + this.array = new byte[capacity]; + this.segment = MemorySegment.ofArray(array); + } + + void withArray(final ArrayConsumer consumer) throws IOException { + consumer.process(array); + } + + MemorySegment segment() { + return segment; + } + + void ensureCapacity(final long size) { + if (size > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Too big!"); + } + + final int capacity = (int) size; + if (array.length >= capacity) { + return; + } + + // Grow to the nearest bigger power of 2 + final int newSize = Integer.highestOneBit(capacity) << 1; + array = new byte[newSize]; + segment = MemorySegment.ofArray(array); + } + + interface ArrayConsumer { + void process(byte[] array) throws IOException; + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/LiveFilteringIterator.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/LiveFilteringIterator.java new file mode 100644 index 000000000..7a9e406e7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/LiveFilteringIterator.java @@ -0,0 +1,52 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Filters non tombstone {@link Entry}s. + * + * @author incubos + */ +final class LiveFilteringIterator implements Iterator> { + private final Iterator> delegate; + private Entry next; + + LiveFilteringIterator(final Iterator> delegate) { + this.delegate = delegate; + skipTombstones(); + } + + private void skipTombstones() { + while (delegate.hasNext()) { + final Entry entry = delegate.next(); + if (entry.value() != null) { + this.next = entry; + break; + } + } + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Consume + final Entry result = next; + next = null; + + skipTombstones(); + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MemTable.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MemTable.java new file mode 100644 index 000000000..88cf2382f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MemTable.java @@ -0,0 +1,49 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; + +/** + * Memory table. + * + * @author incubos + */ +final class MemTable { + private final NavigableMap> map = + new ConcurrentSkipListMap<>( + MemorySegmentComparator.INSTANCE); + + boolean isEmpty() { + return map.isEmpty(); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + if (from == null && to == null) { + // All + return map.values().iterator(); + } else if (from == null) { + // Head + return map.headMap(to).values().iterator(); + } else if (to == null) { + // Tail + return map.tailMap(from).values().iterator(); + } else { + // Slice + return map.subMap(from, to).values().iterator(); + } + } + + Entry get(final MemorySegment key) { + return map.get(key); + } + + Entry upsert(final Entry entry) { + return map.put(entry.key(), entry); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MemorySegmentComparator.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MemorySegmentComparator.java new file mode 100644 index 000000000..4dd9f21a8 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MemorySegmentComparator.java @@ -0,0 +1,89 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Comparator; + +/** + * Compares {@link MemorySegment}s. + * + * @author incubos + */ +final class MemorySegmentComparator implements Comparator { + static final Comparator INSTANCE = + new MemorySegmentComparator(); + + private MemorySegmentComparator() { + // Singleton + } + + @Override + public int compare( + final MemorySegment left, + final MemorySegment right) { + final long mismatch = left.mismatch(right); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == left.byteSize()) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == right.byteSize()) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + left.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch), + right.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + mismatch)); + } + + static int compare( + final MemorySegment srcSegment, + final long srcFromOffset, + final long srcLength, + final MemorySegment dstSegment, + final long dstFromOffset, + final long dstLength) { + final long mismatch = + MemorySegment.mismatch( + srcSegment, + srcFromOffset, + srcFromOffset + srcLength, + dstSegment, + dstFromOffset, + dstFromOffset + dstLength); + if (mismatch == -1L) { + // No mismatch + return 0; + } + + if (mismatch == srcLength) { + // left is prefix of right, so left is smaller + return -1; + } + + if (mismatch == dstLength) { + // right is prefix of left, so left is greater + return 1; + } + + // Compare mismatched bytes as unsigned + return Byte.compareUnsigned( + srcSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + srcFromOffset + mismatch), + dstSegment.getAtIndex( + ValueLayout.OfByte.JAVA_BYTE, + dstFromOffset + mismatch)); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MergingEntryIterator.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MergingEntryIterator.java new file mode 100644 index 000000000..d1801a82e --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/MergingEntryIterator.java @@ -0,0 +1,72 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Queue; + +/** + * Merges entry {@link Iterator}s. + * + * @author incubos + */ +final class MergingEntryIterator implements Iterator> { + private final Queue iterators; + + MergingEntryIterator(final List iterators) { + assert iterators.stream().allMatch(WeightedPeekingEntryIterator::hasNext); + + this.iterators = new PriorityQueue<>(iterators); + } + + @Override + public boolean hasNext() { + return !iterators.isEmpty(); + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final WeightedPeekingEntryIterator top = iterators.remove(); + final Entry result = top.next(); + + if (top.hasNext()) { + // Not exhausted + iterators.add(top); + } + + // Remove older versions of the key + while (true) { + final WeightedPeekingEntryIterator iterator = iterators.peek(); + if (iterator == null) { + // Nothing left + break; + } + + // Skip entries with the same key + final Entry entry = iterator.peek(); + if (MemorySegmentComparator.INSTANCE.compare(result.key(), entry.key()) != 0) { + // Reached another key + break; + } + + // Drop + iterators.remove(); + // Skip + iterator.next(); + if (iterator.hasNext()) { + // Not exhausted + iterators.add(iterator); + } + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/ReferenceDao.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/ReferenceDao.java new file mode 100644 index 000000000..4403d3a1b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/ReferenceDao.java @@ -0,0 +1,292 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import ru.vk.itmo.dao.Config; +import ru.vk.itmo.dao.Dao; +import ru.vk.itmo.dao.Entry; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Reference implementation of {@link Dao}. + * + * @author incubos + */ +public class ReferenceDao implements Dao> { + private final Config config; + private final Arena arena; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + // Guarded by lock + private volatile TableSet tableSet; + + private final ExecutorService flusher = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("flusher"); + return result; + }); + private final ExecutorService compactor = + Executors.newSingleThreadExecutor(r -> { + final Thread result = new Thread(r); + result.setName("compactor"); + return result; + }); + + private final AtomicBoolean closed = new AtomicBoolean(); + + public ReferenceDao(final Config config) throws IOException { + this.config = config; + this.arena = Arena.ofShared(); + + // First complete promotion of compacted SSTables + SSTables.promote( + config.basePath(), + 0, + 1); + + this.tableSet = + TableSet.from( + SSTables.discover( + arena, + config.basePath())); + } + + @Override + public Iterator> get( + final MemorySegment from, + final MemorySegment to) { + return new LiveFilteringIterator( + tableSet.get( + from, + to)); + } + + @Override + public Entry get(final MemorySegment key) { + // Without lock, just snapshot of table set + return tableSet.get(key); + } + + @Override + public void upsert(final Entry entry) { + final boolean autoFlush; + lock.readLock().lock(); + try { + if (tableSet.memTableSize.get() > config.flushThresholdBytes() + && tableSet.flushingTable != null) { + throw new IllegalStateException("Can't keep up with flushing!"); + } + + // Upsert + final Entry previous = tableSet.upsert(entry); + + // Update size estimate + final long size = tableSet.memTableSize.addAndGet(sizeOf(entry) - sizeOf(previous)); + autoFlush = size > config.flushThresholdBytes(); + } finally { + lock.readLock().unlock(); + } + + if (autoFlush) { + initiateFlush(true); + } + } + + private static long sizeOf(final Entry entry) { + if (entry == null) { + return 0L; + } + + if (entry.value() == null) { + return entry.key().byteSize(); + } + + return entry.key().byteSize() + entry.value().byteSize(); + } + + private void initiateFlush(final boolean auto) { + flusher.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + if (this.tableSet.memTable.isEmpty()) { + // Nothing to flush + return; + } + + if (auto && this.tableSet.memTableSize.get() < config.flushThresholdBytes()) { + // Not enough data to flush + return; + } + + // Switch memTable to flushing + currentTableSet = this.tableSet.flushing(); + this.tableSet = currentTableSet; + } finally { + lock.writeLock().unlock(); + } + + // Write + final int sequence = currentTableSet.nextSequence(); + try { + new SSTableWriter() + .write( + config.basePath(), + sequence, + currentTableSet.flushingTable.get(null, null)); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-1); + return; + } + + // Open + final SSTable flushed; + try { + flushed = SSTables.open( + arena, + config.basePath(), + sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-2); + return; + } + + // Switch + lock.writeLock().lock(); + try { + this.tableSet = this.tableSet.flushed(flushed); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void flush() throws IOException { + initiateFlush(false); + } + + @Override + public void compact() throws IOException { + compactor.submit(() -> { + final TableSet currentTableSet; + lock.writeLock().lock(); + try { + currentTableSet = this.tableSet; + if (currentTableSet.ssTables.size() < 2) { + // Nothing to compact + return; + } + } finally { + lock.writeLock().unlock(); + } + + // Compact to 0 + try { + new SSTableWriter() + .write( + config.basePath(), + 0, + new LiveFilteringIterator( + currentTableSet.allSSTableEntries())); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-3); + } + + // Open 0 + final SSTable compacted; + try { + compacted = + SSTables.open( + arena, + config.basePath(), + 0); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-4); + return; + } + + // Replace old SSTables with compacted one to + // keep serving requests + final Set replaced = new HashSet<>(currentTableSet.ssTables); + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + replaced, + compacted); + } finally { + lock.writeLock().unlock(); + } + + // Remove compacted SSTables starting from the oldest ones. + // If we crash, 0 contains all the data, and + // it will be promoted on reopen. + for (final SSTable ssTable : currentTableSet.ssTables.reversed()) { + try { + SSTables.remove( + config.basePath(), + ssTable.sequence); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-5); + } + } + + // Promote zero to one (possibly replacing) + try { + SSTables.promote( + config.basePath(), + 0, + 1); + } catch (IOException e) { + e.printStackTrace(); + Runtime.getRuntime().halt(-6); + } + + // Replace promoted SSTable + lock.writeLock().lock(); + try { + this.tableSet = + this.tableSet.compacted( + Collections.singleton(compacted), + compacted.withSequence(1)); + } finally { + lock.writeLock().unlock(); + } + }).state(); + } + + @Override + public void close() throws IOException { + if (closed.getAndSet(true)) { + // Already closed + return; + } + + // Maybe flush + flush(); + + // Stop all the threads + flusher.close(); + compactor.close(); + + // Close arena + arena.close(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTable.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTable.java new file mode 100644 index 000000000..be6c45f4c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTable.java @@ -0,0 +1,204 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import ru.vk.itmo.dao.BaseEntry; +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.util.Collections; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Persistent SSTable in data file and index file. + * + * @author incubos + * @see SSTables + */ +final class SSTable { + final int sequence; + + private final MemorySegment index; + private final MemorySegment data; + private final long size; + + SSTable( + final int sequence, + final MemorySegment index, + final MemorySegment data) { + this.sequence = sequence; + this.index = index; + this.data = data; + this.size = index.byteSize() / Long.BYTES; + } + + SSTable withSequence(final int sequence) { + return new SSTable( + sequence, + index, + data); + } + + /** + * Returns index of the entry if found; otherwise, (-(insertion point) - 1). + * The insertion point is defined as the point at which the key would be inserted: + * the index of the first element greater than the key, + * or size if all keys are less than the specified key. + * Note that this guarantees that the return value will be >= 0 + * if and only if the key is found. + */ + private long entryBinarySearch(final MemorySegment key) { + long low = 0L; + long high = size - 1; + + while (low <= high) { + final long mid = (low + high) >>> 1; + final long midEntryOffset = entryOffset(mid); + final long midKeyLength = getLength(midEntryOffset); + final int compare = + MemorySegmentComparator.compare( + data, + midEntryOffset + Long.BYTES, // Position at key + midKeyLength, + key, + 0L, + key.byteSize()); + + if (compare < 0) { + low = mid + 1; + } else if (compare > 0) { + high = mid - 1; + } else { + return mid; + } + } + + return -(low + 1); + } + + private long entryOffset(final long entry) { + return index.get( + ValueLayout.OfLong.JAVA_LONG, + entry * Long.BYTES); + } + + private long getLength(final long offset) { + return data.get( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + offset); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + assert from == null || to == null || MemorySegmentComparator.INSTANCE.compare(from, to) <= 0; + + // Slice of SSTable in absolute offsets + final long fromOffset; + final long toOffset; + + // Left offset bound + if (from == null) { + // Start from the beginning + fromOffset = 0L; + } else { + final long fromEntry = entryBinarySearch(from); + if (fromEntry >= 0L) { + fromOffset = entryOffset(fromEntry); + } else if (-fromEntry - 1 == size) { + // No relevant data + return Collections.emptyIterator(); + } else { + // Greater but existing key found + fromOffset = entryOffset(-fromEntry - 1); + } + } + + // Right offset bound + if (to == null) { + // Up to the end + toOffset = data.byteSize(); + } else { + final long toEntry = entryBinarySearch(to); + if (toEntry >= 0L) { + toOffset = entryOffset(toEntry); + } else if (-toEntry - 1 == size) { + // Up to the end + toOffset = data.byteSize(); + } else { + // Greater but existing key found + toOffset = entryOffset(-toEntry - 1); + } + } + + return new SliceIterator(fromOffset, toOffset); + } + + Entry get(final MemorySegment key) { + final long entry = entryBinarySearch(key); + if (entry < 0) { + return null; + } + + // Skip key (will reuse the argument) + long offset = entryOffset(entry); + offset += Long.BYTES + key.byteSize(); + // Extract value length + final long valueLength = getLength(offset); + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new BaseEntry<>(key, null); + } else { + // Get value + offset += Long.BYTES; + final MemorySegment value = data.asSlice(offset, valueLength); + return new BaseEntry<>(key, value); + } + } + + private final class SliceIterator implements Iterator> { + private long offset; + private final long toOffset; + + private SliceIterator( + final long offset, + final long toOffset) { + this.offset = offset; + this.toOffset = toOffset; + } + + @Override + public boolean hasNext() { + return offset < toOffset; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + // Read key length + final long keyLength = getLength(offset); + offset += Long.BYTES; + + // Read key + final MemorySegment key = data.asSlice(offset, keyLength); + offset += keyLength; + + // Read value length + final long valueLength = getLength(offset); + offset += Long.BYTES; + + // Read value + if (valueLength == SSTables.TOMBSTONE_VALUE_LENGTH) { + // Tombstone encountered + return new BaseEntry<>(key, null); + } else { + final MemorySegment value = data.asSlice(offset, valueLength); + offset += valueLength; + return new BaseEntry<>(key, value); + } + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTableWriter.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTableWriter.java new file mode 100644 index 000000000..8127ed714 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTableWriter.java @@ -0,0 +1,166 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import ru.vk.itmo.dao.Entry; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Iterator; + +/** + * Writes {@link Entry} {@link Iterator} to SSTable on disk. + * + *

Index file {@code .index} contains {@code long} offsets to entries in data file: + * {@code [offset0, offset1, ...]} + * + *

Data file {@code .data} contains serialized entries: + * {@code } + * + *

Tombstones are encoded as {@code valueLength} {@code -1} and no subsequent value. + * + * @author incubos + */ +final class SSTableWriter { + private static final int BUFFER_SIZE = 64 * 1024; + + // Reusable buffers to eliminate allocations. + // But excessive memory copying is still there :( + // Long cell + private final ByteArraySegment longBuffer = new ByteArraySegment(Long.BYTES); + // Growable blob cell + private final ByteArraySegment blobBuffer = new ByteArraySegment(512); + + void write( + final Path baseDir, + final int sequence, + final Iterator> entries) throws IOException { + // Write to temporary files + final Path tempIndexName = SSTables.tempIndexName(baseDir, sequence); + final Path tempDataName = SSTables.tempDataName(baseDir, sequence); + + // Delete temporary files to eliminate tails + Files.deleteIfExists(tempIndexName); + Files.deleteIfExists(tempDataName); + + // Iterate in a single pass! + // Will write through FileChannel despite extra memory copying and + // no buffering (which may be implemented later). + // Looking forward to MemorySegment facilities in FileChannel! + try (OutputStream index = + new BufferedOutputStream( + new FileOutputStream( + tempIndexName.toFile()), + BUFFER_SIZE); + OutputStream data = + new BufferedOutputStream( + new FileOutputStream( + tempDataName.toFile()), + BUFFER_SIZE)) { + long entryOffset = 0L; + + // Iterate and serialize + while (entries.hasNext()) { + // First write offset to the entry + writeLong(entryOffset, index); + + // Then write the entry + final Entry entry = entries.next(); + entryOffset += writeEntry(entry, data); + } + } + + // Publish files atomically + // FIRST index, LAST data + final Path indexName = + SSTables.indexName( + baseDir, + sequence); + Files.move( + tempIndexName, + indexName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + final Path dataName = + SSTables.dataName( + baseDir, + sequence); + Files.move( + tempDataName, + dataName, + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + + private void writeLong( + final long value, + final OutputStream os) throws IOException { + longBuffer.segment().set( + ValueLayout.OfLong.JAVA_LONG_UNALIGNED, + 0, + value); + longBuffer.withArray(os::write); + } + + private void writeSegment( + final MemorySegment value, + final OutputStream os) throws IOException { + final long size = value.byteSize(); + blobBuffer.ensureCapacity(size); + MemorySegment.copy( + value, + 0L, + blobBuffer.segment(), + 0L, + size); + blobBuffer.withArray(array -> + os.write( + array, + 0, + (int) size)); + } + + /** + * Writes {@link Entry} to {@link FileChannel}. + * + * @return written bytes + */ + private long writeEntry( + final Entry entry, + final OutputStream os) throws IOException { + final MemorySegment key = entry.key(); + final MemorySegment value = entry.value(); + long result = 0L; + + // Key size + writeLong(key.byteSize(), os); + result += Long.BYTES; + + // Key + writeSegment(key, os); + result += key.byteSize(); + + // Value size and possibly value + if (value == null) { + // Tombstone + writeLong(SSTables.TOMBSTONE_VALUE_LENGTH, os); + result += Long.BYTES; + } else { + // Value length + writeLong(value.byteSize(), os); + result += Long.BYTES; + + // Value + writeSegment(value, os); + result += value.byteSize(); + } + + return result; + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTables.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTables.java new file mode 100644 index 000000000..cdee52b87 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/SSTables.java @@ -0,0 +1,162 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +/** + * Provides {@link SSTable} management facilities: dumping and discovery. + * + * @author incubos + */ +final class SSTables { + public static final String INDEX_SUFFIX = ".index"; + public static final String DATA_SUFFIX = ".data"; + public static final long TOMBSTONE_VALUE_LENGTH = -1L; + + private static final String TEMP_SUFFIX = ".tmp"; + + /** + * Can't instantiate. + */ + private SSTables() { + // Only static methods + } + + static Path indexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX); + } + + static Path dataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX); + } + + static Path tempIndexName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + INDEX_SUFFIX + TEMP_SUFFIX); + } + + static Path tempDataName( + final Path baseDir, + final int sequence) { + return baseDir.resolve(sequence + DATA_SUFFIX + TEMP_SUFFIX); + } + + /** + * Returns {@link List} of {@link SSTable}s from freshest to oldest. + */ + static List discover( + final Arena arena, + final Path baseDir) throws IOException { + if (!Files.exists(baseDir)) { + return Collections.emptyList(); + } + + final List result = new ArrayList<>(); + try (Stream files = Files.list(baseDir)) { + files.forEach(file -> { + final String fileName = file.getFileName().toString(); + if (!fileName.endsWith(DATA_SUFFIX)) { + // Skip non data + return; + } + + final int sequence = + // .data -> N + Integer.parseInt( + fileName.substring( + 0, + fileName.length() - DATA_SUFFIX.length())); + + try { + result.add(open(arena, baseDir, sequence)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + // Sort from freshest to oldest + result.sort((o1, o2) -> Integer.compare(o2.sequence, o1.sequence)); + + return Collections.unmodifiableList(result); + } + + static SSTable open( + final Arena arena, + final Path baseDir, + final int sequence) throws IOException { + final MemorySegment index = + mapReadOnly( + arena, + indexName(baseDir, sequence)); + final MemorySegment data = + mapReadOnly( + arena, + dataName(baseDir, sequence)); + + return new SSTable( + sequence, + index, + data); + } + + private static MemorySegment mapReadOnly( + final Arena arena, + final Path file) throws IOException { + try (FileChannel channel = + FileChannel.open( + file, + StandardOpenOption.READ)) { + return channel.map( + FileChannel.MapMode.READ_ONLY, + 0L, + Files.size(file), + arena); + } + } + + static void remove( + final Path baseDir, + final int sequence) throws IOException { + // First delete data file to make SSTable invisible + Files.delete(dataName(baseDir, sequence)); + Files.delete(indexName(baseDir, sequence)); + } + + static void promote( + final Path baseDir, + final int from, + final int to) throws IOException { + // Build to progress to the same outcome + if (Files.exists(indexName(baseDir, from))) { + Files.move( + indexName(baseDir, from), + indexName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + if (Files.exists(dataName(baseDir, from))) { + Files.move( + dataName(baseDir, from), + dataName(baseDir, to), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/TableSet.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/TableSet.java new file mode 100644 index 000000000..0097df2eb --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/TableSet.java @@ -0,0 +1,205 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Data set in various tables. + * + * @author incubos + */ +final class TableSet { + final MemTable memTable; + final AtomicLong memTableSize; + // null or read-only + final MemTable flushingTable; + // From freshest to oldest + final List ssTables; + + private TableSet( + final MemTable memTable, + final AtomicLong memTableSize, + final MemTable flushingTable, + final List ssTables) { + this.memTable = memTable; + this.memTableSize = memTableSize; + this.flushingTable = flushingTable; + this.ssTables = ssTables; + } + + static TableSet from(final List ssTables) { + return new TableSet( + new MemTable(), + new AtomicLong(), + null, + ssTables); + } + + int nextSequence() { + return ssTables.stream() + .mapToInt(t -> t.sequence) + .max() + .orElse(0) + 1; + } + + TableSet flushing() { + if (memTable.isEmpty()) { + throw new IllegalStateException("Nothing to flush"); + } + + if (flushingTable != null) { + throw new IllegalStateException("Already flushing"); + } + + return new TableSet( + new MemTable(), + new AtomicLong(), + memTable, + ssTables); + } + + TableSet flushed(final SSTable flushed) { + final List newSSTables = new ArrayList<>(ssTables.size() + 1); + newSSTables.add(flushed); + newSSTables.addAll(ssTables); + return new TableSet( + memTable, + memTableSize, + null, + newSSTables); + } + + TableSet compacted( + final Set replaced, + final SSTable with) { + final List newSsTables = new ArrayList<>(this.ssTables.size() + 1); + + // Keep not replaced SSTables + for (final SSTable ssTable : this.ssTables) { + if (!replaced.contains(ssTable)) { + newSsTables.add(ssTable); + } + } + + // Logically the oldest one + newSsTables.add(with); + + return new TableSet( + memTable, + memTableSize, + flushingTable, + newSsTables); + } + + Iterator> get( + final MemorySegment from, + final MemorySegment to) { + final List iterators = + new ArrayList<>(2 + ssTables.size()); + + // MemTable goes first + final Iterator> memTableIterator = + memTable.get(from, to); + if (memTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE, + memTableIterator)); + } + + // Then goes flushing + if (flushingTable != null) { + final Iterator> flushingIterator = + flushingTable.get(from, to); + if (flushingIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + Integer.MIN_VALUE + 1, + flushingIterator)); + } + } + + // Then go all the SSTables + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(from, to); + if (ssTableIterator.hasNext()) { + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + } + + return switch (iterators.size()) { + case 0 -> Collections.emptyIterator(); + case 1 -> iterators.get(0); + default -> new MergingEntryIterator(iterators); + }; + } + + Entry get(final MemorySegment key) { + // Slightly optimized version not to pollute the heap + + // First check MemTable + Entry result = memTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + + // Then check flushing + if (flushingTable != null) { + result = flushingTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + } + + // At last check SSTables from freshest to oldest + for (final SSTable ssTable : ssTables) { + result = ssTable.get(key); + if (result != null) { + // Transform tombstone + return swallowTombstone(result); + } + } + + // Nothing found + return null; + } + + private static Entry swallowTombstone(final Entry entry) { + return entry.value() == null ? null : entry; + } + + Entry upsert(final Entry entry) { + return memTable.upsert(entry); + } + + Iterator> allSSTableEntries() { + final List iterators = + new ArrayList<>(ssTables.size()); + + for (int i = 0; i < ssTables.size(); i++) { + final SSTable ssTable = ssTables.get(i); + final Iterator> ssTableIterator = + ssTable.get(null, null); + iterators.add( + new WeightedPeekingEntryIterator( + i, + ssTableIterator)); + } + + return new MergingEntryIterator(iterators); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/WeightedPeekingEntryIterator.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/WeightedPeekingEntryIterator.java new file mode 100644 index 000000000..d28fd52e2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/dao/WeightedPeekingEntryIterator.java @@ -0,0 +1,67 @@ +package ru.vk.itmo.test.tarazanovmaxim.dao; + +import ru.vk.itmo.dao.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * Peeking {@link Iterator} wrapper. + * + * @author incubos + */ +final class WeightedPeekingEntryIterator + implements Iterator>, + Comparable { + private final int weight; + private final Iterator> delegate; + private Entry next; + + WeightedPeekingEntryIterator( + final int weight, + final Iterator> delegate) { + this.weight = weight; + this.delegate = delegate; + this.next = delegate.hasNext() ? delegate.next() : null; + } + + @Override + public boolean hasNext() { + return next != null; + } + + @Override + public Entry next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + final Entry result = next; + next = delegate.hasNext() ? delegate.next() : null; + return result; + } + + Entry peek() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + return next; + } + + @Override + public int compareTo(final WeightedPeekingEntryIterator other) { + // First compare keys + int result = + MemorySegmentComparator.INSTANCE.compare( + peek().key(), + other.peek().key()); + if (result != 0) { + return result; + } + + // Then compare weights if keys are equal + return Integer.compare(weight, other.weight); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/hash/ConsistentHashing.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/hash/ConsistentHashing.java new file mode 100644 index 000000000..2dabd30fe --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/hash/ConsistentHashing.java @@ -0,0 +1,33 @@ +package ru.vk.itmo.test.tarazanovmaxim.hash; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +public class ConsistentHashing { + private final Map ring = new TreeMap<>(); + private final Hasher hasher = new Hasher(); + + private int hashKey(String key) { + return hasher.digest(key.getBytes(StandardCharsets.UTF_8)); + } + + public String getShardByKey(String key) { + int keyHash = hashKey(key); + + for (Map.Entry e : ring.entrySet()) { + if (e.getKey() >= keyHash) { + return e.getValue(); + } + } + + return ring.isEmpty() ? null : ring.get(ring.keySet().iterator().next()); + } + + public void addShard(String newShard, Set nodeHashes) { + for (final int nodeHash : nodeHashes) { + ring.put(nodeHash, newShard); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/hash/Hasher.java b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/hash/Hasher.java new file mode 100644 index 000000000..780492f23 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/hash/Hasher.java @@ -0,0 +1,29 @@ +package ru.vk.itmo.test.tarazanovmaxim.hash; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class Hasher { + private final ThreadLocal messageDigest = ThreadLocal.withInitial(() -> { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + logger.error("Error initializing Hasher: ", e.getMessage()); + throw new IllegalArgumentException("Error initializing MessageDigest", e); + } + }); + private static final Logger logger = LoggerFactory.getLogger(Hasher.class); + + public int digest(byte[] bytes) { + final byte[] hash = messageDigest.get().digest(bytes); + return ByteBuffer.wrap(hash).getInt(); + } + + public void close() { + messageDigest.remove(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/getmany_3500_alloc.png b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/getmany_3500_alloc.png new file mode 100644 index 000000000..224d960c0 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/getmany_3500_alloc.png differ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/getmany_3500_cpu.png b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/getmany_3500_cpu.png new file mode 100644 index 000000000..7df6191ff Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/getmany_3500_cpu.png differ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/putmany_3500_alloc.png b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/putmany_3500_alloc.png new file mode 100644 index 000000000..223833c7b Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/putmany_3500_alloc.png differ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/putmany_3500_cpu.png b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/putmany_3500_cpu.png new file mode 100644 index 000000000..0ae4a2555 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/images/putmany_3500_cpu.png differ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/get_20s_12000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/get_20s_12000.txt new file mode 100644 index 000000000..8de69d0b6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/get_20s_12000.txt @@ -0,0 +1,114 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 1.806ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.46ms 2.29ms 10.01ms 81.14% + Req/Sec 12.68k 0.94k 16.44k 67.57% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.38ms + 75.000% 3.68ms + 90.000% 6.48ms + 99.000% 8.96ms + 99.900% 9.71ms + 99.990% 9.96ms + 99.999% 10.01ms +100.000% 10.02ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.017 0.000000 2 1.00 + 0.490 0.100000 12013 1.11 + 0.728 0.200000 24033 1.25 + 0.935 0.300000 36021 1.43 + 1.138 0.400000 48026 1.67 + 1.375 0.500000 59974 2.00 + 1.566 0.550000 65977 2.22 + 1.838 0.600000 71973 2.50 + 2.237 0.650000 77962 2.86 + 2.917 0.700000 83952 3.33 + 3.683 0.750000 89954 4.00 + 3.981 0.775000 92957 4.44 + 4.275 0.800000 95947 5.00 + 4.859 0.825000 98951 5.71 + 5.511 0.850000 101951 6.67 + 5.979 0.875000 104949 8.00 + 6.227 0.887500 106439 8.89 + 6.479 0.900000 107944 10.00 + 6.679 0.912500 109460 11.43 + 6.871 0.925000 110957 13.33 + 7.063 0.937500 112445 16.00 + 7.175 0.943750 113203 17.78 + 7.319 0.950000 113952 20.00 + 7.491 0.956250 114692 22.86 + 7.611 0.962500 115441 26.67 + 7.795 0.968750 116197 32.00 + 7.907 0.971875 116559 35.56 + 8.047 0.975000 116936 40.00 + 8.187 0.978125 117312 45.71 + 8.351 0.981250 117686 53.33 + 8.591 0.984375 118067 64.00 + 8.743 0.985938 118251 71.11 + 8.839 0.987500 118445 80.00 + 8.911 0.989062 118624 91.43 + 8.991 0.990625 118819 106.67 + 9.063 0.992188 119002 128.00 + 9.095 0.992969 119096 142.22 + 9.127 0.993750 119191 160.00 + 9.167 0.994531 119288 182.86 + 9.207 0.995313 119382 213.33 + 9.247 0.996094 119480 256.00 + 9.263 0.996484 119519 284.44 + 9.279 0.996875 119557 320.00 + 9.311 0.997266 119604 365.71 + 9.367 0.997656 119652 426.67 + 9.447 0.998047 119697 512.00 + 9.511 0.998242 119721 568.89 + 9.567 0.998437 119745 640.00 + 9.615 0.998633 119770 731.43 + 9.663 0.998828 119792 853.33 + 9.719 0.999023 119814 1024.00 + 9.775 0.999121 119826 1137.78 + 9.815 0.999219 119838 1280.00 + 9.847 0.999316 119852 1462.86 + 9.871 0.999414 119864 1706.67 + 9.879 0.999512 119873 2048.00 + 9.887 0.999561 119879 2275.56 + 9.911 0.999609 119895 2560.00 + 9.911 0.999658 119895 2925.71 + 9.919 0.999707 119898 3413.33 + 9.935 0.999756 119907 4096.00 + 9.935 0.999780 119907 4551.11 + 9.943 0.999805 119912 5120.00 + 9.943 0.999829 119912 5851.43 + 9.951 0.999854 119916 6826.67 + 9.959 0.999878 119919 8192.00 + 9.959 0.999890 119919 9102.22 + 9.967 0.999902 119923 10240.00 + 9.967 0.999915 119923 11702.86 + 9.967 0.999927 119923 13653.33 + 9.975 0.999939 119924 16384.00 + 9.983 0.999945 119926 18204.44 + 9.983 0.999951 119926 20480.00 + 9.983 0.999957 119926 23405.71 + 9.991 0.999963 119929 27306.67 + 9.991 0.999969 119929 32768.00 + 9.991 0.999973 119929 36408.89 + 9.991 0.999976 119929 40960.00 + 9.991 0.999979 119929 46811.43 + 9.991 0.999982 119929 54613.33 + 10.007 0.999985 119930 65536.00 + 10.007 0.999986 119930 72817.78 + 10.007 0.999988 119930 81920.00 + 10.007 0.999989 119930 93622.86 + 10.007 0.999991 119930 109226.67 + 10.015 0.999992 119931 131072.00 + 10.015 1.000000 119931 inf +#[Mean = 2.455, StdDeviation = 2.291] +#[Max = 10.008, Total count = 119931] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 239980 requests in 20.00s, 17.57MB read + Non-2xx or 3xx responses: 151149 +Requests/sec: 11999.03 +Transfer/sec: 0.88MB diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/get_20s_13000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/get_20s_13000.txt new file mode 100644 index 000000000..5393df082 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/get_20s_13000.txt @@ -0,0 +1,85 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 210.083ms, rate sampling interval: 844ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 710.55ms 149.39ms 983.55ms 59.66% + Req/Sec 12.36k 265.91 12.74k 54.55% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 690.17ms + 75.000% 849.92ms + 90.000% 918.53ms + 99.000% 976.38ms + 99.900% 983.55ms + 99.990% 984.06ms + 99.999% 984.06ms +100.000% 984.06ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 478.975 0.000000 3 1.00 + 522.239 0.100000 12484 1.11 + 563.711 0.200000 24810 1.25 + 599.551 0.300000 37310 1.43 + 619.007 0.400000 49517 1.67 + 690.175 0.500000 61714 2.00 + 732.159 0.550000 67926 2.22 + 767.999 0.600000 74031 2.50 + 806.399 0.650000 80212 2.86 + 836.607 0.700000 86459 3.33 + 849.919 0.750000 92586 4.00 + 857.087 0.775000 95671 4.44 + 865.279 0.800000 98700 5.00 + 872.959 0.825000 101783 5.71 + 887.807 0.850000 104935 6.67 + 904.703 0.875000 108100 8.00 + 910.847 0.887500 109500 8.89 + 918.527 0.900000 111103 10.00 + 922.623 0.912500 112589 11.43 + 934.399 0.925000 114254 13.33 + 944.127 0.937500 115720 16.00 + 948.735 0.943750 116614 17.78 + 951.295 0.950000 117291 20.00 + 954.367 0.956250 117988 22.86 + 959.999 0.962500 118787 26.67 + 963.071 0.968750 119577 32.00 + 964.607 0.971875 119967 35.56 + 967.167 0.975000 120357 40.00 + 969.215 0.978125 120714 45.71 + 971.263 0.981250 121073 53.33 + 973.311 0.984375 121541 64.00 + 974.335 0.985938 121692 71.11 + 975.359 0.987500 121918 80.00 + 975.871 0.989062 122026 91.43 + 976.895 0.990625 122302 106.67 + 978.431 0.992188 122481 128.00 + 978.943 0.992969 122562 142.22 + 979.455 0.993750 122716 160.00 + 979.455 0.994531 122716 182.86 + 979.967 0.995313 122819 213.33 + 980.479 0.996094 122958 256.00 + 980.479 0.996484 122958 284.44 + 980.991 0.996875 123018 320.00 + 981.503 0.997266 123079 365.71 + 982.015 0.997656 123101 426.67 + 982.527 0.998047 123133 512.00 + 983.039 0.998242 123223 568.89 + 983.039 0.998437 123223 640.00 + 983.039 0.998633 123223 731.43 + 983.551 0.998828 123312 853.33 + 983.551 0.999023 123312 1024.00 + 983.551 0.999121 123312 1137.78 + 983.551 0.999219 123312 1280.00 + 983.551 0.999316 123312 1462.86 + 983.551 0.999414 123312 1706.67 + 983.551 0.999512 123312 2048.00 + 984.063 0.999561 123372 2275.56 + 984.063 1.000000 123372 inf +#[Mean = 710.546, StdDeviation = 149.394] +#[Max = 983.552, Total count = 123372] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 247207 requests in 20.00s, 18.10MB read + Non-2xx or 3xx responses: 155643 +Requests/sec: 12360.30 +Transfer/sec: 0.91MB diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/put_20s_100000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/put_20s_100000.txt new file mode 100644 index 000000000..f2e53d5ae --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/put_20s_100000.txt @@ -0,0 +1,91 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 361.853ms, rate sampling interval: 1349ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.06s 195.37ms 1.47s 62.18% + Req/Sec 92.55k 2.38k 95.25k 71.43% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.05s + 75.000% 1.21s + 90.000% 1.31s + 99.000% 1.47s + 99.900% 1.47s + 99.990% 1.47s + 99.999% 1.47s +100.000% 1.47s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 716.799 0.000000 744 1.00 + 779.775 0.100000 92592 1.11 + 871.423 0.200000 185178 1.25 + 942.079 0.300000 279263 1.43 + 1004.543 0.400000 369958 1.67 + 1049.599 0.500000 461924 2.00 + 1076.223 0.550000 508083 2.22 + 1108.991 0.600000 554944 2.50 + 1132.543 0.650000 601126 2.86 + 1169.407 0.700000 646660 3.33 + 1205.247 0.750000 693773 4.00 + 1222.655 0.775000 717257 4.44 + 1241.087 0.800000 739571 5.00 + 1257.471 0.825000 762454 5.71 + 1273.855 0.850000 786189 6.67 + 1293.311 0.875000 808513 8.00 + 1304.575 0.887500 820196 8.89 + 1309.695 0.900000 831641 10.00 + 1319.935 0.912500 843416 11.43 + 1328.127 0.925000 855735 13.33 + 1388.543 0.937500 865981 16.00 + 1423.359 0.943750 872504 17.78 + 1430.527 0.950000 877677 20.00 + 1436.671 0.956250 885017 22.86 + 1440.767 0.962500 889161 26.67 + 1449.983 0.968750 896209 32.00 + 1451.007 0.971875 898591 35.56 + 1461.247 0.975000 903094 40.00 + 1462.271 0.978125 906335 45.71 + 1463.295 0.981250 907421 53.33 + 1464.319 0.984375 909864 64.00 + 1465.343 0.985938 911277 71.11 + 1466.367 0.987500 913299 80.00 + 1467.391 0.989062 915166 91.43 + 1467.391 0.990625 915166 106.67 + 1468.415 0.992188 916984 128.00 + 1469.439 0.992969 919013 142.22 + 1469.439 0.993750 919013 160.00 + 1469.439 0.994531 919013 182.86 + 1470.463 0.995313 920642 213.33 + 1470.463 0.996094 920642 256.00 + 1470.463 0.996484 920642 284.44 + 1471.487 0.996875 922935 320.00 + 1471.487 0.997266 922935 365.71 + 1471.487 0.997656 922935 426.67 + 1471.487 0.998047 922935 512.00 + 1471.487 0.998242 922935 568.89 + 1471.487 0.998437 922935 640.00 + 1471.487 0.998633 922935 731.43 + 1471.487 0.998828 922935 853.33 + 1471.487 0.999023 922935 1024.00 + 1471.487 0.999121 922935 1137.78 + 1472.511 0.999219 923435 1280.00 + 1472.511 0.999316 923435 1462.86 + 1472.511 0.999414 923435 1706.67 + 1472.511 0.999512 923435 2048.00 + 1472.511 0.999561 923435 2275.56 + 1472.511 0.999609 923435 2560.00 + 1472.511 0.999658 923435 2925.71 + 1473.535 0.999707 923535 3413.33 + 1473.535 0.999756 923535 4096.00 + 1473.535 0.999780 923535 4551.11 + 1473.535 0.999805 923535 5120.00 + 1474.559 0.999829 923709 5851.43 + 1474.559 1.000000 923709 inf +#[Mean = 1058.136, StdDeviation = 195.375] +#[Max = 1473.536, Total count = 923709] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1852558 requests in 20.00s, 118.37MB read +Requests/sec: 92627.79 +Transfer/sec: 5.92MB diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/put_20s_90000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/put_20s_90000.txt new file mode 100644 index 000000000..57e14ef50 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/outputs/put_20s_90000.txt @@ -0,0 +1,83 @@ + +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 40.408ms, rate sampling interval: 275ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 52.59ms 54.48ms 155.14ms 75.72% + Req/Sec 91.39k 2.60k 94.79k 66.67% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 29.39ms + 75.000% 102.27ms + 90.000% 139.26ms + 99.000% 152.83ms + 99.900% 155.01ms + 99.990% 155.26ms + 99.999% 155.26ms +100.000% 155.26ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.008 0.000000 4 1.00 + 0.509 0.100000 91338 1.11 + 1.016 0.200000 182479 1.25 + 5.415 0.300000 273518 1.43 + 11.535 0.400000 364762 1.67 + 29.391 0.500000 456080 2.00 + 37.727 0.550000 501465 2.22 + 57.823 0.600000 547066 2.50 + 73.663 0.650000 592617 2.86 + 86.719 0.700000 638473 3.33 + 102.271 0.750000 683840 4.00 + 113.087 0.775000 706690 4.44 + 121.791 0.800000 729512 5.00 + 127.295 0.825000 752563 5.71 + 130.687 0.850000 775505 6.67 + 134.527 0.875000 798724 8.00 + 136.831 0.887500 809585 8.89 + 139.263 0.900000 820543 10.00 + 141.567 0.912500 832408 11.43 + 144.383 0.925000 843622 13.33 + 147.327 0.937500 856403 16.00 + 147.839 0.943750 860639 17.78 + 148.479 0.950000 866672 20.00 + 148.863 0.956250 872392 22.86 + 149.247 0.962500 878635 26.67 + 149.631 0.968750 883665 32.00 + 149.887 0.971875 886651 35.56 + 150.143 0.975000 889366 40.00 + 150.527 0.978125 892169 45.71 + 151.167 0.981250 895038 53.33 + 151.679 0.984375 897719 64.00 + 152.063 0.985938 899205 71.11 + 152.447 0.987500 900780 80.00 + 152.703 0.989062 902591 91.43 + 152.959 0.990625 903371 106.67 + 153.343 0.992188 904888 128.00 + 153.471 0.992969 905320 142.22 + 153.727 0.993750 906045 160.00 + 153.983 0.994531 906849 182.86 + 154.239 0.995313 908098 213.33 + 154.367 0.996094 908581 256.00 + 154.367 0.996484 908581 284.44 + 154.495 0.996875 909019 320.00 + 154.623 0.997266 909542 365.71 + 154.751 0.997656 910094 426.67 + 154.751 0.998047 910094 512.00 + 154.879 0.998242 910496 568.89 + 154.879 0.998437 910496 640.00 + 154.879 0.998633 910496 731.43 + 155.007 0.998828 910914 853.33 + 155.007 0.999023 910914 1024.00 + 155.007 0.999121 910914 1137.78 + 155.135 0.999219 911159 1280.00 + 155.135 0.999316 911159 1462.86 + 155.263 0.999414 911712 1706.67 + 155.263 1.000000 911712 inf +#[Mean = 52.586, StdDeviation = 54.482] +#[Max = 155.136, Total count = 911712] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1799951 requests in 20.00s, 115.01MB read +Requests/sec: 89997.44 +Transfer/sec: 5.75MB diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/report.md b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/report.md new file mode 100644 index 000000000..65ff7b692 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/report.md @@ -0,0 +1,28 @@ +# Таразанов Максим, ИТМО, М4139, Stage 1 + +Работа выполнена на WSL2. + +При выполнении задания использовалась референсный **dao**. +В первую очередь было решено проверить точку разладки. В результате было получено, что разладка для **put** наступает при 4000rps, а для **get** -- при 3500rps. +Далее тестирование проводилось на 3500rps при замере времени в 120 секунд. +## GET-запросы +- Alloc +![getmany_3500_alloc.png](images%2Fgetmany_3500_alloc.png) +Около 60% выделяется на handleRequest, из которых 20% на создание реквеста, 40% на сервер. Стоит заметить, что больша'я часть памяти в совокупности нужна для создания memorySegment'ов. +Также около 20% аллокаций происходит на парсинг реквеста. + +- CPU +![getmany_3500_cpu.png](images%2Fgetmany_3500_cpu.png) +Больше половины времени уходит на запись в сокет. + +## PUT-запросы +- Alloc +![putmany_3500_alloc.png](images%2Fputmany_3500_alloc.png) +Около 60% также уходит на handleRequest, но на этот раз на сервер уходит ме'ньшая часть (порядка 35%). В совокупности опять же бо'льшая часть памяти уходит также на создание memorySegment'ов. + +- CPU +![putmany_3500_cpu.png](images%2Fputmany_3500_cpu.png) +Больше половины времени уходит на запись в сокет. + +## Улучшения +Предлагается сделать меньше операций со String путём работы с байтами запроса. \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/get.lua b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/get.lua new file mode 100644 index 000000000..b1f8629a2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/get.lua @@ -0,0 +1,6 @@ +wrk.method = "GET" + +request = function() + path = "/v0/entity?id=1" + return wrk.format(nil, path) +end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/getmany.lua b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/getmany.lua new file mode 100644 index 000000000..a33a3024a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/getmany.lua @@ -0,0 +1,6 @@ +wrk.method = "GET" + +request = function() + path = "/v0/entity?id=" .. math.random(0, 170000) + return wrk.format(nil, path) +end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/put.lua b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/put.lua new file mode 100644 index 000000000..2c9cbae99 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/put.lua @@ -0,0 +1,9 @@ +wrk.method = "PUT" + +request = function() + path = "/v0/entity?id=1" + body = "hello world!" + headers = {} + headers["Content-Type"] = "text/plain" + return wrk.format(nil, path, headers, body) +end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/putmany.lua b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/putmany.lua new file mode 100644 index 000000000..8bf99543a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage1/scripts/putmany.lua @@ -0,0 +1,12 @@ +wrk.method = "PUT" + +key = 0 + +request = function() + key = key + 1 + path = "/v0/entity?id=" .. key + body = "hello world" .. key + headers = {} + headers["Content-Type"] = "text/plain" + return wrk.format(nil, path, headers, body) +end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_alloc.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_alloc.html new file mode 100644 index 000000000..4ccdacbbe --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_alloc.html @@ -0,0 +1,453 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_cpu.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_cpu.html new file mode 100644 index 000000000..e47518103 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_cpu.html @@ -0,0 +1,1087 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_lock.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_lock.html new file mode 100644 index 000000000..31d5b9017 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_128_lock.html @@ -0,0 +1,338 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/results/ALLOCLinkedGET.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_alloc.html similarity index 87% rename from src/main/java/ru/vk/itmo/test/kachmareugene/results/ALLOCLinkedGET.html rename to src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_alloc.html index 5d76eb08a..57e2fae9d 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/results/ALLOCLinkedGET.html +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_alloc.html @@ -14,7 +14,7 @@ #status {left: 0} #match {right: 0} #reset {cursor: pointer} - #canvas {width: 100%; height: 336px} + #canvas {width: 100%; height: 320px} @@ -32,7 +32,7 @@

Allocation profile

let root, rootLevel, px, pattern; let level0 = 0, left0 = 0, width0 = 0; let reverse = false; - const levels = Array(21); + const levels = Array(20); for (let h = 0; h < levels.length; h++) { levels[h] = []; } @@ -251,7 +251,7 @@

Allocation profile

const cpool = [ 'all', -' RequestHandler1_getEntry.handleRequest', +' RequestHandler0_get.handleRequest', ' byte[]', ' java.lang.String', '0[]', @@ -265,11 +265,11 @@

Allocation profile

'*Thread.run', '4With', '*foreign/MemorySegment.ofArray', -'*invoke/DirectMethodHandle$Holder.newInvokeSpecial', -'C.allocateInstance', -'1Invokers$Holder.linkToTargetMethod', -'1LambdaForm$DMH.0x00007f8f7c092800.newInvokeSpecial', -'Allocation profile '!dk.internal.foreign.GlobalSession', '5HeapMemorySegmentImpl$OfByte', '5MappedMemorySegmentImpl', -'#/internal/foreign/AbstractMemorySegmentImpl$$Lambda.0x00007f8f7c0ef8c8.apply', +'#/internal/foreign/AbstractMemorySegmentImpl$$Lambda.0x00007f1ef80654d0.apply', 'N.asSlice', 'VNoCheck', 'Olambda$toArray$1', @@ -306,7 +306,9 @@

Allocation profile

'-Request.', '5getParameter', ':th', +'8RequiredParameter', '/sponse.', +'6ok', '6toBytes', '(net/NativeSelector.select', ',Session.process', @@ -316,18 +318,19 @@

Allocation profile

'-Utf8.read', '2toAsciiString', ' ru.vk.itmo.dao.BaseEntry', -'+test.kachmareugene.HttpServerImpl$$Lambda.0x00007f8f7c08ecd0', -'"/vk/itmo/test/kachmareugene/HttpServerImpl$$Lambda.0x00007f8f7c08ecd0.run', -'L.getEntry', -'MhandleRequest', -'Mlambda$handleRequest$0', -'0reference/dao/ReferenceDao.get', -'>SSTable.get', -'>TableSet.get' +'+test.tarazanovmaxim.MyServer$$Lambda.0x00007f1ef808ec68', +'"/vk/itmo/test/tarazanovmaxim/MyServer$$Lambda.0x00007f1ef808ec68.run', +'G.get', +'HhandleRequest', +'Hlambda$handleRequest$0', +'HtoMemorySegment', +'?dao/ReferenceDao.get', +'CSSTable.get', +'CTableSet.get' ]; unpack(cpool); -n(3,241) +n(3,252) u(97,154) u(105) u(193) @@ -337,98 +340,105 @@

Allocation profile

u(225) u(233) u(50) -f(537,5,8,146) -u(561) +f(553,5,8,146) +u(577) u(369) -u(9,131) -u(409,23) +u(9,127) +u(409,43) u(417) -u(457,16) -u(497) +u(473,28) +u(513) u(18) -f(481,11,16,7) +f(497,11,28,15) u(362) -f(433,9,7,15) +f(449,9,15,11) +u(433) u(81) u(81) u(89) -u(26,4) -n(161,11) +u(26,3) +n(161,8) u(169) u(18) -f(545,9,11,93) -u(73,5) +f(561,9,8,73) +u(297,6) +u(297) +u(265) +u(289) +u(18) +f(346,10,6,4) +n(457,17) +u(18,4) +n(26,6) +n(34,7) +f(465,10,7,14) +u(346,5) +n(457,9) +u(18,4) +n(26,2) +n(34,3) +f(585,10,3,29) +u(73,8) u(57) u(65) u(18) -f(113,10,5,21) +f(113,11,8,21) u(305) -u(250,8) -n(329,13) +u(250,12) +n(329,9) u(242) -f(297,10,13,10) -u(297) -u(265) -u(289) -u(18) -f(346,10,10,9) -n(449,32) -u(18,9) -n(26,10) -n(34,13) -f(569,10,13,16) -u(569) -u(585) -u(577) -u(313,5) +f(593,10,9,3) +u(609) +u(601) +u(313,2) u(313) u(273) u(281) u(321) u(321) u(258) -f(522,14,5,11) -f(441,8,11,15) +f(538,13,2,1) +f(441,8,1,19) u(81) u(89) u(26,5) -n(161,10) +n(161,14) u(169) u(18) -f(489,1,10,87) -u(465,1) +f(505,1,14,98) +u(481,13) u(354) -f(473,2,1,86) +f(489,2,13,85) u(401) u(393) -u(377,15) -u(553) -u(137,6) -u(121) +u(377,17) +u(569) +u(153,14) u(129) -u(530) -f(201,7,6,9) +u(121) +u(546) +f(201,7,14,3) u(177) u(42) -f(385,5,9,53) -u(338,11) -n(425,24) +f(385,5,3,49) +u(338,10) +n(425,23) u(34) -f(505,6,24,18) -u(505) -u(513) -u(18,11) -n(153,7) -u(145) -u(129) +f(521,6,23,16) +u(521) +u(529) +u(18,12) +n(145,4) +u(137) +u(121) u(26) -f(505,5,7,18) -u(505) -u(513) -u(18,13) -n(153,5) -u(145) -u(129) +f(521,5,4,19) +u(521) +u(529) +u(18,11) +n(145,8) +u(137) +u(121) u(26) search(); diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_cpu.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_cpu.html new file mode 100644 index 000000000..dfd0b0ddf --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_cpu.html @@ -0,0 +1,823 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_lock.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_lock.html new file mode 100644 index 000000000..7c8d28584 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/get_20s_12000_lock.html @@ -0,0 +1,285 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_alloc.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_alloc.html new file mode 100644 index 000000000..e40d8ac78 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_alloc.html @@ -0,0 +1,521 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_cpu.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_cpu.html new file mode 100644 index 000000000..102721851 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_cpu.html @@ -0,0 +1,1983 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_lock.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_lock.html new file mode 100644 index 000000000..be9b6435c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_128_lock.html @@ -0,0 +1,382 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_alloc.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_alloc.html new file mode 100644 index 000000000..5a7a4366d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_alloc.html @@ -0,0 +1,466 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_cpu.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_cpu.html new file mode 100644 index 000000000..94d3c53f5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_cpu.html @@ -0,0 +1,1508 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_lock.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_lock.html new file mode 100644 index 000000000..e392ffeba --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/images/put_20s_90000_lock.html @@ -0,0 +1,325 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_10000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_10000.txt new file mode 100644 index 000000000..1d6050a17 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_10000.txt @@ -0,0 +1,85 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 1700.520ms, rate sampling interval: 5570ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 4.21s 741.30ms 5.49s 57.69% + Req/Sec 7.39k 0.00 7.39k 0.00% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 4.29s + 75.000% 4.85s + 90.000% 5.19s + 99.000% 5.46s + 99.900% 5.49s + 99.990% 5.49s + 99.999% 5.49s +100.000% 5.49s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 2949.119 0.000000 27 1.00 + 3174.399 0.100000 7468 1.11 + 3395.583 0.200000 14955 1.25 + 3674.111 0.300000 22366 1.43 + 3973.119 0.400000 29917 1.67 + 4288.511 0.500000 37314 2.00 + 4399.103 0.550000 41046 2.22 + 4505.599 0.600000 44862 2.50 + 4612.095 0.650000 48469 2.86 + 4751.359 0.700000 52174 3.33 + 4845.567 0.750000 56045 4.00 + 4890.623 0.775000 57855 4.44 + 4947.967 0.800000 59716 5.00 + 5005.311 0.825000 61581 5.71 + 5066.751 0.850000 63489 6.67 + 5115.903 0.875000 65285 8.00 + 5152.767 0.887500 66204 8.89 + 5185.535 0.900000 67148 10.00 + 5226.495 0.912500 68009 11.43 + 5259.263 0.925000 68949 13.33 + 5320.703 0.937500 69880 16.00 + 5345.279 0.943750 70397 17.78 + 5365.759 0.950000 70799 20.00 + 5386.239 0.956250 71391 22.86 + 5398.527 0.962500 71729 26.67 + 5410.815 0.968750 72297 32.00 + 5414.911 0.971875 72536 35.56 + 5423.103 0.975000 72723 40.00 + 5427.199 0.978125 72895 45.71 + 5435.391 0.981250 73155 53.33 + 5443.583 0.984375 73435 64.00 + 5447.679 0.985938 73569 71.11 + 5451.775 0.987500 73678 80.00 + 5455.871 0.989062 73804 91.43 + 5459.967 0.990625 73949 106.67 + 5459.967 0.992188 73949 128.00 + 5464.063 0.992969 74064 142.22 + 5464.063 0.993750 74064 160.00 + 5472.255 0.994531 74164 182.86 + 5476.351 0.995313 74213 213.33 + 5480.447 0.996094 74290 256.00 + 5480.447 0.996484 74290 284.44 + 5480.447 0.996875 74290 320.00 + 5484.543 0.997266 74365 365.71 + 5484.543 0.997656 74365 426.67 + 5488.639 0.998047 74486 512.00 + 5488.639 0.998242 74486 568.89 + 5488.639 0.998437 74486 640.00 + 5488.639 0.998633 74486 731.43 + 5488.639 0.998828 74486 853.33 + 5488.639 0.999023 74486 1024.00 + 5488.639 0.999121 74486 1137.78 + 5488.639 0.999219 74486 1280.00 + 5488.639 0.999316 74486 1462.86 + 5488.639 0.999414 74486 1706.67 + 5488.639 0.999512 74486 2048.00 + 5492.735 0.999561 74522 2275.56 + 5492.735 1.000000 74522 inf +#[Mean = 4207.386, StdDeviation = 741.300] +#[Max = 5488.640, Total count = 74522] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 145096 requests in 20.00s, 10.84MB read + Non-2xx or 3xx responses: 80540 +Requests/sec: 7254.78 +Transfer/sec: 555.04KB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_12000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_12000.txt new file mode 100644 index 000000000..56f3a0f3e --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_12000.txt @@ -0,0 +1,83 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 2172.972ms, rate sampling interval: 7495ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 6.11s 1.08s 8.11s 58.79% + Req/Sec 7.31k 0.00 7.31k 0.00% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 6.07s + 75.000% 7.07s + 90.000% 7.65s + 99.000% 8.05s + 99.900% 8.11s + 99.990% 8.11s + 99.999% 8.11s +100.000% 8.11s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 4159.487 0.000000 14 1.00 + 4694.015 0.100000 7318 1.11 + 5038.079 0.200000 14590 1.25 + 5369.855 0.300000 21844 1.43 + 5713.919 0.400000 29050 1.67 + 6074.367 0.500000 36344 2.00 + 6238.207 0.550000 39924 2.22 + 6397.951 0.600000 43555 2.50 + 6594.559 0.650000 47198 2.86 + 6848.511 0.700000 50827 3.33 + 7069.695 0.750000 54428 4.00 + 7159.807 0.775000 56285 4.44 + 7233.535 0.800000 58082 5.00 + 7327.743 0.825000 59913 5.71 + 7434.239 0.850000 61734 6.67 + 7544.831 0.875000 63518 8.00 + 7602.175 0.887500 64428 8.89 + 7647.231 0.900000 65324 10.00 + 7684.095 0.912500 66202 11.43 + 7737.343 0.925000 67108 13.33 + 7794.687 0.937500 68080 16.00 + 7815.167 0.943750 68512 17.78 + 7839.743 0.950000 68995 20.00 + 7872.511 0.956250 69420 22.86 + 7913.471 0.962500 69871 26.67 + 7938.047 0.968750 70295 32.00 + 7954.431 0.971875 70546 35.56 + 7966.719 0.975000 70738 40.00 + 7987.199 0.978125 71009 45.71 + 8003.583 0.981250 71205 53.33 + 8024.063 0.984375 71447 64.00 + 8032.255 0.985938 71590 71.11 + 8036.351 0.987500 71646 80.00 + 8044.543 0.989062 71755 91.43 + 8056.831 0.990625 71893 106.67 + 8069.119 0.992188 72026 128.00 + 8073.215 0.992969 72060 142.22 + 8077.311 0.993750 72096 160.00 + 8081.407 0.994531 72158 182.86 + 8085.503 0.995313 72213 213.33 + 8089.599 0.996094 72266 256.00 + 8093.695 0.996484 72313 284.44 + 8097.791 0.996875 72372 320.00 + 8097.791 0.997266 72372 365.71 + 8101.887 0.997656 72434 426.67 + 8101.887 0.998047 72434 512.00 + 8101.887 0.998242 72434 568.89 + 8105.983 0.998437 72502 640.00 + 8105.983 0.998633 72502 731.43 + 8105.983 0.998828 72502 853.33 + 8105.983 0.999023 72502 1024.00 + 8105.983 0.999121 72502 1137.78 + 8105.983 0.999219 72502 1280.00 + 8105.983 0.999316 72502 1462.86 + 8110.079 0.999414 72548 1706.67 + 8110.079 1.000000 72548 inf +#[Mean = 6110.005, StdDeviation = 1082.557] +#[Max = 8105.984, Total count = 72548] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 142690 requests in 20.00s, 10.67MB read + Non-2xx or 3xx responses: 78927 +Requests/sec: 7134.48 +Transfer/sec: 546.12KB diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_12000_128.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_12000_128.txt new file mode 100644 index 000000000..25da487cb --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_12000_128.txt @@ -0,0 +1,117 @@ +Running 20s test @ http://localhost:8080 + 4 threads and 128 connections + Thread calibration: mean lat.: 1.024ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.018ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.015ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.014ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.02ms 471.72us 3.59ms 63.68% + Req/Sec 3.16k 190.86 3.89k 73.93% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.00ms + 75.000% 1.37ms + 90.000% 1.66ms + 99.000% 2.08ms + 99.900% 2.36ms + 99.990% 2.64ms + 99.999% 3.55ms +100.000% 3.59ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.024 0.000000 1 1.00 + 0.405 0.100000 11845 1.11 + 0.575 0.200000 23626 1.25 + 0.722 0.300000 35481 1.43 + 0.862 0.400000 47273 1.67 + 1.002 0.500000 59056 2.00 + 1.072 0.550000 64961 2.22 + 1.145 0.600000 70853 2.50 + 1.220 0.650000 76823 2.86 + 1.295 0.700000 82715 3.33 + 1.374 0.750000 88618 4.00 + 1.416 0.775000 91551 4.44 + 1.458 0.800000 94485 5.00 + 1.505 0.825000 97470 5.71 + 1.552 0.850000 100391 6.67 + 1.603 0.875000 103350 8.00 + 1.631 0.887500 104822 8.89 + 1.658 0.900000 106266 10.00 + 1.691 0.912500 107756 11.43 + 1.727 0.925000 109254 13.33 + 1.766 0.937500 110696 16.00 + 1.788 0.943750 111449 17.78 + 1.813 0.950000 112185 20.00 + 1.840 0.956250 112929 22.86 + 1.868 0.962500 113656 26.67 + 1.902 0.968750 114400 32.00 + 1.920 0.971875 114764 35.56 + 1.938 0.975000 115125 40.00 + 1.960 0.978125 115490 45.71 + 1.984 0.981250 115858 53.33 + 2.013 0.984375 116232 64.00 + 2.027 0.985938 116417 71.11 + 2.042 0.987500 116601 80.00 + 2.063 0.989062 116784 91.43 + 2.087 0.990625 116972 106.67 + 2.113 0.992188 117156 128.00 + 2.127 0.992969 117248 142.22 + 2.141 0.993750 117345 160.00 + 2.159 0.994531 117430 182.86 + 2.181 0.995313 117519 213.33 + 2.207 0.996094 117613 256.00 + 2.221 0.996484 117663 284.44 + 2.233 0.996875 117704 320.00 + 2.251 0.997266 117752 365.71 + 2.271 0.997656 117800 426.67 + 2.291 0.998047 117845 512.00 + 2.303 0.998242 117865 568.89 + 2.313 0.998437 117888 640.00 + 2.333 0.998633 117911 731.43 + 2.347 0.998828 117934 853.33 + 2.365 0.999023 117958 1024.00 + 2.375 0.999121 117971 1137.78 + 2.383 0.999219 117979 1280.00 + 2.401 0.999316 117993 1462.86 + 2.417 0.999414 118002 1706.67 + 2.439 0.999512 118014 2048.00 + 2.443 0.999561 118021 2275.56 + 2.467 0.999609 118025 2560.00 + 2.483 0.999658 118031 2925.71 + 2.493 0.999707 118037 3413.33 + 2.509 0.999756 118043 4096.00 + 2.529 0.999780 118046 4551.11 + 2.535 0.999805 118048 5120.00 + 2.559 0.999829 118051 5851.43 + 2.575 0.999854 118054 6826.67 + 2.615 0.999878 118057 8192.00 + 2.641 0.999890 118059 9102.22 + 2.649 0.999902 118060 10240.00 + 2.669 0.999915 118061 11702.86 + 2.701 0.999927 118063 13653.33 + 2.765 0.999939 118064 16384.00 + 2.843 0.999945 118065 18204.44 + 2.851 0.999951 118066 20480.00 + 2.851 0.999957 118066 23405.71 + 2.887 0.999963 118067 27306.67 + 3.121 0.999969 118068 32768.00 + 3.121 0.999973 118068 36408.89 + 3.127 0.999976 118069 40960.00 + 3.127 0.999979 118069 46811.43 + 3.127 0.999982 118069 54613.33 + 3.553 0.999985 118070 65536.00 + 3.553 0.999986 118070 72817.78 + 3.553 0.999988 118070 81920.00 + 3.553 0.999989 118070 93622.86 + 3.553 0.999991 118070 109226.67 + 3.589 0.999992 118071 131072.00 + 3.589 1.000000 118071 inf +#[Mean = 1.022, StdDeviation = 0.472] +#[Max = 3.588, Total count = 118071] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 239114 requests in 20.00s, 17.87MB read + Non-2xx or 3xx responses: 132576 +Requests/sec: 11955.70 +Transfer/sec: 0.89MB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_20000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_20000.txt new file mode 100644 index 000000000..b6dfedc3b --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_20000.txt @@ -0,0 +1,88 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 3735.344ms, rate sampling interval: 13795ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 11.41s 2.14s 15.28s 58.99% + Req/Sec -nan -nan 0.00 0.00% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 11.43s + 75.000% 13.22s + 90.000% 14.39s + 99.000% 15.15s + 99.900% 15.27s + 99.990% 15.29s + 99.999% 15.29s +100.000% 15.29s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 7667.711 0.000000 10 1.00 + 8429.567 0.100000 4785 1.11 + 9232.383 0.200000 9540 1.25 + 9953.279 0.300000 14280 1.43 + 10682.367 0.400000 19079 1.67 + 11427.839 0.500000 23808 2.00 + 11771.903 0.550000 26181 2.22 + 12148.735 0.600000 28580 2.50 + 12550.143 0.650000 30931 2.86 + 12902.399 0.700000 33344 3.33 + 13221.887 0.750000 35692 4.00 + 13418.495 0.775000 36926 4.44 + 13590.527 0.800000 38110 5.00 + 13795.327 0.825000 39288 5.71 + 13975.551 0.850000 40461 6.67 + 14139.391 0.875000 41639 8.00 + 14229.503 0.887500 42237 8.89 + 14385.151 0.900000 42830 10.00 + 14499.839 0.912500 43447 11.43 + 14598.143 0.925000 44039 13.33 + 14696.447 0.937500 44617 16.00 + 14745.599 0.943750 44936 17.78 + 14794.751 0.950000 45262 20.00 + 14835.711 0.956250 45543 22.86 + 14876.671 0.962500 45844 26.67 + 14917.631 0.968750 46105 32.00 + 14942.207 0.971875 46286 35.56 + 14958.591 0.975000 46401 40.00 + 14991.359 0.978125 46578 45.71 + 15024.127 0.981250 46711 53.33 + 15065.087 0.984375 46846 64.00 + 15089.663 0.985938 46932 71.11 + 15114.239 0.987500 47013 80.00 + 15130.623 0.989062 47070 91.43 + 15155.199 0.990625 47142 106.67 + 15179.775 0.992188 47220 128.00 + 15196.159 0.992969 47272 142.22 + 15204.351 0.993750 47292 160.00 + 15220.735 0.994531 47350 182.86 + 15228.927 0.995313 47376 213.33 + 15237.119 0.996094 47401 256.00 + 15245.311 0.996484 47426 284.44 + 15253.503 0.996875 47462 320.00 + 15253.503 0.997266 47462 365.71 + 15261.695 0.997656 47495 426.67 + 15261.695 0.998047 47495 512.00 + 15269.887 0.998242 47540 568.89 + 15269.887 0.998437 47540 640.00 + 15269.887 0.998633 47540 731.43 + 15269.887 0.998828 47540 853.33 + 15269.887 0.999023 47540 1024.00 + 15278.079 0.999121 47572 1137.78 + 15278.079 0.999219 47572 1280.00 + 15278.079 0.999316 47572 1462.86 + 15278.079 0.999414 47572 1706.67 + 15278.079 0.999512 47572 2048.00 + 15278.079 0.999561 47572 2275.56 + 15278.079 0.999609 47572 2560.00 + 15278.079 0.999658 47572 2925.71 + 15286.271 0.999707 47586 3413.33 + 15286.271 1.000000 47586 inf +#[Mean = 11410.733, StdDeviation = 2143.921] +#[Max = 15278.080, Total count = 47586] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 94374 requests in 20.00s, 6.66MB read + Non-2xx or 3xx responses: 71842 +Requests/sec: 4718.63 +Transfer/sec: 341.06KB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_40000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_40000.txt new file mode 100644 index 000000000..0091ee4cf --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_40000.txt @@ -0,0 +1,79 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 4460.627ms, rate sampling interval: 15777ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 13.08s 2.52s 17.48s 57.73% + Req/Sec -nan -nan 0.00 0.00% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 13.01s + 75.000% 15.29s + 90.000% 16.58s + 99.000% 17.40s + 99.900% 17.50s + 99.990% 17.50s + 99.999% 17.50s +100.000% 17.50s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 8749.055 0.000000 21 1.00 + 9592.831 0.100000 5014 1.11 + 10510.335 0.200000 9985 1.25 + 11370.495 0.300000 14984 1.43 + 12165.119 0.400000 19979 1.67 + 13008.895 0.500000 24958 2.00 + 13443.071 0.550000 27447 2.22 + 13918.207 0.600000 29967 2.50 + 14360.575 0.650000 32442 2.86 + 14802.943 0.700000 34950 3.33 + 15286.271 0.750000 37438 4.00 + 15532.031 0.775000 38712 4.44 + 15728.639 0.800000 39932 5.00 + 15941.631 0.825000 41205 5.71 + 16162.815 0.850000 42454 6.67 + 16367.615 0.875000 43693 8.00 + 16465.919 0.887500 44287 8.89 + 16580.607 0.900000 44929 10.00 + 16695.295 0.912500 45542 11.43 + 16809.983 0.925000 46191 13.33 + 16924.671 0.937500 46826 16.00 + 16973.823 0.943750 47095 17.78 + 17039.359 0.950000 47466 20.00 + 17088.511 0.956250 47744 22.86 + 17154.047 0.962500 48077 26.67 + 17219.583 0.968750 48417 32.00 + 17235.967 0.971875 48519 35.56 + 17268.735 0.975000 48716 40.00 + 17301.503 0.978125 48878 45.71 + 17317.887 0.981250 48977 53.33 + 17350.655 0.984375 49163 64.00 + 17367.039 0.985938 49252 71.11 + 17383.423 0.987500 49343 80.00 + 17399.807 0.989062 49440 91.43 + 17399.807 0.990625 49440 106.67 + 17416.191 0.992188 49530 128.00 + 17432.575 0.992969 49612 142.22 + 17432.575 0.993750 49612 160.00 + 17448.959 0.994531 49696 182.86 + 17448.959 0.995313 49696 213.33 + 17465.343 0.996094 49773 256.00 + 17465.343 0.996484 49773 284.44 + 17465.343 0.996875 49773 320.00 + 17465.343 0.997266 49773 365.71 + 17481.727 0.997656 49846 426.67 + 17481.727 0.998047 49846 512.00 + 17481.727 0.998242 49846 568.89 + 17481.727 0.998437 49846 640.00 + 17481.727 0.998633 49846 731.43 + 17481.727 0.998828 49846 853.33 + 17498.111 0.999023 49899 1024.00 + 17498.111 1.000000 49899 inf +#[Mean = 13078.796, StdDeviation = 2515.303] +#[Max = 17481.728, Total count = 49899] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 100285 requests in 20.00s, 7.08MB read + Non-2xx or 3xx responses: 76401 +Requests/sec: 5014.18 +Transfer/sec: 362.36KB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_7000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_7000.txt new file mode 100644 index 000000000..23587a81e --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_7000.txt @@ -0,0 +1,110 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 3.627ms, rate sampling interval: 19ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 5.09ms 6.98ms 27.92ms 82.97% + Req/Sec 7.19k 573.76 9.84k 76.57% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.37ms + 75.000% 7.16ms + 90.000% 17.26ms + 99.000% 25.73ms + 99.900% 27.28ms + 99.990% 27.79ms + 99.999% 27.89ms +100.000% 27.93ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.015 0.000000 1 1.00 + 0.458 0.100000 6997 1.11 + 0.707 0.200000 14011 1.25 + 0.937 0.300000 20994 1.43 + 1.148 0.400000 28013 1.67 + 1.369 0.500000 34999 2.00 + 1.546 0.550000 38484 2.22 + 1.852 0.600000 41991 2.50 + 2.419 0.650000 45485 2.86 + 4.175 0.700000 48980 3.33 + 7.163 0.750000 52481 4.00 + 8.695 0.775000 54228 4.44 + 10.055 0.800000 55982 5.00 + 11.743 0.825000 57734 5.71 + 13.527 0.850000 59476 6.67 + 15.743 0.875000 61225 8.00 + 16.431 0.887500 62098 8.89 + 17.263 0.900000 62978 10.00 + 18.383 0.912500 63859 11.43 + 19.775 0.925000 64732 13.33 + 20.975 0.937500 65596 16.00 + 21.423 0.943750 66037 17.78 + 22.047 0.950000 66475 20.00 + 22.543 0.956250 66915 22.86 + 22.959 0.962500 67353 26.67 + 23.407 0.968750 67801 32.00 + 23.615 0.971875 68012 35.56 + 23.871 0.975000 68220 40.00 + 24.111 0.978125 68440 45.71 + 24.367 0.981250 68659 53.33 + 24.735 0.984375 68886 64.00 + 24.911 0.985938 68991 71.11 + 25.119 0.987500 69096 80.00 + 25.455 0.989062 69206 91.43 + 25.855 0.990625 69313 106.67 + 26.319 0.992188 69426 128.00 + 26.431 0.992969 69477 142.22 + 26.543 0.993750 69532 160.00 + 26.655 0.994531 69594 182.86 + 26.751 0.995313 69641 213.33 + 26.863 0.996094 69698 256.00 + 26.911 0.996484 69734 284.44 + 26.943 0.996875 69755 320.00 + 26.991 0.997266 69787 365.71 + 27.039 0.997656 69812 426.67 + 27.087 0.998047 69832 512.00 + 27.119 0.998242 69849 568.89 + 27.167 0.998437 69864 640.00 + 27.199 0.998633 69873 731.43 + 27.247 0.998828 69888 853.33 + 27.279 0.999023 69901 1024.00 + 27.295 0.999121 69907 1137.78 + 27.327 0.999219 69915 1280.00 + 27.343 0.999316 69921 1462.86 + 27.375 0.999414 69928 1706.67 + 27.407 0.999512 69934 2048.00 + 27.439 0.999561 69938 2275.56 + 27.455 0.999609 69941 2560.00 + 27.519 0.999658 69945 2925.71 + 27.615 0.999707 69949 3413.33 + 27.679 0.999756 69951 4096.00 + 27.711 0.999780 69953 4551.11 + 27.743 0.999805 69955 5120.00 + 27.759 0.999829 69958 5851.43 + 27.759 0.999854 69958 6826.67 + 27.791 0.999878 69961 8192.00 + 27.791 0.999890 69961 9102.22 + 27.807 0.999902 69962 10240.00 + 27.823 0.999915 69963 11702.86 + 27.823 0.999927 69963 13653.33 + 27.855 0.999939 69964 16384.00 + 27.871 0.999945 69966 18204.44 + 27.871 0.999951 69966 20480.00 + 27.871 0.999957 69966 23405.71 + 27.871 0.999963 69966 27306.67 + 27.871 0.999969 69966 32768.00 + 27.887 0.999973 69967 36408.89 + 27.887 0.999976 69967 40960.00 + 27.887 0.999979 69967 46811.43 + 27.887 0.999982 69967 54613.33 + 27.887 0.999985 69967 65536.00 + 27.935 0.999986 69968 72817.78 + 27.935 1.000000 69968 inf +#[Mean = 5.092, StdDeviation = 6.980] +#[Max = 27.920, Total count = 69968] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 139997 requests in 20.00s, 10.47MB read + Non-2xx or 3xx responses: 77259 +Requests/sec: 6999.82 +Transfer/sec: 536.00KB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_8000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_8000.txt new file mode 100644 index 000000000..5e6c778f2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/get_20s_8000.txt @@ -0,0 +1,87 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 488.396ms, rate sampling interval: 1342ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 879.90ms 155.57ms 1.14s 53.71% + Req/Sec 7.64k 239.88 7.99k 71.43% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 838.14ms + 75.000% 1.04s + 90.000% 1.12s + 99.000% 1.14s + 99.900% 1.14s + 99.990% 1.14s + 99.999% 1.14s +100.000% 1.14s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 696.319 0.000000 115 1.00 + 705.535 0.100000 8062 1.11 + 717.311 0.200000 15305 1.25 + 741.887 0.300000 23160 1.43 + 772.095 0.400000 30741 1.67 + 838.143 0.500000 38287 2.00 + 869.375 0.550000 42187 2.22 + 976.895 0.600000 45900 2.50 + 992.767 0.650000 49974 2.86 + 1001.471 0.700000 53621 3.33 + 1036.799 0.750000 57523 4.00 + 1044.479 0.775000 59316 4.44 + 1048.063 0.800000 61594 5.00 + 1049.599 0.825000 63278 5.71 + 1056.767 0.850000 65039 6.67 + 1088.511 0.875000 66938 8.00 + 1111.039 0.887500 67862 8.89 + 1119.231 0.900000 69310 10.00 + 1121.279 0.912500 70420 11.43 + 1122.303 0.925000 71220 13.33 + 1123.327 0.937500 71893 16.00 + 1125.375 0.943750 72680 17.78 + 1125.375 0.950000 72680 20.00 + 1127.423 0.956250 73396 22.86 + 1128.447 0.962500 73794 26.67 + 1129.471 0.968750 74117 32.00 + 1130.495 0.971875 74400 35.56 + 1134.591 0.975000 75016 40.00 + 1134.591 0.978125 75016 45.71 + 1135.615 0.981250 75958 53.33 + 1135.615 0.984375 75958 64.00 + 1135.615 0.985938 75958 71.11 + 1135.615 0.987500 75958 80.00 + 1135.615 0.989062 75958 91.43 + 1135.615 0.990625 75958 106.67 + 1135.615 0.992188 75958 128.00 + 1135.615 0.992969 75958 142.22 + 1136.639 0.993750 76429 160.00 + 1136.639 0.994531 76429 182.86 + 1136.639 0.995313 76429 213.33 + 1136.639 0.996094 76429 256.00 + 1136.639 0.996484 76429 284.44 + 1136.639 0.996875 76429 320.00 + 1136.639 0.997266 76429 365.71 + 1136.639 0.997656 76429 426.67 + 1136.639 0.998047 76429 512.00 + 1136.639 0.998242 76429 568.89 + 1136.639 0.998437 76429 640.00 + 1136.639 0.998633 76429 731.43 + 1136.639 0.998828 76429 853.33 + 1136.639 0.999023 76429 1024.00 + 1136.639 0.999121 76429 1137.78 + 1136.639 0.999219 76429 1280.00 + 1136.639 0.999316 76429 1462.86 + 1136.639 0.999414 76429 1706.67 + 1136.639 0.999512 76429 2048.00 + 1136.639 0.999561 76429 2275.56 + 1136.639 0.999609 76429 2560.00 + 1137.663 0.999658 76457 2925.71 + 1137.663 1.000000 76457 inf +#[Mean = 879.896, StdDeviation = 155.574] +#[Max = 1136.640, Total count = 76457] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 150913 requests in 20.00s, 11.28MB read + Non-2xx or 3xx responses: 83287 +Requests/sec: 7545.63 +Transfer/sec: 577.79KB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_30000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_30000.txt new file mode 100644 index 000000000..6382cb584 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_30000.txt @@ -0,0 +1,120 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 0.774ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 837.29us 0.86ms 9.13ms 93.23% + Req/Sec 31.59k 3.03k 55.22k 78.01% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 700.00us + 75.000% 1.03ms + 90.000% 1.22ms + 99.000% 5.06ms + 99.900% 7.73ms + 99.990% 8.91ms + 99.999% 9.13ms +100.000% 9.14ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.014 0.000000 15 1.00 + 0.160 0.100000 30127 1.11 + 0.298 0.200000 60008 1.25 + 0.433 0.300000 89978 1.43 + 0.567 0.400000 119976 1.67 + 0.700 0.500000 150012 2.00 + 0.767 0.550000 165083 2.22 + 0.833 0.600000 180020 2.50 + 0.899 0.650000 195099 2.86 + 0.964 0.700000 209970 3.33 + 1.028 0.750000 224929 4.00 + 1.059 0.775000 232563 4.44 + 1.089 0.800000 240099 5.00 + 1.117 0.825000 247468 5.71 + 1.142 0.850000 255077 6.67 + 1.167 0.875000 262584 8.00 + 1.184 0.887500 266270 8.89 + 1.220 0.900000 269901 10.00 + 1.360 0.912500 273613 11.43 + 1.555 0.925000 277357 13.33 + 1.799 0.937500 281107 16.00 + 1.962 0.943750 282972 17.78 + 2.143 0.950000 284844 20.00 + 2.385 0.956250 286725 22.86 + 2.745 0.962500 288599 26.67 + 3.079 0.968750 290474 32.00 + 3.235 0.971875 291412 35.56 + 3.383 0.975000 292342 40.00 + 3.577 0.978125 293276 45.71 + 3.831 0.981250 294214 53.33 + 4.151 0.984375 295153 64.00 + 4.359 0.985938 295624 71.11 + 4.579 0.987500 296087 80.00 + 4.883 0.989062 296558 91.43 + 5.199 0.990625 297024 106.67 + 5.451 0.992188 297498 128.00 + 5.579 0.992969 297728 142.22 + 5.759 0.993750 297964 160.00 + 6.023 0.994531 298197 182.86 + 6.251 0.995313 298433 213.33 + 6.407 0.996094 298668 256.00 + 6.495 0.996484 298787 284.44 + 6.571 0.996875 298899 320.00 + 6.763 0.997266 299016 365.71 + 6.975 0.997656 299133 426.67 + 7.147 0.998047 299252 512.00 + 7.199 0.998242 299309 568.89 + 7.311 0.998437 299368 640.00 + 7.407 0.998633 299427 731.43 + 7.607 0.998828 299483 853.33 + 7.763 0.999023 299542 1024.00 + 7.859 0.999121 299572 1137.78 + 7.931 0.999219 299601 1280.00 + 8.011 0.999316 299631 1462.86 + 8.095 0.999414 299659 1706.67 + 8.239 0.999512 299688 2048.00 + 8.311 0.999561 299703 2275.56 + 8.399 0.999609 299717 2560.00 + 8.479 0.999658 299735 2925.71 + 8.543 0.999707 299750 3413.33 + 8.599 0.999756 299761 4096.00 + 8.639 0.999780 299770 4551.11 + 8.679 0.999805 299777 5120.00 + 8.719 0.999829 299787 5851.43 + 8.743 0.999854 299791 6826.67 + 8.855 0.999878 299798 8192.00 + 8.895 0.999890 299802 9102.22 + 8.919 0.999902 299805 10240.00 + 8.991 0.999915 299810 11702.86 + 9.023 0.999927 299813 13653.33 + 9.063 0.999939 299816 16384.00 + 9.087 0.999945 299819 18204.44 + 9.103 0.999951 299823 20480.00 + 9.103 0.999957 299823 23405.71 + 9.111 0.999963 299826 27306.67 + 9.111 0.999969 299826 32768.00 + 9.111 0.999973 299826 36408.89 + 9.119 0.999976 299830 40960.00 + 9.119 0.999979 299830 46811.43 + 9.119 0.999982 299830 54613.33 + 9.119 0.999985 299830 65536.00 + 9.119 0.999986 299830 72817.78 + 9.127 0.999988 299833 81920.00 + 9.127 0.999989 299833 93622.86 + 9.127 0.999991 299833 109226.67 + 9.127 0.999992 299833 131072.00 + 9.127 0.999993 299833 145635.56 + 9.127 0.999994 299833 163840.00 + 9.127 0.999995 299833 187245.71 + 9.127 0.999995 299833 218453.33 + 9.127 0.999996 299833 262144.00 + 9.127 0.999997 299833 291271.11 + 9.135 0.999997 299834 327680.00 + 9.135 1.000000 299834 inf +#[Mean = 0.837, StdDeviation = 0.858] +#[Max = 9.128, Total count = 299834] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 599950 requests in 20.00s, 38.33MB read +Requests/sec: 29997.63 +Transfer/sec: 1.92MB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_40000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_40000.txt new file mode 100644 index 000000000..e69c7600d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_40000.txt @@ -0,0 +1,122 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 1.748ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.10ms 1.68ms 15.02ms 92.92% + Req/Sec 42.14k 4.22k 62.78k 75.05% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 720.00us + 75.000% 1.06ms + 90.000% 1.77ms + 99.000% 9.68ms + 99.900% 13.81ms + 99.990% 14.96ms + 99.999% 15.01ms +100.000% 15.03ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.012 0.000000 11 1.00 + 0.162 0.100000 40213 1.11 + 0.303 0.200000 79975 1.25 + 0.443 0.300000 120045 1.43 + 0.582 0.400000 160005 1.67 + 0.720 0.500000 200174 2.00 + 0.788 0.550000 220080 2.22 + 0.856 0.600000 239936 2.50 + 0.925 0.650000 260198 2.86 + 0.992 0.700000 280126 3.33 + 1.059 0.750000 300152 4.00 + 1.091 0.775000 309960 4.44 + 1.124 0.800000 320051 5.00 + 1.155 0.825000 330000 5.71 + 1.192 0.850000 339926 6.67 + 1.328 0.875000 349837 8.00 + 1.525 0.887500 354853 8.89 + 1.766 0.900000 359843 10.00 + 2.097 0.912500 364853 11.43 + 2.579 0.925000 369847 13.33 + 3.217 0.937500 374829 16.00 + 3.631 0.943750 377324 17.78 + 4.315 0.950000 379830 20.00 + 4.987 0.956250 382327 22.86 + 5.603 0.962500 384831 26.67 + 6.231 0.968750 387324 32.00 + 6.555 0.971875 388581 35.56 + 6.847 0.975000 389830 40.00 + 7.335 0.978125 391068 45.71 + 8.039 0.981250 392316 53.33 + 8.623 0.984375 393573 64.00 + 8.975 0.985938 394197 71.11 + 9.191 0.987500 394830 80.00 + 9.423 0.989062 395447 91.43 + 9.831 0.990625 396074 106.67 + 10.319 0.992188 396698 128.00 + 10.559 0.992969 397002 142.22 + 10.767 0.993750 397320 160.00 + 10.927 0.994531 397643 182.86 + 11.127 0.995313 397953 213.33 + 11.303 0.996094 398265 256.00 + 11.391 0.996484 398409 284.44 + 11.487 0.996875 398581 320.00 + 11.591 0.997266 398725 365.71 + 11.759 0.997656 398875 426.67 + 12.191 0.998047 399033 512.00 + 12.591 0.998242 399111 568.89 + 12.967 0.998437 399189 640.00 + 13.303 0.998633 399268 731.43 + 13.567 0.998828 399344 853.33 + 13.839 0.999023 399423 1024.00 + 14.007 0.999121 399463 1137.78 + 14.135 0.999219 399501 1280.00 + 14.319 0.999316 399540 1462.86 + 14.455 0.999414 399581 1706.67 + 14.575 0.999512 399620 2048.00 + 14.631 0.999561 399637 2275.56 + 14.727 0.999609 399659 2560.00 + 14.783 0.999658 399683 2925.71 + 14.839 0.999707 399696 3413.33 + 14.903 0.999756 399718 4096.00 + 14.919 0.999780 399729 4551.11 + 14.927 0.999805 399735 5120.00 + 14.943 0.999829 399749 5851.43 + 14.951 0.999854 399766 6826.67 + 14.951 0.999878 399766 8192.00 + 14.959 0.999890 399775 9102.22 + 14.959 0.999902 399775 10240.00 + 14.967 0.999915 399787 11702.86 + 14.967 0.999927 399787 13653.33 + 14.975 0.999939 399794 16384.00 + 14.975 0.999945 399794 18204.44 + 14.975 0.999951 399794 20480.00 + 14.983 0.999957 399797 23405.71 + 14.991 0.999963 399802 27306.67 + 14.991 0.999969 399802 32768.00 + 14.991 0.999973 399802 36408.89 + 14.999 0.999976 399807 40960.00 + 14.999 0.999979 399807 46811.43 + 14.999 0.999982 399807 54613.33 + 14.999 0.999985 399807 65536.00 + 14.999 0.999986 399807 72817.78 + 15.007 0.999988 399808 81920.00 + 15.007 0.999989 399808 93622.86 + 15.015 0.999991 399811 109226.67 + 15.015 0.999992 399811 131072.00 + 15.015 0.999993 399811 145635.56 + 15.015 0.999994 399811 163840.00 + 15.015 0.999995 399811 187245.71 + 15.015 0.999995 399811 218453.33 + 15.015 0.999996 399811 262144.00 + 15.015 0.999997 399811 291271.11 + 15.015 0.999997 399811 327680.00 + 15.015 0.999997 399811 374491.43 + 15.031 0.999998 399812 436906.67 + 15.031 1.000000 399812 inf +#[Mean = 1.103, StdDeviation = 1.681] +#[Max = 15.024, Total count = 399812] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 799951 requests in 20.00s, 51.11MB read +Requests/sec: 39997.06 +Transfer/sec: 2.56MB diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_50000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_50000.txt new file mode 100644 index 000000000..a60f64827 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_50000.txt @@ -0,0 +1,118 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 1.328ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.37ms 2.12ms 18.67ms 90.45% + Req/Sec 52.56k 5.79k 72.11k 77.56% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 770.00us + 75.000% 1.13ms + 90.000% 3.28ms + 99.000% 11.29ms + 99.900% 17.12ms + 99.990% 18.54ms + 99.999% 18.67ms +100.000% 18.69ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.012 0.000000 26 1.00 + 0.174 0.100000 50038 1.11 + 0.326 0.200000 100218 1.25 + 0.474 0.300000 150274 1.43 + 0.622 0.400000 200297 1.67 + 0.770 0.500000 250096 2.00 + 0.845 0.550000 275236 2.22 + 0.918 0.600000 300408 2.50 + 0.989 0.650000 325331 2.86 + 1.062 0.700000 350119 3.33 + 1.135 0.750000 375282 4.00 + 1.173 0.775000 387798 4.44 + 1.216 0.800000 400223 5.00 + 1.356 0.825000 412629 5.71 + 1.832 0.850000 425139 6.67 + 2.423 0.875000 437656 8.00 + 2.809 0.887500 443916 8.89 + 3.283 0.900000 450162 10.00 + 3.811 0.912500 456414 11.43 + 4.343 0.925000 462699 13.33 + 4.963 0.937500 468905 16.00 + 5.351 0.943750 472033 17.78 + 5.787 0.950000 475155 20.00 + 6.371 0.956250 478297 22.86 + 6.939 0.962500 481407 26.67 + 7.547 0.968750 484529 32.00 + 7.943 0.971875 486090 35.56 + 8.375 0.975000 487659 40.00 + 8.855 0.978125 489214 45.71 + 9.503 0.981250 490800 53.33 + 10.047 0.984375 492342 64.00 + 10.399 0.985938 493144 71.11 + 10.735 0.987500 493908 80.00 + 11.095 0.989062 494685 91.43 + 11.391 0.990625 495477 106.67 + 11.751 0.992188 496248 128.00 + 12.055 0.992969 496659 142.22 + 12.335 0.993750 497033 160.00 + 12.631 0.994531 497429 182.86 + 12.999 0.995313 497814 213.33 + 13.495 0.996094 498205 256.00 + 13.743 0.996484 498398 284.44 + 14.063 0.996875 498600 320.00 + 14.303 0.997266 498789 365.71 + 15.015 0.997656 498984 426.67 + 15.567 0.998047 499178 512.00 + 15.863 0.998242 499275 568.89 + 16.223 0.998437 499373 640.00 + 16.559 0.998633 499473 731.43 + 16.879 0.998828 499575 853.33 + 17.151 0.999023 499671 1024.00 + 17.311 0.999121 499717 1137.78 + 17.487 0.999219 499766 1280.00 + 17.647 0.999316 499816 1462.86 + 17.823 0.999414 499865 1706.67 + 17.983 0.999512 499915 2048.00 + 18.047 0.999561 499940 2275.56 + 18.095 0.999609 499961 2560.00 + 18.159 0.999658 499985 2925.71 + 18.239 0.999707 500010 3413.33 + 18.303 0.999756 500037 4096.00 + 18.335 0.999780 500046 4551.11 + 18.383 0.999805 500057 5120.00 + 18.447 0.999829 500072 5851.43 + 18.495 0.999854 500087 6826.67 + 18.511 0.999878 500097 8192.00 + 18.527 0.999890 500103 9102.22 + 18.559 0.999902 500111 10240.00 + 18.575 0.999915 500113 11702.86 + 18.607 0.999927 500119 13653.33 + 18.623 0.999939 500125 16384.00 + 18.639 0.999945 500131 18204.44 + 18.639 0.999951 500131 20480.00 + 18.655 0.999957 500143 23405.71 + 18.655 0.999963 500143 27306.67 + 18.655 0.999969 500143 32768.00 + 18.655 0.999973 500143 36408.89 + 18.655 0.999976 500143 40960.00 + 18.671 0.999979 500152 46811.43 + 18.671 0.999982 500152 54613.33 + 18.671 0.999985 500152 65536.00 + 18.671 0.999986 500152 72817.78 + 18.671 0.999988 500152 81920.00 + 18.671 0.999989 500152 93622.86 + 18.671 0.999991 500152 109226.67 + 18.671 0.999992 500152 131072.00 + 18.671 0.999993 500152 145635.56 + 18.671 0.999994 500152 163840.00 + 18.671 0.999995 500152 187245.71 + 18.671 0.999995 500152 218453.33 + 18.687 0.999996 500154 262144.00 + 18.687 1.000000 500154 inf +#[Mean = 1.367, StdDeviation = 2.118] +#[Max = 18.672, Total count = 500154] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 999935 requests in 20.00s, 63.89MB read +Requests/sec: 49996.65 +Transfer/sec: 3.19MB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_60000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_60000.txt new file mode 100644 index 000000000..ecb81f0ce --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_60000.txt @@ -0,0 +1,110 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 128.281ms, rate sampling interval: 384ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 23.31ms 34.48ms 135.42ms 88.28% + Req/Sec 60.80k 2.98k 68.21k 80.77% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 4.24ms + 75.000% 36.26ms + 90.000% 78.59ms + 99.000% 131.33ms + 99.900% 134.91ms + 99.990% 135.42ms + 99.999% 135.55ms +100.000% 135.55ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.012 0.000000 24 1.00 + 0.390 0.100000 60772 1.11 + 0.736 0.200000 121435 1.25 + 1.026 0.300000 182077 1.43 + 2.023 0.400000 242731 1.67 + 4.239 0.500000 303396 2.00 + 6.435 0.550000 333721 2.22 + 11.287 0.600000 364057 2.50 + 19.759 0.650000 394444 2.86 + 28.863 0.700000 424779 3.33 + 36.255 0.750000 455322 4.00 + 38.015 0.775000 470233 4.44 + 42.207 0.800000 485464 5.00 + 46.015 0.825000 500619 5.71 + 51.519 0.850000 515820 6.67 + 55.711 0.875000 530980 8.00 + 62.143 0.887500 538488 8.89 + 78.591 0.900000 546085 10.00 + 93.375 0.912500 553690 11.43 + 104.895 0.925000 561261 13.33 + 108.415 0.937500 568841 16.00 + 109.951 0.943750 572622 17.78 + 111.423 0.950000 576502 20.00 + 113.727 0.956250 580298 22.86 + 116.223 0.962500 584007 26.67 + 120.191 0.968750 587830 32.00 + 121.983 0.971875 589707 35.56 + 123.839 0.975000 591679 40.00 + 124.671 0.978125 593641 45.71 + 126.527 0.981250 595432 53.33 + 128.383 0.984375 597285 64.00 + 129.023 0.985938 598238 71.11 + 129.983 0.987500 599237 80.00 + 130.751 0.989062 600149 91.43 + 131.711 0.990625 601165 106.67 + 132.607 0.992188 602085 128.00 + 132.735 0.992969 602862 142.22 + 132.863 0.993750 603065 160.00 + 133.247 0.994531 603521 182.86 + 133.631 0.995313 603995 213.33 + 134.015 0.996094 604716 256.00 + 134.015 0.996484 604716 284.44 + 134.143 0.996875 605204 320.00 + 134.143 0.997266 605204 365.71 + 134.271 0.997656 605364 426.67 + 134.527 0.998047 605622 512.00 + 134.655 0.998242 605870 568.89 + 134.655 0.998437 605870 640.00 + 134.783 0.998633 606002 731.43 + 134.911 0.998828 606170 853.33 + 134.911 0.999023 606170 1024.00 + 135.039 0.999121 606261 1137.78 + 135.167 0.999219 606371 1280.00 + 135.167 0.999316 606371 1462.86 + 135.295 0.999414 606562 1706.67 + 135.295 0.999512 606562 2048.00 + 135.295 0.999561 606562 2275.56 + 135.295 0.999609 606562 2560.00 + 135.295 0.999658 606562 2925.71 + 135.423 0.999707 606739 3413.33 + 135.423 0.999756 606739 4096.00 + 135.423 0.999780 606739 4551.11 + 135.423 0.999805 606739 5120.00 + 135.423 0.999829 606739 5851.43 + 135.423 0.999854 606739 6826.67 + 135.423 0.999878 606739 8192.00 + 135.423 0.999890 606739 9102.22 + 135.423 0.999902 606739 10240.00 + 135.423 0.999915 606739 11702.86 + 135.423 0.999927 606739 13653.33 + 135.423 0.999939 606739 16384.00 + 135.423 0.999945 606739 18204.44 + 135.423 0.999951 606739 20480.00 + 135.423 0.999957 606739 23405.71 + 135.423 0.999963 606739 27306.67 + 135.423 0.999969 606739 32768.00 + 135.423 0.999973 606739 36408.89 + 135.423 0.999976 606739 40960.00 + 135.423 0.999979 606739 46811.43 + 135.423 0.999982 606739 54613.33 + 135.423 0.999985 606739 65536.00 + 135.423 0.999986 606739 72817.78 + 135.551 0.999988 606747 81920.00 + 135.551 1.000000 606747 inf +#[Mean = 23.315, StdDeviation = 34.482] +#[Max = 135.424, Total count = 606747] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1198914 requests in 20.00s, 76.61MB read +Requests/sec: 59945.64 +Transfer/sec: 3.83MB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_65000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_65000.txt new file mode 100644 index 000000000..9c9bac421 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_65000.txt @@ -0,0 +1,100 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 644.784ms, rate sampling interval: 2820ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.85s 249.30ms 2.35s 61.29% + Req/Sec 59.43k 1.26k 60.91k 33.33% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.83s + 75.000% 2.03s + 90.000% 2.25s + 99.000% 2.33s + 99.900% 2.35s + 99.990% 2.35s + 99.999% 2.35s +100.000% 2.35s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 1464.319 0.000000 7 1.00 + 1530.879 0.100000 60687 1.11 + 1609.727 0.200000 118595 1.25 + 1653.759 0.300000 177805 1.43 + 1773.567 0.400000 237665 1.67 + 1831.935 0.500000 297856 2.00 + 1857.535 0.550000 325885 2.22 + 1902.591 0.600000 355303 2.50 + 1954.815 0.650000 385445 2.86 + 1987.583 0.700000 415750 3.33 + 2027.519 0.750000 446234 4.00 + 2069.503 0.775000 459103 4.44 + 2103.295 0.800000 474318 5.00 + 2154.495 0.825000 489092 5.71 + 2193.407 0.850000 503359 6.67 + 2226.175 0.875000 518340 8.00 + 2236.415 0.887500 525559 8.89 + 2250.751 0.900000 534161 10.00 + 2252.799 0.912500 543091 11.43 + 2260.991 0.925000 552740 13.33 + 2267.135 0.937500 556181 16.00 + 2269.183 0.943750 559493 17.78 + 2275.327 0.950000 563171 20.00 + 2285.567 0.956250 566971 22.86 + 2289.663 0.962500 572298 26.67 + 2293.759 0.968750 573915 32.00 + 2301.951 0.971875 576389 35.56 + 2306.047 0.975000 578083 40.00 + 2310.143 0.978125 579442 45.71 + 2318.335 0.981250 581574 53.33 + 2322.431 0.984375 583499 64.00 + 2324.479 0.985938 584241 71.11 + 2326.527 0.987500 585067 80.00 + 2328.575 0.989062 585964 91.43 + 2332.671 0.990625 587008 106.67 + 2334.719 0.992188 587643 128.00 + 2336.767 0.992969 588127 142.22 + 2338.815 0.993750 588775 160.00 + 2340.863 0.994531 589429 182.86 + 2340.863 0.995313 589429 213.33 + 2342.911 0.996094 590001 256.00 + 2344.959 0.996484 590644 284.44 + 2344.959 0.996875 590644 320.00 + 2344.959 0.997266 590644 365.71 + 2347.007 0.997656 591338 426.67 + 2347.007 0.998047 591338 512.00 + 2347.007 0.998242 591338 568.89 + 2347.007 0.998437 591338 640.00 + 2347.007 0.998633 591338 731.43 + 2349.055 0.998828 592118 853.33 + 2349.055 0.999023 592118 1024.00 + 2349.055 0.999121 592118 1137.78 + 2349.055 0.999219 592118 1280.00 + 2349.055 0.999316 592118 1462.86 + 2349.055 0.999414 592118 1706.67 + 2349.055 0.999512 592118 2048.00 + 2349.055 0.999561 592118 2275.56 + 2349.055 0.999609 592118 2560.00 + 2349.055 0.999658 592118 2925.71 + 2349.055 0.999707 592118 3413.33 + 2349.055 0.999756 592118 4096.00 + 2349.055 0.999780 592118 4551.11 + 2349.055 0.999805 592118 5120.00 + 2349.055 0.999829 592118 5851.43 + 2349.055 0.999854 592118 6826.67 + 2349.055 0.999878 592118 8192.00 + 2349.055 0.999890 592118 9102.22 + 2349.055 0.999902 592118 10240.00 + 2349.055 0.999915 592118 11702.86 + 2349.055 0.999927 592118 13653.33 + 2349.055 0.999939 592118 16384.00 + 2349.055 0.999945 592118 18204.44 + 2351.103 0.999951 592147 20480.00 + 2351.103 1.000000 592147 inf +#[Mean = 1852.702, StdDeviation = 249.302] +#[Max = 2349.056, Total count = 592147] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1147295 requests in 20.00s, 73.31MB read +Requests/sec: 57364.71 +Transfer/sec: 3.67MB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_90000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_90000.txt new file mode 100644 index 000000000..3f0dfe37a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_90000.txt @@ -0,0 +1,90 @@ +Running 20s test @ http://localhost:8080 + 1 threads and 1 connections + Thread calibration: mean lat.: 1792.257ms, rate sampling interval: 6430ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 5.13s 984.33ms 6.93s 58.60% + Req/Sec 60.78k 0.00 60.78k 0.00% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 5.03s + 75.000% 5.95s + 90.000% 6.55s + 99.000% 6.91s + 99.900% 6.93s + 99.990% 6.94s + 99.999% 6.94s +100.000% 6.94s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 3540.991 0.000000 410 1.00 + 3831.807 0.100000 59733 1.11 + 4136.959 0.200000 118802 1.25 + 4435.967 0.300000 179015 1.43 + 4702.207 0.400000 238300 1.67 + 5033.983 0.500000 297161 2.00 + 5206.015 0.550000 326786 2.22 + 5410.815 0.600000 356941 2.50 + 5582.847 0.650000 386516 2.86 + 5783.551 0.700000 416365 3.33 + 5947.391 0.750000 445378 4.00 + 6057.983 0.775000 460398 4.44 + 6172.671 0.800000 475844 5.00 + 6287.359 0.825000 490114 5.71 + 6410.239 0.850000 504997 6.67 + 6492.159 0.875000 520187 8.00 + 6524.927 0.887500 528069 8.89 + 6553.599 0.900000 534489 10.00 + 6602.751 0.912500 542323 11.43 + 6639.615 0.925000 549664 13.33 + 6705.151 0.937500 557309 16.00 + 6721.535 0.943750 560812 17.78 + 6746.111 0.950000 564166 20.00 + 6770.687 0.956250 568099 22.86 + 6799.359 0.962500 571930 26.67 + 6819.839 0.968750 575559 32.00 + 6836.223 0.971875 577524 35.56 + 6852.607 0.975000 579302 40.00 + 6864.895 0.978125 580866 45.71 + 6881.279 0.981250 583235 53.33 + 6889.471 0.984375 584643 64.00 + 6893.567 0.985938 585486 71.11 + 6897.663 0.987500 586432 80.00 + 6901.759 0.989062 587386 91.43 + 6905.855 0.990625 588277 106.67 + 6909.951 0.992188 589209 128.00 + 6914.047 0.992969 589893 142.22 + 6918.143 0.993750 590660 160.00 + 6918.143 0.994531 590660 182.86 + 6922.239 0.995313 591275 213.33 + 6926.335 0.996094 592064 256.00 + 6926.335 0.996484 592064 284.44 + 6926.335 0.996875 592064 320.00 + 6930.431 0.997266 592974 365.71 + 6930.431 0.997656 592974 426.67 + 6930.431 0.998047 592974 512.00 + 6930.431 0.998242 592974 568.89 + 6930.431 0.998437 592974 640.00 + 6934.527 0.998633 593701 731.43 + 6934.527 0.998828 593701 853.33 + 6934.527 0.999023 593701 1024.00 + 6934.527 0.999121 593701 1137.78 + 6934.527 0.999219 593701 1280.00 + 6934.527 0.999316 593701 1462.86 + 6934.527 0.999414 593701 1706.67 + 6934.527 0.999512 593701 2048.00 + 6934.527 0.999561 593701 2275.56 + 6934.527 0.999609 593701 2560.00 + 6934.527 0.999658 593701 2925.71 + 6934.527 0.999707 593701 3413.33 + 6934.527 0.999756 593701 4096.00 + 6934.527 0.999780 593701 4551.11 + 6938.623 0.999805 593828 5120.00 + 6938.623 1.000000 593828 inf +#[Mean = 5127.906, StdDeviation = 984.328] +#[Max = 6934.528, Total count = 593828] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1175759 requests in 20.00s, 75.13MB read +Requests/sec: 58788.16 +Transfer/sec: 3.76MB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_90000_128.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_90000_128.txt new file mode 100644 index 000000000..032d860dd --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/outputs/put_20s_90000_128.txt @@ -0,0 +1,131 @@ +Running 20s test @ http://localhost:8080 + 4 threads and 128 connections + Thread calibration: mean lat.: 0.901ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.905ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.904ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.910ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.91ms 431.73us 3.37ms 62.69% + Req/Sec 23.88k 1.05k 28.89k 73.31% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 0.89ms + 75.000% 1.24ms + 90.000% 1.49ms + 99.000% 1.79ms + 99.900% 2.12ms + 99.990% 2.73ms + 99.999% 3.12ms +100.000% 3.37ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.013 0.000000 1 1.00 + 0.348 0.100000 88983 1.11 + 0.488 0.200000 177501 1.25 + 0.626 0.300000 266343 1.43 + 0.760 0.400000 354293 1.67 + 0.895 0.500000 442831 2.00 + 0.964 0.550000 487644 2.22 + 1.032 0.600000 531533 2.50 + 1.102 0.650000 576211 2.86 + 1.171 0.700000 620333 3.33 + 1.240 0.750000 664445 4.00 + 1.276 0.775000 686840 4.44 + 1.313 0.800000 708921 5.00 + 1.353 0.825000 731082 5.71 + 1.396 0.850000 752901 6.67 + 1.444 0.875000 775257 8.00 + 1.468 0.887500 786004 8.89 + 1.494 0.900000 797234 10.00 + 1.521 0.912500 808393 11.43 + 1.549 0.925000 819521 13.33 + 1.578 0.937500 830370 16.00 + 1.594 0.943750 835933 17.78 + 1.611 0.950000 841550 20.00 + 1.629 0.956250 847068 22.86 + 1.648 0.962500 852409 26.67 + 1.670 0.968750 858053 32.00 + 1.682 0.971875 860766 35.56 + 1.695 0.975000 863511 40.00 + 1.709 0.978125 866240 45.71 + 1.725 0.981250 869001 53.33 + 1.743 0.984375 871785 64.00 + 1.754 0.985938 873215 71.11 + 1.766 0.987500 874614 80.00 + 1.779 0.989062 875955 91.43 + 1.795 0.990625 877326 106.67 + 1.814 0.992188 878681 128.00 + 1.825 0.992969 879368 142.22 + 1.838 0.993750 880051 160.00 + 1.854 0.994531 880757 182.86 + 1.872 0.995313 881442 213.33 + 1.893 0.996094 882127 256.00 + 1.907 0.996484 882484 284.44 + 1.923 0.996875 882820 320.00 + 1.942 0.997266 883162 365.71 + 1.964 0.997656 883507 426.67 + 1.994 0.998047 883854 512.00 + 2.012 0.998242 884024 568.89 + 2.029 0.998437 884199 640.00 + 2.053 0.998633 884372 731.43 + 2.087 0.998828 884542 853.33 + 2.129 0.999023 884720 1024.00 + 2.155 0.999121 884803 1137.78 + 2.197 0.999219 884892 1280.00 + 2.233 0.999316 884975 1462.86 + 2.289 0.999414 885063 1706.67 + 2.349 0.999512 885148 2048.00 + 2.383 0.999561 885192 2275.56 + 2.417 0.999609 885237 2560.00 + 2.449 0.999658 885277 2925.71 + 2.481 0.999707 885321 3413.33 + 2.539 0.999756 885364 4096.00 + 2.561 0.999780 885385 4551.11 + 2.581 0.999805 885407 5120.00 + 2.601 0.999829 885428 5851.43 + 2.631 0.999854 885451 6826.67 + 2.685 0.999878 885471 8192.00 + 2.705 0.999890 885482 9102.22 + 2.735 0.999902 885494 10240.00 + 2.775 0.999915 885504 11702.86 + 2.817 0.999927 885515 13653.33 + 2.857 0.999939 885527 16384.00 + 2.913 0.999945 885531 18204.44 + 2.923 0.999951 885537 20480.00 + 2.937 0.999957 885542 23405.71 + 2.973 0.999963 885548 27306.67 + 2.989 0.999969 885552 32768.00 + 2.997 0.999973 885557 36408.89 + 3.009 0.999976 885558 40960.00 + 3.023 0.999979 885561 46811.43 + 3.029 0.999982 885563 54613.33 + 3.039 0.999985 885566 65536.00 + 3.043 0.999986 885567 72817.78 + 3.115 0.999988 885569 81920.00 + 3.119 0.999989 885570 93622.86 + 3.153 0.999991 885571 109226.67 + 3.217 0.999992 885573 131072.00 + 3.217 0.999993 885573 145635.56 + 3.225 0.999994 885574 163840.00 + 3.245 0.999995 885575 187245.71 + 3.245 0.999995 885575 218453.33 + 3.249 0.999996 885576 262144.00 + 3.249 0.999997 885576 291271.11 + 3.259 0.999997 885577 327680.00 + 3.259 0.999997 885577 374491.43 + 3.259 0.999998 885577 436906.67 + 3.329 0.999998 885578 524288.00 + 3.329 0.999998 885578 582542.22 + 3.329 0.999998 885578 655360.00 + 3.329 0.999999 885578 748982.86 + 3.329 0.999999 885578 873813.33 + 3.367 0.999999 885579 1048576.00 + 3.367 1.000000 885579 inf +#[Mean = 0.906, StdDeviation = 0.432] +#[Max = 3.366, Total count = 885579] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1793023 requests in 20.00s, 114.57MB read +Requests/sec: 89649.43 +Transfer/sec: 5.73MB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/report.md b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/report.md new file mode 100644 index 000000000..482a7e6b3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/report.md @@ -0,0 +1,65 @@ +# Таразанов Максим, ИТМО, М4139, Stage 2 + +*для начала хочу извиниться, что накосячил с веткой первого +стейджа и не отправил (как выяснилось), я её нормально сделаю чуть позже, пусть это не в счет баллов пойдёт. +Сравнение проводилось на другом железе (не на WSL, а на Ubuntu. Поэтому из-за смены ноута чуть позже 1й отчет загружу).* + +## Executors +* **corePoolSize** выбрано так, чтобы брать примерно половину доступных серверных ядер. +Данный подход нужен для компромиса между высокой параллелизацией и +избеганием лишнего контекстного переключения между потоками. +Плюсом остаются ресурсы на *некоторые другие задачи у сервера*; + обозначает максимальное количество потоков в пуле, чтобы не создавать лишних; +* **keepAliveTime** = 0, чтобы уменьшить ресурсоёмкость при низкой нагрузке; +* **queueCapacity** кажется, что 100 нормально для того чтобы нормально работать и не реджектить лишнего; +* **AbortPolicy** -- реджект при переполнении. + +## GET +Прошлая реализация держалась на [12000rps](..%2Fstage1%2Foutputs%2Fget_20s_12000.txt): +* передача **0.88МВ/с**; +* ср.латенси **2.455**. + +### -t 1 -c 1 +Новая реализация показывает ужасный результат на [12000rps](outputs%2Fget_20s_12000.txt): +* передача **0.56МВ/с**; +* латенси **!6110!**. + +Может жить на [7000rps](outputs%2Fget_20s_7000.txt). + +Одиночные запросы обрабатываются очень плохо :( + +### -t 4 -c 128 +[Значительно лучше](outputs%2Fget_20s_12000_128.txt) +* передача **0.89МВ/с** +* латенси **1.022** (меньше в 2.5 раза) + +Параллельные запросы в совокупности улучшили работу сервера (: + +## PUT +Прошлая реализация [90000rps](..%2Fstage1%2Foutputs%2Fput_20s_90000.txt): +* передача **5.75МВ/с**; +* латенси **52.58**. + +### -t 1 -c 1 +[Новый вариант](outputs%2Fput_20s_90000.txt) показал себя хуже (а вообще может жить на нагрузке в [1.5 раза меньше](outputs%2Fput_20s_60000.txt)): +* передача **3.76MB/с**; +* латенси **5127.906**. + +### -t 4 -c 128 +[Параллельные запросы](outputs%2Fput_20s_90000_128.txt) обрабатываются значительно лучше: +* передача **5.73MB/с**; +* латенси **0.906** (значительно лучше). + +В разы лучше! + +## Asprof +### CPU +Параллельные запросы позволяют серверу нагружаться в основном на работу с базой +Разгруженность сокетов улучшает работу + +### Alloc +Появляются дополнительные затраты памяти на локи + +### Lock +Чем больше воркеров - тем больше локов на их синхронизацию, +что не очень хорошо и надо поменять очередь на локфри (например flatcombining), чтобы не сильно терять время на синхронизацию \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/scripts/get.lua b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/scripts/get.lua new file mode 100644 index 000000000..af0501626 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/scripts/get.lua @@ -0,0 +1,9 @@ +math.randomseed(os.time()) + +function request() + counter = math.random(100000100, 100000000 + 5000000) + headers = {} + headers["Host"] = "localhost:8080" + return wrk.format("GET", "/v0/entity?id=" .. tostring(counter), headers) + +end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/scripts/put.lua b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/scripts/put.lua new file mode 100644 index 000000000..90ba7a1a7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage2/scripts/put.lua @@ -0,0 +1,12 @@ +counter = 100000000 + +function request() + counter = counter + 1 + body = tostring(counter) .. tostring(counter) .. tostring(counter) + headers = {} + headers["Content-Type"] = "text/plain" + headers["Content-Length"] = #{string.byte(body, 1, -1)} + headers["Host"] = "localhost:8080" + return wrk.format("PUT", "/v0/entity?id=" .. tostring(counter), headers, body) + +end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_alloc.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_alloc.html new file mode 100644 index 000000000..a8d3b1c55 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_alloc.html @@ -0,0 +1,747 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_cpu.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_cpu.html new file mode 100644 index 000000000..5a7af264a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_cpu.html @@ -0,0 +1,1774 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_lock.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_lock.html new file mode 100644 index 000000000..c5a201508 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/get_20s_lock.html @@ -0,0 +1,572 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_alloc.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_alloc.html new file mode 100644 index 000000000..bb3633f13 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_alloc.html @@ -0,0 +1,810 @@ + + + + + + + +

Allocation profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_cpu.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_cpu.html new file mode 100644 index 000000000..f64cfc67e --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_cpu.html @@ -0,0 +1,3621 @@ + + + + + + + +

CPU profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_lock.html b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_lock.html new file mode 100644 index 000000000..6ec4cc2f7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/images/put_20s_lock.html @@ -0,0 +1,454 @@ + + + + + + + +

Lock profile

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/outputs/get_20_12000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/outputs/get_20_12000.txt new file mode 100644 index 000000000..5a0a6b0a4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/outputs/get_20_12000.txt @@ -0,0 +1,117 @@ +Running 20s test @ http://localhost:8080 + 4 threads and 128 connections + Thread calibration: mean lat.: 0.853ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.854ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.855ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.847ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.85ms 437.07us 2.31ms 65.82% + Req/Sec 3.24k 181.83 3.89k 64.37% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 0.85ms + 75.000% 1.15ms + 90.000% 1.40ms + 99.000% 1.92ms + 99.900% 2.10ms + 99.990% 2.22ms + 99.999% 2.29ms +100.000% 2.31ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.016 0.000000 1 1.00 + 0.270 0.100000 11816 1.11 + 0.426 0.200000 23672 1.25 + 0.581 0.300000 35478 1.43 + 0.726 0.400000 47263 1.67 + 0.850 0.500000 59056 2.00 + 0.912 0.550000 65006 2.22 + 0.973 0.600000 70933 2.50 + 1.033 0.650000 76769 2.86 + 1.093 0.700000 82716 3.33 + 1.154 0.750000 88637 4.00 + 1.184 0.775000 91542 4.44 + 1.216 0.800000 94500 5.00 + 1.250 0.825000 97439 5.71 + 1.289 0.850000 100430 6.67 + 1.335 0.875000 103349 8.00 + 1.366 0.887500 104825 8.89 + 1.405 0.900000 106290 10.00 + 1.454 0.912500 107769 11.43 + 1.518 0.925000 109244 13.33 + 1.587 0.937500 110725 16.00 + 1.621 0.943750 111469 17.78 + 1.655 0.950000 112189 20.00 + 1.691 0.956250 112929 22.86 + 1.727 0.962500 113665 26.67 + 1.763 0.968750 114412 32.00 + 1.781 0.971875 114770 35.56 + 1.801 0.975000 115136 40.00 + 1.822 0.978125 115508 45.71 + 1.844 0.981250 115875 53.33 + 1.868 0.984375 116249 64.00 + 1.881 0.985938 116434 71.11 + 1.894 0.987500 116616 80.00 + 1.909 0.989062 116802 91.43 + 1.926 0.990625 116984 106.67 + 1.944 0.992188 117167 128.00 + 1.956 0.992969 117265 142.22 + 1.968 0.993750 117353 160.00 + 1.979 0.994531 117443 182.86 + 1.994 0.995313 117543 213.33 + 2.007 0.996094 117630 256.00 + 2.015 0.996484 117675 284.44 + 2.024 0.996875 117720 320.00 + 2.034 0.997266 117767 365.71 + 2.042 0.997656 117813 426.67 + 2.053 0.998047 117857 512.00 + 2.059 0.998242 117881 568.89 + 2.067 0.998437 117904 640.00 + 2.079 0.998633 117929 731.43 + 2.089 0.998828 117950 853.33 + 2.103 0.999023 117972 1024.00 + 2.113 0.999121 117987 1137.78 + 2.119 0.999219 117997 1280.00 + 2.127 0.999316 118007 1462.86 + 2.137 0.999414 118020 1706.67 + 2.147 0.999512 118030 2048.00 + 2.153 0.999561 118036 2275.56 + 2.163 0.999609 118043 2560.00 + 2.167 0.999658 118047 2925.71 + 2.177 0.999707 118054 3413.33 + 2.189 0.999756 118060 4096.00 + 2.193 0.999780 118062 4551.11 + 2.197 0.999805 118064 5120.00 + 2.205 0.999829 118068 5851.43 + 2.211 0.999854 118071 6826.67 + 2.213 0.999878 118073 8192.00 + 2.221 0.999890 118075 9102.22 + 2.223 0.999902 118077 10240.00 + 2.223 0.999915 118077 11702.86 + 2.237 0.999927 118079 13653.33 + 2.239 0.999939 118080 16384.00 + 2.247 0.999945 118081 18204.44 + 2.249 0.999951 118082 20480.00 + 2.249 0.999957 118082 23405.71 + 2.253 0.999963 118083 27306.67 + 2.269 0.999969 118084 32768.00 + 2.269 0.999973 118084 36408.89 + 2.271 0.999976 118085 40960.00 + 2.271 0.999979 118085 46811.43 + 2.271 0.999982 118085 54613.33 + 2.289 0.999985 118086 65536.00 + 2.289 0.999986 118086 72817.78 + 2.289 0.999988 118086 81920.00 + 2.289 0.999989 118086 93622.86 + 2.289 0.999991 118086 109226.67 + 2.313 0.999992 118087 131072.00 + 2.313 1.000000 118087 inf +#[Mean = 0.854, StdDeviation = 0.437] +#[Max = 2.312, Total count = 118087] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 239133 requests in 20.00s, 15.74MB read + Non-2xx or 3xx responses: 239133 +Requests/sec: 11956.18 +Transfer/sec: 805.67KB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/outputs/put_20_90000.txt b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/outputs/put_20_90000.txt new file mode 100644 index 000000000..5d34bfed3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/outputs/put_20_90000.txt @@ -0,0 +1,132 @@ +Running 20s test @ http://localhost:8080 + 4 threads and 128 connections + Thread calibration: mean lat.: 1.118ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.120ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.117ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.115ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.11ms 520.35us 4.22ms 64.09% + Req/Sec 23.92k 1.47k 29.11k 72.04% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.08ms + 75.000% 1.48ms + 90.000% 1.81ms + 99.000% 2.30ms + 99.900% 3.07ms + 99.990% 3.60ms + 99.999% 3.90ms +100.000% 4.22ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.019 0.000000 3 1.00 + 0.446 0.100000 88800 1.11 + 0.606 0.200000 177519 1.25 + 0.771 0.300000 266102 1.43 + 0.929 0.400000 354342 1.67 + 1.083 0.500000 443262 2.00 + 1.161 0.550000 487609 2.22 + 1.239 0.600000 531719 2.50 + 1.319 0.650000 576200 2.86 + 1.399 0.700000 620205 3.33 + 1.482 0.750000 664244 4.00 + 1.526 0.775000 686636 4.44 + 1.572 0.800000 708720 5.00 + 1.621 0.825000 730683 5.71 + 1.675 0.850000 752784 6.67 + 1.737 0.875000 775031 8.00 + 1.771 0.887500 786076 8.89 + 1.808 0.900000 797048 10.00 + 1.850 0.912500 808332 11.43 + 1.895 0.925000 819389 13.33 + 1.944 0.937500 830265 16.00 + 1.972 0.943750 835940 17.78 + 2.000 0.950000 841416 20.00 + 2.029 0.956250 846891 22.86 + 2.061 0.962500 852489 26.67 + 2.097 0.968750 858108 32.00 + 2.117 0.971875 860982 35.56 + 2.137 0.975000 863569 40.00 + 2.161 0.978125 866384 45.71 + 2.187 0.981250 869097 53.33 + 2.219 0.984375 871878 64.00 + 2.237 0.985938 873286 71.11 + 2.257 0.987500 874587 80.00 + 2.281 0.989062 875935 91.43 + 2.311 0.990625 877303 106.67 + 2.349 0.992188 878708 128.00 + 2.373 0.992969 879421 142.22 + 2.401 0.993750 880106 160.00 + 2.435 0.994531 880789 182.86 + 2.475 0.995313 881454 213.33 + 2.537 0.996094 882153 256.00 + 2.575 0.996484 882496 284.44 + 2.621 0.996875 882835 320.00 + 2.681 0.997266 883192 365.71 + 2.749 0.997656 883527 426.67 + 2.823 0.998047 883874 512.00 + 2.869 0.998242 884046 568.89 + 2.911 0.998437 884221 640.00 + 2.967 0.998633 884397 731.43 + 3.019 0.998828 884567 853.33 + 3.073 0.999023 884740 1024.00 + 3.107 0.999121 884826 1137.78 + 3.141 0.999219 884912 1280.00 + 3.177 0.999316 884998 1462.86 + 3.225 0.999414 885084 1706.67 + 3.271 0.999512 885173 2048.00 + 3.295 0.999561 885215 2275.56 + 3.321 0.999609 885258 2560.00 + 3.353 0.999658 885300 2925.71 + 3.389 0.999707 885343 3413.33 + 3.437 0.999756 885387 4096.00 + 3.461 0.999780 885410 4551.11 + 3.489 0.999805 885430 5120.00 + 3.511 0.999829 885451 5851.43 + 3.537 0.999854 885473 6826.67 + 3.581 0.999878 885494 8192.00 + 3.591 0.999890 885505 9102.22 + 3.605 0.999902 885516 10240.00 + 3.629 0.999915 885527 11702.86 + 3.641 0.999927 885540 13653.33 + 3.653 0.999939 885548 16384.00 + 3.673 0.999945 885554 18204.44 + 3.687 0.999951 885560 20480.00 + 3.709 0.999957 885565 23405.71 + 3.735 0.999963 885570 27306.67 + 3.769 0.999969 885576 32768.00 + 3.777 0.999973 885578 36408.89 + 3.783 0.999976 885582 40960.00 + 3.793 0.999979 885584 46811.43 + 3.805 0.999982 885586 54613.33 + 3.833 0.999985 885589 65536.00 + 3.841 0.999986 885590 72817.78 + 3.855 0.999988 885592 81920.00 + 3.897 0.999989 885593 93622.86 + 3.931 0.999991 885595 109226.67 + 3.953 0.999992 885596 131072.00 + 3.953 0.999993 885596 145635.56 + 3.965 0.999994 885597 163840.00 + 3.981 0.999995 885598 187245.71 + 3.981 0.999995 885598 218453.33 + 4.047 0.999996 885599 262144.00 + 4.047 0.999997 885599 291271.11 + 4.049 0.999997 885600 327680.00 + 4.049 0.999997 885600 374491.43 + 4.049 0.999998 885600 436906.67 + 4.059 0.999998 885601 524288.00 + 4.059 0.999998 885601 582542.22 + 4.059 0.999998 885601 655360.00 + 4.059 0.999999 885601 748982.86 + 4.059 0.999999 885601 873813.33 + 4.223 0.999999 885602 1048576.00 + 4.223 1.000000 885602 inf +#[Mean = 1.109, StdDeviation = 0.520] +#[Max = 4.220, Total count = 885602] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 1793007 requests in 20.00s, 114.59MB read + Non-2xx or 3xx responses: 4424 +Requests/sec: 89647.42 +Transfer/sec: 5.73MB \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/report.md b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/report.md new file mode 100644 index 000000000..884c52d55 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/report.md @@ -0,0 +1,46 @@ +# Таразанов Максим, ИТМО ФИТИП, М4139, Stage 3 + +## Шардирование +При шардировании использовалась +**Consistent hashing** и хеш-функция **SHA-256**. + +Согласованный хеш увеличивает перформанc +за счет равномерного распределения, масштабируем и быстрее. +Весовой хеш при этом имеет сложность, зависющую от логарифма, +но проще реализуемый и подойдёт для меньших систем +(там где не надо будет прогревать огромным количеством данных). + +## GET +Тестирование запросов проводилось на показателях из предыдущего этапа. +(хотя rps на пару тысяч тоже держался нормально, но мне кажется, +моё железо как-то странно иногда работает) +В шардированном варианте иногда начали проскакивать ошибки +при запросах. Предположив, что это может быть из-за нехватки мощности ресурсов, +я немного уменьшил количество воркеров и действительно ошибок стало меньше. + +В шардированном варианте также улучшилась латенси. + +## PUT +Аналогично GET-запросам, тесты были на предыдущих показателях. +Также в новом варианте были некоторые ошибки в запросах и также +их стало меньше при уменьшении количества воркеров. +Также, увеличив в этом моменте TTL, на мой взгляд в логах стало +ещё меньше ошибок. Я пока не могу придумать ещё причины ошибок, +но пока, по-моему, получается убедительный вариант. + +Ну и латенси в среднем стало получше. + +## Asprof + +### CPU +В новой версии уходит очень много процессорного времени при поиске узла. +Нагрузки прошлого этапа процентно (друг относительно друга) не изменились, +поэтому есть предложение маршрутизировать запросы извне +(по типу балансировшик с прокси, шардирование на клиенте) + +### Alloc +Очень сильно возрасли аллокации из-за того, что берется нагрузка, по факту, прокси, +и HTTP очень сильно начинает требовать память. + +### Lock +Изменений обнаружено не было diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/scripts/get.lua b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/scripts/get.lua new file mode 100644 index 000000000..af0501626 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/scripts/get.lua @@ -0,0 +1,9 @@ +math.randomseed(os.time()) + +function request() + counter = math.random(100000100, 100000000 + 5000000) + headers = {} + headers["Host"] = "localhost:8080" + return wrk.format("GET", "/v0/entity?id=" .. tostring(counter), headers) + +end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/scripts/put.lua b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/scripts/put.lua new file mode 100644 index 000000000..90ba7a1a7 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tarazanovmaxim/stages/stage3/scripts/put.lua @@ -0,0 +1,12 @@ +counter = 100000000 + +function request() + counter = counter + 1 + body = tostring(counter) .. tostring(counter) .. tostring(counter) + headers = {} + headers["Content-Type"] = "text/plain" + headers["Content-Length"] = #{string.byte(body, 1, -1)} + headers["Host"] = "localhost:8080" + return wrk.format("PUT", "/v0/entity?id=" .. tostring(counter), headers, body) + +end \ No newline at end of file diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/ResponseProcessor.java b/src/main/java/ru/vk/itmo/test/trofimovmaxim/ResponseProcessor.java new file mode 100644 index 000000000..4f70998f5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/ResponseProcessor.java @@ -0,0 +1,45 @@ +package ru.vk.itmo.test.trofimovmaxim; + +import one.nio.http.Response; + +import java.net.HttpURLConnection; +import java.net.http.HttpResponse; +import java.util.Optional; +import java.util.function.BiConsumer; + +import static ru.vk.itmo.test.trofimovmaxim.TrofikServer.HEADER_TIMESTAMP; + +public class ResponseProcessor implements BiConsumer, Throwable> { + MergeHandleResult mergeHandleResult; + int index; + + public ResponseProcessor(MergeHandleResult mergeHandleResult, int index) { + this.mergeHandleResult = mergeHandleResult; + this.index = index; + } + + public void processResult(HandleResult result) { + mergeHandleResult.add(index, result); + } + + @Override + public void accept(HttpResponse httpResponse, Throwable throwable) { + if (throwable != null) { + mergeHandleResult.add(index, new HandleResult(HttpURLConnection.HTTP_INTERNAL_ERROR, Response.EMPTY)); + return; + } + Optional string = httpResponse.headers().firstValue(HEADER_TIMESTAMP); + long timestamp; + if (string.isPresent()) { + try { + timestamp = Long.parseLong(string.get()); + } catch (Exception e) { + timestamp = 0; + } + } else { + timestamp = 0; + } + + mergeHandleResult.add(index, new HandleResult(httpResponse.statusCode(), httpResponse.body(), timestamp)); + } +} diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/TrofikServer.java b/src/main/java/ru/vk/itmo/test/trofimovmaxim/TrofikServer.java index 9520e647f..fb2ff05e3 100644 --- a/src/main/java/ru/vk/itmo/test/trofimovmaxim/TrofikServer.java +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/TrofikServer.java @@ -25,7 +25,7 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; -import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -33,10 +33,11 @@ public class TrofikServer extends HttpServer { private static final String HEADER_REMOTE = "X-flag-remote-reference-server-to-node-by-trofik"; private static final String HEADER_REMOTE_ONE_NIO_HEADER = HEADER_REMOTE + ": da"; - private static final String HEADER_TIMESTAMP = "X-flag-remote-reference-timestamp-trofik"; + static final String HEADER_TIMESTAMP = "X-flag-remote-reference-timestamp-trofik"; private static final String HEADER_TIMESTAMP_ONE_NIO_HEADER = HEADER_TIMESTAMP + ": "; private static final Logger log = LoggerFactory.getLogger(TrofikServer.class); private static final int THREADS = Runtime.getRuntime().availableProcessors(); + private static final int CLIENT_TIMEOUT = 500; private final ExecutorService executorLocal = Executors.newFixedThreadPool( THREADS / 2, @@ -56,7 +57,7 @@ public TrofikServer(ServiceConfig config, this.httpClient = HttpClient.newBuilder() .executor(Executors.newFixedThreadPool(THREADS)) - .connectTimeout(Duration.ofMillis(500)) + .connectTimeout(Duration.ofMillis(CLIENT_TIMEOUT)) .version(HttpClient.Version.HTTP_1_1) .build(); } @@ -111,10 +112,13 @@ public void handleRequest(Request request, HttpSession session) throws IOExcepti for (int i = 0; i < indexes.length; i++) { int index = indexes[i]; String executorNode = config.clusterUrls().get(index); + ResponseProcessor responseProcessor = new ResponseProcessor(mergeHandleResult, i); if (executorNode.equals(config.selfUrl())) { - handleAsync(executorLocal, i, mergeHandleResult, () -> local(request, id)); + handleAsync(executorLocal, i, + mergeHandleResult, () -> responseProcessor.processResult(local(request, id))); } else { - handleAsync(executorRemote, i, mergeHandleResult, () -> remote(request, executorNode)); + handleAsync(executorRemote, i, + mergeHandleResult, () -> remote(request, executorNode, responseProcessor)); } } } @@ -153,16 +157,16 @@ private int getInt(Request request, String param, int defaultValue) { return ack; } - private HandleResult remote(Request request, String executorNode) { + private void remote(Request request, String executorNode, ResponseProcessor responseProcessor) { try { - return invokeRemote(executorNode, request); + invokeRemote(executorNode, request, responseProcessor); } catch (IOException e) { log.info("I/O exception while calling remote node", e); - return new HandleResult(HttpURLConnection.HTTP_INTERNAL_ERROR, Response.EMPTY); + responseProcessor.processResult(new HandleResult(HttpURLConnection.HTTP_INTERNAL_ERROR, Response.EMPTY)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); log.info("Thread interrupted"); - return new HandleResult(HttpURLConnection.HTTP_UNAVAILABLE, Response.EMPTY); + responseProcessor.processResult(new HandleResult(HttpURLConnection.HTTP_UNAVAILABLE, Response.EMPTY)); } } @@ -171,15 +175,13 @@ private void handleAsync(ExecutorService executor, HandlerRunnable runnable) { try { executor.execute(() -> { - HandleResult handleResult; try { - handleResult = runnable.run(); + runnable.run(); } catch (Exception e) { log.error("Exception during handleRequest", e); - handleResult = new HandleResult(HttpURLConnection.HTTP_INTERNAL_ERROR, Response.EMPTY); + mergeHandleResult.add(index, + new HandleResult(HttpURLConnection.HTTP_INTERNAL_ERROR, Response.EMPTY)); } - - mergeHandleResult.add(index, handleResult); }); } catch (Exception e) { mergeHandleResult.add(index, new HandleResult(HttpURLConnection.HTTP_INTERNAL_ERROR, Response.EMPTY)); @@ -196,7 +198,9 @@ public Response status() { return Response.ok("OK"); } - private HandleResult invokeRemote(String executorNode, Request request) throws IOException, InterruptedException { + private void invokeRemote(String executorNode, + Request request, + ResponseProcessor responseProcessor) throws IOException, InterruptedException { HttpRequest httpRequest = HttpRequest.newBuilder(URI.create(executorNode + request.getURI())) .method( request.getMethodName(), @@ -207,20 +211,11 @@ private HandleResult invokeRemote(String executorNode, Request request) throws I .header(HEADER_REMOTE, "da") .timeout(Duration.ofMillis(500)) .build(); - HttpResponse httpResponse = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofByteArray()); - Optional string = httpResponse.headers().firstValue(HEADER_TIMESTAMP); - long timestamp; - if (string.isPresent()) { - try { - timestamp = Long.parseLong(string.get()); - } catch (Exception e) { - log.error("Cannot parse timestamp from header", e); - timestamp = 0; - } - } else { - timestamp = 0; + CompletableFuture> sending = httpClient.sendAsync(httpRequest, + HttpResponse.BodyHandlers.ofByteArray()).whenComplete(responseProcessor); + if (sending == null) { + log.error("error while send async"); } - return new HandleResult(httpResponse.statusCode(), httpResponse.body(), timestamp); } private HandleResult local(Request request, String id) { @@ -286,7 +281,7 @@ private int[] getIndexes(String id, int count) { } private interface HandlerRunnable { - HandleResult run(); + void run(); } @Override diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/TrofikService.java b/src/main/java/ru/vk/itmo/test/trofimovmaxim/TrofikService.java index 7c1f6ef97..48e675475 100644 --- a/src/main/java/ru/vk/itmo/test/trofimovmaxim/TrofikService.java +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/TrofikService.java @@ -61,7 +61,7 @@ public static void shutdownAndAwaitTermination(ExecutorService pool) { } } - @ServiceFactory(stage = 4) + @ServiceFactory(stage = 5) public static class Factory implements ServiceFactory.Factory { @Override diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_alloc_st5.html b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_alloc_st5.html new file mode 100644 index 000000000..049fa3e52 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_alloc_st5.html @@ -0,0 +1,3216 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_cpu_st5.html b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_cpu_st5.html new file mode 100644 index 000000000..d4c221654 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_cpu_st5.html @@ -0,0 +1,5874 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_lock_st5.html b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_lock_st5.html new file mode 100644 index 000000000..b944e7f57 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_get_lock_st5.html @@ -0,0 +1,1193 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_alloc_st5.html b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_alloc_st5.html new file mode 100644 index 000000000..8beeef2db --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_alloc_st5.html @@ -0,0 +1,3205 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_cpu_st5.html b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_cpu_st5.html new file mode 100644 index 000000000..c618c3887 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_cpu_st5.html @@ -0,0 +1,5764 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_lock_st5.html b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_lock_st5.html new file mode 100644 index 000000000..27d22fd1c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/prof_put_lock_st5.html @@ -0,0 +1,1141 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/report.md b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/report.md new file mode 100644 index 000000000..2e55a6dd2 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/trofimovmaxim/reports/stage5/report.md @@ -0,0 +1,128 @@ +## PUT + +``` +i111433450:wrk2-arm trofik00777$ ./wrk -c 64 -t 4 -s /Users/trofik00777/Documents/itmo_s6/highload/2024-highload-dht/src/main/java/ru/vk/itmo/test/trofimovmaxim/lua/stage1/check_put.lua -d 1m -R 30000 "http://localhost:8080" -L +Running 1m test @ http://localhost:8080 + 4 threads and 64 connections + Thread calibration: mean lat.: 992.135ms, rate sampling interval: 3676ms + Thread calibration: mean lat.: 995.437ms, rate sampling interval: 3700ms + Thread calibration: mean lat.: 1001.864ms, rate sampling interval: 3713ms + Thread calibration: mean lat.: 988.766ms, rate sampling interval: 3661ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 7.27s 3.06s 12.62s 56.31% + Req/Sec 3.99k 173.20 4.29k 63.46% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 7.67s + 75.000% 9.91s + 90.000% 11.29s + 99.000% 12.13s + 99.900% 12.42s + 99.990% 12.53s + 99.999% 12.62s +100.000% 12.62s +Requests/sec: 25921.37 +Transfer/sec: 0.90MB +``` + +Будем профилировать на 25k rps: + +``` +i111433450:wrk2-arm trofik00777$ ./wrk -c 64 -t 4 -s /Users/trofik00777/Documents/itmo_s6/highload/2024-highload-dht/src/main/java/ru/vk/itmo/test/trofimovmaxim/lua/stage1/check_put.lua -d 1m -R 25000 "http://localhost:8080" -L +Running 1m test @ http://localhost:8080 + 4 threads and 64 connections + Thread calibration: mean lat.: 32.138ms, rate sampling interval: 164ms + Thread calibration: mean lat.: 30.932ms, rate sampling interval: 157ms + Thread calibration: mean lat.: 32.172ms, rate sampling interval: 164ms + Thread calibration: mean lat.: 29.839ms, rate sampling interval: 149ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 40.33ms 44.02ms 241.02ms 84.16% + Req/Sec 3.75k 548.98 7.27k 73.87% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 23.92ms + 75.000% 62.21ms + 90.000% 105.66ms + 99.000% 177.28ms + 99.900% 206.59ms + 99.990% 225.28ms + 99.999% 237.69ms +100.000% 241.15ms +Requests/sec: 24960.85 +Transfer/sec: 862.00KB +``` + +[cpu](prof_put_cpu_st5.html) + +[alloc](prof_put_alloc_st5.html) + +[lock](prof_put_lock_st5.html) + +По сравнению с прошлым этапом видим изменения в lock, конкретно наши функции invoke теперь меньше локают, тк перешли на асинхронное общение + +## GET + +Заполнил базу 600к значениями + +``` +i111433450:wrk2-arm trofik00777$ ./wrk -c 64 -t 4 -s /Users/trofik00777/Documents/itmo_s6/highload/2024-highload-dht/src/main/java/ru/vk/itmo/test/trofimovmaxim/lua/stage1/check_get.lua -d 1m -R 30000 "http://localhost:8080" -L +Running 1m test @ http://localhost:8080 + 4 threads and 64 connections + Thread calibration: mean lat.: 260.369ms, rate sampling interval: 1033ms + Thread calibration: mean lat.: 247.281ms, rate sampling interval: 1000ms + Thread calibration: mean lat.: 246.821ms, rate sampling interval: 990ms + Thread calibration: mean lat.: 266.203ms, rate sampling interval: 1040ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 3.05s 1.52s 6.13s 55.22% + Req/Sec 4.50k 281.23 5.13k 62.89% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 3.01s + 75.000% 4.39s + 90.000% 5.27s + 99.000% 5.71s + 99.900% 5.93s + 99.990% 6.11s + 99.999% 6.13s +100.000% 6.13s +Requests/sec: 25107.58 +Transfer/sec: 3.32MB +``` + +Будем профилировать на 25k rps: + +``` +i111433450:wrk2-arm trofik00777$ ./wrk -c 64 -t 4 -s /Users/trofik00777/Documents/itmo_s6/highload/2024-highload-dht/src/main/java/ru/vk/itmo/test/trofimovmaxim/lua/stage1/check_get.lua -d 1m -R 25000 "http://localhost:8080" -L +Running 1m test @ http://localhost:8080 + 4 threads and 64 connections + Thread calibration: mean lat.: 1.707ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.697ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.700ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.687ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.79ms 1.13ms 18.46ms 85.87% + Req/Sec 3.95k 485.91 7.40k 79.16% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.55ms + 75.000% 2.15ms + 90.000% 2.86ms + 99.000% 6.59ms + 99.900% 10.36ms + 99.990% 13.35ms + 99.999% 15.99ms +100.000% 18.48ms +Requests/sec: 24990.54 +Transfer/sec: 2.75MB +``` + +[cpu](prof_get_cpu_st5.html) + +[alloc](prof_get_alloc_st5.html) + +[lock](prof_get_lock_st5.html) + +Также видим, что по сравнению с прошлым этапом изменился lock, конкретно наши функции invoke теперь меньше локают, тк перешли на асинхронное общение + +## вывод + +В целом сервер стал работать получше и держать немного больше rps по сравнению с 4 этапом. +Но все также хуже, чем в 3 этапе. +Оно в целом ожидаемо, тк в нагрузочном тестировании мы задаем from=3,ack=2, соответственно ходим вместо 1 места аж в 3 (нагрузка на сеть растет кратно, от этого и дольше обработка, тк мы ждем несколько походов). +Собственно лучше 4 этапа, тк асинхронность добавилась при общении. diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/Client.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/Client.java new file mode 100644 index 000000000..884f2ae0e --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/Client.java @@ -0,0 +1,87 @@ +package ru.vk.itmo.test.tyapuevdmitrij; + +import one.nio.http.Request; +import one.nio.http.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.Optional; + +public class Client { + private static final Logger logger = LoggerFactory.getLogger(Client.class); + private final HttpClient httpClient; + + private String url; + + private long timeStamp; + + public static final String PROXY_TIMESTAMP_HEADER = "proxy"; + + public static final String NODE_TIMESTAMP_HEADER = "node"; + + Client() { + httpClient = HttpClient.newBuilder().build(); + } + + private byte[] getRequestBody(Request request) { + return request.getBody() == null ? new byte[0] : request.getBody(); + } + + private HttpRequest getProxyRequest(Request request) { + return HttpRequest.newBuilder(URI.create(url + request.getURI())) + .method(request.getMethodName(), + HttpRequest.BodyPublishers.ofByteArray(getRequestBody(request))) + .header(PROXY_TIMESTAMP_HEADER, String.valueOf(timeStamp)) + .build(); + } + + private HttpResponse getProxyResponse(Request request) { + try { + return httpClient.send(getProxyRequest(request), + HttpResponse.BodyHandlers.ofByteArray()); + } catch (IOException e) { + logger.error("can't reach target node"); + return null; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("can't reach target node"); + return null; + } + } + + public Response handleProxyRequest(Request request) { + HttpResponse response = getProxyResponse(request); + if (response == null) { + return null; + } + String statusCode = switch (response.statusCode()) { + case HttpURLConnection.HTTP_OK -> Response.OK; + case HttpURLConnection.HTTP_CREATED -> Response.CREATED; + case HttpURLConnection.HTTP_ACCEPTED -> Response.ACCEPTED; + case HttpURLConnection.HTTP_BAD_REQUEST -> Response.BAD_REQUEST; + case HttpURLConnection.HTTP_NOT_FOUND -> Response.NOT_FOUND; + default -> throw new IllegalStateException("Unexpected value: " + response.statusCode()); + }; + Optional nodeHeader = response.headers().firstValue(NODE_TIMESTAMP_HEADER); + Response finalResponse = new Response(statusCode, response.body()); + if (nodeHeader.isPresent()) { + long time = Long.parseLong(nodeHeader.get()); + finalResponse.addHeader(Client.NODE_TIMESTAMP_HEADER + ":" + time); + } + return finalResponse; + } + + public void setUrl(String url) { + this.url = url; + } + + public void setTimeStamp(long timeStamp) { + this.timeStamp = timeStamp; + } +} diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/ServerImplementation.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/ServerImplementation.java index 2ef7744b7..b95fac5dd 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/ServerImplementation.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/ServerImplementation.java @@ -10,20 +10,27 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ru.vk.itmo.ServiceConfig; -import ru.vk.itmo.dao.BaseEntry; -import ru.vk.itmo.dao.Entry; +import ru.vk.itmo.test.tyapuevdmitrij.dao.BaseEntry; +import ru.vk.itmo.test.tyapuevdmitrij.dao.Entry; import ru.vk.itmo.test.tyapuevdmitrij.dao.MemorySegmentDao; import java.io.IOException; import java.lang.foreign.MemorySegment; import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static one.nio.util.Hash.murmur3; + public class ServerImplementation extends HttpServer { private static final Logger logger = LoggerFactory.getLogger(ServerImplementation.class); @@ -32,6 +39,12 @@ public class ServerImplementation extends HttpServer { private static final String REQUEST_KEY = "id="; + private static final String FROM_PARAMETER = "from="; + + private static final String ACK_PARAMETER = "ack="; + + private static final String NOT_ENOUGH_REPLICAS = "504 Not Enough Replicas"; + private static final int THREAD_POOL_SIZE = Runtime.getRuntime().availableProcessors(); private static final int POOL_KEEP_ALIVE_SECONDS = 10; @@ -42,8 +55,13 @@ public class ServerImplementation extends HttpServer { private final ExecutorService executor; + private final ServiceConfig config; + private final Client client; + public ServerImplementation(ServiceConfig config, MemorySegmentDao memorySegmentDao) throws IOException { super(createServerConfig(config)); + this.config = config; + this.client = new Client(); this.memorySegmentDao = memorySegmentDao; this.executor = new ThreadPoolExecutor(THREAD_POOL_SIZE, THREAD_POOL_SIZE, @@ -70,21 +88,37 @@ public void handleRequest(Request request, HttpSession session) throws IOExcepti handleDefault(request, session); return; } - int requestMethod = request.getMethod(); String id = request.getParameter(REQUEST_KEY); - switch (requestMethod) { - case Request.METHOD_GET -> session.sendResponse(get(id)); - case Request.METHOD_PUT -> session.sendResponse(put(id, request)); - case Request.METHOD_DELETE -> session.sendResponse(delete(id)); - default -> session.sendResponse(getUnsupportedMethodResponse()); + if (id == null || id.isEmpty()) { + session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); + return; + } + String proxyHeader = request.getHeader(Client.PROXY_TIMESTAMP_HEADER + ":"); + if (proxyHeader != null) { + long timeStamp = Long.parseLong(proxyHeader); + Response response = handleNodeRequest(request, id, timeStamp); + session.sendResponse(response); + return; + } + int from = getValueFromRequest(request, FROM_PARAMETER, config.clusterUrls().size()); + int ack = getValueFromRequest(request, ACK_PARAMETER, (config.clusterUrls().size() + / 2) + 1); + if (ack == 0 || ack > from || from > config.clusterUrls().size()) { + session.sendResponse(new Response(Response.BAD_REQUEST, Response.EMPTY)); + return; } + long timeNow = System.currentTimeMillis(); + List url = getTargetNodesUrls(id, from); + List responses = collectResponses(request, from, url, id, timeNow); + Response finalResponse = aggregateResponses(responses, ack); + session.sendResponse(finalResponse); } catch (Exception e) { logger.error("Exception in request method", e); sendErrorResponse(session, Response.INTERNAL_ERROR); } }); } catch (RejectedExecutionException exception) { - logger.warn("ThreadPool queue overflow", exception); + logger.error("ThreadPool queue overflow", exception); sendErrorResponse(session, Response.SERVICE_UNAVAILABLE); } @@ -116,36 +150,36 @@ private static HttpServerConfig createServerConfig(ServiceConfig serviceConfig) } private Response get(String id) { - if (id == null || id.isEmpty()) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); - } MemorySegment key = MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)); Entry entry = memorySegmentDao.get(key); if (entry == null) { return new Response(Response.NOT_FOUND, Response.EMPTY); } - return Response.ok(entry.value().toArray(ValueLayout.JAVA_BYTE)); + Response response; + if (entry.value() == null) { + response = new Response(Response.NOT_FOUND, Response.EMPTY); + } else { + response = new Response(Response.OK, entry.value().toArray(ValueLayout.JAVA_BYTE)); + } + response.addHeader(Client.NODE_TIMESTAMP_HEADER + ":" + entry.timeStamp()); + return response; } - private Response put(String id, Request request) { - if (id == null || id.isEmpty() || request.getBody() == null) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); - } + private Response put(String id, Request request, long timeNow) { byte[] requestBody = request.getBody(); if (requestBody == null) { return new Response(Response.BAD_REQUEST, Response.EMPTY); } Entry entry = new BaseEntry<>(MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)), - MemorySegment.ofArray(requestBody)); + MemorySegment.ofArray(requestBody), timeNow); memorySegmentDao.upsert(entry); return new Response(Response.CREATED, Response.EMPTY); } - private Response delete(String id) { - if (id == null || id.isEmpty()) { - return new Response(Response.BAD_REQUEST, Response.EMPTY); - } - Entry entry = new BaseEntry<>(MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)), null); + private Response delete(String id, long timeNow) { + Entry entry = new BaseEntry<>(MemorySegment.ofArray(id.getBytes(StandardCharsets.UTF_8)), + null, + timeNow); memorySegmentDao.upsert(entry); return new Response(Response.ACCEPTED, Response.EMPTY); } @@ -153,5 +187,90 @@ private Response delete(String id) { private Response getUnsupportedMethodResponse() { return new Response(Response.METHOD_NOT_ALLOWED, Response.EMPTY); } + + private Response handleNodeRequest(Request request, String id, long timeNow) { + long timeStamp = timeNow; + String header = request.getHeader(Client.PROXY_TIMESTAMP_HEADER + ":"); + if (header != null) { + timeStamp = Long.parseLong(header); + } + switch (request.getMethod()) { + case Request.METHOD_GET -> { + return get(id); + } + case Request.METHOD_PUT -> { + return put(id, request, timeStamp); + } + case Request.METHOD_DELETE -> { + return delete(id, timeStamp); + } + default -> { + return getUnsupportedMethodResponse(); + } + } + } + + private List getTargetNodesUrls(String key, int size) { + Map treeMap = new TreeMap<>(Collections.reverseOrder()); + for (int i = 0; i < config.clusterUrls().size(); i++) { + treeMap.put(getCustomHashCode(key, i), config.clusterUrls().get(i)); + } + return new ArrayList<>(treeMap.values()).subList(0, size); + } + + private int getCustomHashCode(String key, int nodeNumber) { + return murmur3(key + nodeNumber); + } + + private int getValueFromRequest(Request request, String parameter, int defaultValue) { + String value = request.getParameter(parameter); + if (value == null) { + return defaultValue; + } else return Integer.parseInt(value); + } + + private List collectResponses(Request request, int from, List url, String id, long timeNow) { + List responses = new ArrayList<>(from); + for (int i = 0; i < from; i++) { + if (url.get(i).equals(config.selfUrl())) { + responses.add(handleNodeRequest(request, id, timeNow)); + } else { + client.setUrl(url.get(i)); + client.setTimeStamp(timeNow); + responses.add(client.handleProxyRequest(request)); + } + } + return responses; + } + + private Response aggregateResponses(List responses, int ack) { + int successResponses = 0; + Response finalResponse = null; + long finalTime = 0; + for (Response response : responses) { + if (response == null) { + continue; + } + String dataHeader = response.getHeader(Client.NODE_TIMESTAMP_HEADER + ":"); + if (dataHeader != null) { + long headerTime = Long.parseLong(dataHeader); + if (headerTime >= finalTime) { + finalTime = headerTime; + finalResponse = response; + } + } else { + if (finalResponse == null) { + finalResponse = response; + } + } + successResponses++; + } + if (successResponses >= ack) { + return finalResponse; + } else { + return new Response(NOT_ENOUGH_REPLICAS, Response.EMPTY); + } + } } + diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/ServiceImplementation.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/ServiceImplementation.java index 25fdbe0ec..acb547385 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/ServiceImplementation.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/ServiceImplementation.java @@ -1,5 +1,7 @@ package ru.vk.itmo.test.tyapuevdmitrij; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import ru.vk.itmo.ServiceConfig; import ru.vk.itmo.dao.Config; import ru.vk.itmo.test.ServiceFactory; @@ -9,13 +11,16 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; public class ServiceImplementation implements ru.vk.itmo.Service { private static final long FLUSH_THRESHOLD_BYTES = 1 << 20; // 1 MB + private static final Logger logger = LoggerFactory.getLogger(ServiceImplementation.class); private final ServiceConfig config; private ServerImplementation server; private MemorySegmentDao memorySegmentDao; @@ -47,7 +52,7 @@ public CompletableFuture stop() throws IOException { return CompletableFuture.completedFuture(null); } - @ServiceFactory(stage = 2) + @ServiceFactory(stage = 4) public static class Factory implements ServiceFactory.Factory { @Override @@ -57,11 +62,31 @@ public ru.vk.itmo.Service create(ServiceConfig config) { } public static void main(String[] args) throws IOException { - Path tempPath = new File("/home/dmitrij/Документы/JavaProjects/DaoServerData/").toPath(); - ServerImplementation server = new ServerImplementation(new ServiceConfig(8080, - "http://localhost", - List.of("http://localhost"), - tempPath), new MemorySegmentDao(new Config(tempPath, FLUSH_THRESHOLD_BYTES))); - server.start(); + int[] ports = new int[3]; + Path[] paths = new Path[ports.length]; + List cluster = new ArrayList<>(ports.length); + for (int i = 0; i < ports.length; i++) { + ports[i] = i + 8080; + paths[i] = new File("/home/dmitrij/Documents/javaProjects/DaoServerData/" + ports[i] + '/') + .toPath(); + if (!Files.exists(paths[i])) { + Files.createDirectory(paths[i]); + } + cluster.add("http://localhost:" + ports[i]); + } + + for (int i = 0; i < ports.length; i++) { + String url = cluster.get(i); + ServiceConfig cfg = new ServiceConfig( + ports[i], + url, + cluster, + Files.createTempDirectory("server") + ); + new ServerImplementation(cfg, + new MemorySegmentDao(new Config(paths[i], + FLUSH_THRESHOLD_BYTES))).start(); + logger.info("Socket is ready: " + url); + } } } diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/BaseEntry.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/BaseEntry.java new file mode 100644 index 000000000..cc99a06fa --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/BaseEntry.java @@ -0,0 +1,8 @@ +package ru.vk.itmo.test.tyapuevdmitrij.dao; + +public record BaseEntry(D key, D value, long timeStamp) implements Entry { + @Override + public String toString() { + return "{" + key + ":" + value + timeStamp + "}"; + } +} diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Dao.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Dao.java new file mode 100644 index 000000000..c1f98df9c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Dao.java @@ -0,0 +1,93 @@ +package ru.vk.itmo.test.tyapuevdmitrij.dao; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; + +public interface Dao> extends Closeable { + /** + * Returns ordered iterator of entries with keys between from (inclusive) and to (exclusive). + * + * @param from lower bound of range (inclusive) + * @param to upper bound of range (exclusive) + * @return entries [from;to) + */ + Iterator get(D from, D to); + + /** + * Returns entry by key. Note: default implementation is far from optimal. + * + * @param key entry`s key + * @return entry + */ + default E get(D key) { + Iterator iterator = get(key, null); + if (!iterator.hasNext()) { + return null; + } + + E next = iterator.next(); + if (next.key().equals(key)) { + return next; + } + return null; + } + + /** + * Returns ordered iterator of all entries with keys from (inclusive). + * + * @param from lower bound of range (inclusive) + * @return entries with key >= from + */ + default Iterator allFrom(D from) { + return get(from, null); + } + + /** + * Returns ordered iterator of all entries with keys < to. + * + * @param to upper bound of range (exclusive) + * @return entries with key < to + */ + default Iterator allTo(D to) { + return get(null, to); + } + + /** + * Returns ordered iterator of all entries. + * + * @return all entries + */ + default Iterator all() { + return get(null, null); + } + + /** + * Inserts of replaces entry. + * + * @param entry element to upsert + */ + void upsert(E entry); + + /** + * Persists data (no-op by default). + */ + default void flush() throws IOException { + // Do nothing + } + + /** + * Compacts data (no-op by default). + */ + default void compact() throws IOException { + // Do nothing + } + + /* + * Releases Dao (calls flush by default). + */ + @Override + default void close() throws IOException { + flush(); + } +} diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Entry.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Entry.java new file mode 100644 index 000000000..5a5e0cdda --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Entry.java @@ -0,0 +1,9 @@ +package ru.vk.itmo.test.tyapuevdmitrij.dao; + +public interface Entry { + D key(); + + D value(); + + long timeStamp(); +} diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/MemorySegmentDao.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/MemorySegmentDao.java index 077351ff0..e23be1d26 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/MemorySegmentDao.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/MemorySegmentDao.java @@ -1,8 +1,6 @@ package ru.vk.itmo.test.tyapuevdmitrij.dao; import ru.vk.itmo.dao.Config; -import ru.vk.itmo.dao.Dao; -import ru.vk.itmo.dao.Entry; import java.io.IOException; import java.lang.foreign.MemorySegment; @@ -90,13 +88,13 @@ public Entry get(MemorySegment key) { State currentState = this.state; Entry value = currentState.memTable.get(key); if (value != null && value.value() == null) { - return null; + return value; } if (value == null && currentState.flushMemTable != null) { value = currentState.flushMemTable.get(key); } if (value != null && value.value() == null) { - return null; + return value; } if (value != null || currentState.storage.ssTables == null) { return value; diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/MergeIterator.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/MergeIterator.java index 3ad5e2e09..2cf0ef35e 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/MergeIterator.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/MergeIterator.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.tyapuevdmitrij.dao; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.util.Collection; import java.util.Comparator; @@ -81,15 +79,6 @@ private PeekIterator peek() { peekFromPriorityQueue(); if (tableIterator.peek() == null) { tableIterator = null; - continue; - } - - if (skip(tableIterator.peek())) { - tableIterator.next(); - if (tableIterator.hasNext()) { - priorityQueue.add(tableIterator); - } - tableIterator = null; } } @@ -117,13 +106,6 @@ private void peekFromPriorityQueue() { } } - protected boolean skip(Entry entry) { - if (entry != null) { - return entry.value() == null; - } - return false; - } - @Override public boolean hasNext() { return peek() != null; diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/State.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/State.java index 50ef47a9f..0650ca5d4 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/State.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/State.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.tyapuevdmitrij.dao; -import ru.vk.itmo.dao.Entry; - import java.lang.foreign.MemorySegment; import java.util.ArrayList; import java.util.List; diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Storage.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Storage.java index 30d4c9429..e55e189a9 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Storage.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/Storage.java @@ -1,8 +1,5 @@ package ru.vk.itmo.test.tyapuevdmitrij.dao; -import ru.vk.itmo.dao.BaseEntry; -import ru.vk.itmo.dao.Entry; - import java.io.IOException; import java.lang.foreign.Arena; import java.lang.foreign.MemorySegment; @@ -99,6 +96,9 @@ private void writeMemTableDataToFile(MemorySegment buffer, MemorySegment.copy(entry.value(), 0, buffer, offset, entry.value().byteSize()); offset += entry.value().byteSize(); } + // write timeStamp next to the value + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, offset, entry.timeStamp()); + offset += Long.BYTES; } } @@ -134,7 +134,8 @@ private Entry getSsTableEntryByIndex(MemorySegment ssTable, long - (memTableSize - 1L) * Long.BYTES * 2L + index * 2L * Long.BYTES; MemorySegment readKey = getKeyByOffset(ssTable, keyIndexOffset); MemorySegment readValue = getValueByOffset(ssTable, keyIndexOffset + Long.BYTES); - return new BaseEntry<>(readKey, readValue); + long timeStamp = getTimeStampByOffset(ssTable, keyIndexOffset + Long.BYTES); + return new BaseEntry<>(readKey, readValue, timeStamp); } private MemorySegment getKeyByOffset(MemorySegment ssTable, long offset) { @@ -154,6 +155,15 @@ private MemorySegment getValueByOffset(MemorySegment ssTable, long offset) { return ssTable.asSlice(valueOffset, valueByteSize); } + private long getTimeStampByOffset(MemorySegment ssTable, long offset) { + long valueByteSizeOffset = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, offset); + long valueByteSize = ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, valueByteSizeOffset); + if (valueByteSize < 0) { + return ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, valueByteSizeOffset + Long.BYTES); + } + return ssTable.get(ValueLayout.JAVA_LONG_UNALIGNED, valueByteSizeOffset + Long.BYTES + valueByteSize); + } + public Iterator> range( Iterator> flushMemTableIterator, Iterator> memTableIterator, diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/StorageHelper.java b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/StorageHelper.java index e8776b976..a7ad61f9c 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/StorageHelper.java +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/dao/StorageHelper.java @@ -1,7 +1,5 @@ package ru.vk.itmo.test.tyapuevdmitrij.dao; -import ru.vk.itmo.dao.Entry; - import java.io.File; import java.io.IOException; import java.lang.foreign.Arena; @@ -78,6 +76,7 @@ public long getSsTableDataByteSize(Iterable> memTableEntrie if (entry.value() != null) { ssTableDataByteSize += entry.value().byteSize(); } + ssTableDataByteSize += Long.BYTES; entriesCount++; } memTableEntriesCount = entriesCount; @@ -110,6 +109,8 @@ public void saveDataForCompaction(State stateNow, Path ssTablePath) { dataOffset += Long.BYTES; MemorySegment.copy(entry.value(), 0, buffer, dataOffset, entry.value().byteSize()); dataOffset += entry.value().byteSize(); + buffer.set(ValueLayout.JAVA_LONG_UNALIGNED, dataOffset, Long.BYTES); + dataOffset += Long.BYTES; } } catch (IOException e) { throw new IllegalStateException(e); @@ -130,6 +131,7 @@ private long getCompactionTableByteSize(State stateNow) { Entry entry = dataIterator.next(); compactionTableByteSize += entry.key().byteSize(); compactionTableByteSize += entry.value().byteSize(); + compactionTableByteSize += Long.BYTES; countEntry++; } ssTablesEntryQuantity = countEntry; @@ -142,9 +144,9 @@ public static long sizeOf(final Entry entry) { } if (entry.value() == null) { - return entry.key().byteSize(); + return entry.key().byteSize() + Long.BYTES; } - return entry.key().byteSize() + entry.value().byteSize(); + return entry.key().byteSize() + entry.value().byteSize() + Long.BYTES; } } diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs.jfr b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs.jfr deleted file mode 100644 index 9b256a8eb..000000000 Binary files a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs.jfr and /dev/null differ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs.jfr b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs.jfr deleted file mode 100644 index 71738aa75..000000000 Binary files a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs.jfr and /dev/null differ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/get_3700rs_alloc.html similarity index 88% rename from src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs_alloc.html rename to src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/get_3700rs_alloc.html index 27cadb153..768d6ac46 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs_alloc.html +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/get_3700rs_alloc.html @@ -14,7 +14,7 @@ #status {left: 0} #match {right: 0} #reset {cursor: pointer} - #canvas {width: 100%; height: 528px} + #canvas {width: 100%; height: 544px} @@ -32,7 +32,7 @@

Flame Graph

let root, rootLevel, px, pattern; let level0 = 0, left0 = 0, width0 = 0; let reverse = false; - const levels = Array(33); + const levels = Array(34); for (let h = 0; h < levels.length; h++) { levels[h] = []; } @@ -257,9 +257,9 @@

Flame Graph

'*String', '0[]', '%util.ArrayList', -'*Comparator$$Lambda.0x00007b41eb0ee968', -'Kf058', -'L2f0', +'*Comparator$$Lambda.0x00007692530e5078', +'L768', +'La00', '*PriorityQueue', '$/lang/String.encode', '7UTF8', @@ -270,12 +270,12 @@

Flame Graph

'*invoke/DirectMethodHandle$Holder.newInvokeSpecial', 'C.allocateInstance', '1Invokers$Holder.linkToTargetMethod', -'1LambdaForm$DMH.0x00007b41eb092800.newInvokeSpecial', -'', '/s.copyOfRange', 'Flame Graph '!dk.internal.foreign.GlobalSession', '5HeapMemorySegmentImpl$OfByte', '5MappedMemorySegmentImpl', -'#/internal/foreign/AbstractMemorySegmentImpl$$Lambda.0x00007b41eb0ef588.apply', +'#/internal/foreign/AbstractMemorySegmentImpl$$Lambda.0x00007692530e5c98.apply', 'N.asSlice', 'VNoCheck', 'Olambda$toArray$1', @@ -309,7 +309,6 @@

Flame Graph

'-Request.', '5getParameter', ':th', -'8RequiredParameter', '/sponse.', '6ok', '6toBytes', @@ -321,12 +320,13 @@

Flame Graph

'2toAsciiString', ' ru.vk.itmo.dao.BaseEntry', '+test.tyapuevdmitrij.dao.MergeIterator', -'P$$Lambda.0x00007b41eb08f778', +'P$$Lambda.0x000076925308b7e0', 'QPeekIterator', 'CStorage$1', '"/vk/itmo/test/tyapuevdmitrij/ServerImplementation.get', +'ThandleRequest', '?dao/MemorySegmentDao.get', -'ErgeIterator$$Lambda.0x00007b41eb08f778.compare', +'ErgeIterator$$Lambda.0x000076925308b7e0.compare', 'QPeekIterator.peek', 'P.', 'QhasNext', @@ -344,50 +344,58 @@

Flame Graph

]; unpack(cpool); -n(3,20272) -u(521) -u(505) +n(3,21186) +u(513) +u(497) u(425) u(417) -u(401,20245) +u(401,21082) +u(593) u(393) -u(9,20242) -u(433,18) +u(9,21058) +u(433,52) u(441) -u(497,14) -u(529) +u(489,34) +u(521) u(18) -f(513,10,14,4) +f(505,11,34,18) u(386) -f(473,8,4,5) -u(457) +f(457,9,18,21) u(113) u(113) u(121) -u(34) -f(593,8,5,20219) -u(105,3) +u(34,13) +n(185,8) +u(193) +u(18) +f(585,9,8,20985) +u(105,14) u(89) u(97) u(18) -f(129,9,3,9) +f(129,10,14,28) u(337) -u(282,3) -n(361,6) +u(282,17) +n(361,11) u(274) -f(329,9,6,3) +f(329,10,11,18) u(329) u(297) u(321) u(18) -f(489,9,3,14) -u(378,4) -n(481,10) -u(18,7) +f(378,10,18,2) +n(473,7) +u(18,4) n(34,2) n(42,1) -f(601,9,1,20190) -u(633,58) +f(481,10,1,41) +u(378,9) +n(473,32) +u(18,18) +n(34,5) +n(42,9) +f(601,10,9,20875) +u(633,43) u(657) u(665) u(241) @@ -402,8 +410,8 @@

Flame Graph

u(673) u(673) u(689) -u(554,13) -n(681,20) +u(546,15) +n(681) u(345) u(345) u(305) @@ -411,7 +419,7 @@

Flame Graph

u(353) u(353) u(290) -f(705,25,20,25) +f(705,26,15,13) u(345) u(345) u(305) @@ -419,7 +427,7 @@

Flame Graph

u(353) u(353) u(290) -f(649,10,25,11) +f(649,11,13,39) u(649) u(241) u(249) @@ -433,8 +441,8 @@

Flame Graph

u(673) u(673) u(689) -u(554,3) -n(681,5) +u(546,7) +n(681,19) u(345) u(345) u(305) @@ -442,7 +450,7 @@

Flame Graph

u(353) u(353) u(290) -f(705,24,5,3) +f(705,25,19,13) u(345) u(345) u(305) @@ -450,36 +458,36 @@

Flame Graph

u(353) u(353) u(290) -f(721,10,3,20121) -u(50,2) -n(177,90) +f(721,11,13,20793) +u(50,13) +n(177,108) u(26) -f(209,11,90,4) +f(209,12,108,7) u(153) u(137) u(145) u(58) -f(562,11,4,2) -n(625,3505) -u(82,2) -n(153) +f(554,12,7,8) +n(625,3682) +u(82,9) +n(153,4) u(137) u(145) -u(570) -f(225,12,2,4) -u(209,2) +u(562) +f(225,13,4,14) +u(209,6) u(153) u(137) u(145) u(66) -f(225,13,2) +f(225,14,6,8) u(153) u(137) u(145) u(74) -f(233,12,2,119) +f(233,13,8,109) u(26) -f(241,12,119,2783) +f(241,13,109,2868) u(249) u(257) u(265) @@ -491,8 +499,8 @@

Flame Graph

u(673) u(673) u(689) -u(554,652) -n(681,1060) +u(546,717) +n(681,1057) u(345) u(345) u(305) @@ -500,7 +508,7 @@

Flame Graph

u(353) u(353) u(290) -f(705,24,1060,1071) +f(705,25,1057,1094) u(345) u(345) u(305) @@ -508,10 +516,10 @@

Flame Graph

u(353) u(353) u(290) -f(578,12,1071,595) -f(713,11,595,16518) -u(586,1256) -n(697,15262) +f(570,13,1094,678) +f(713,12,678,16975) +u(578,1292) +n(697,15683) u(681) u(345) u(345) @@ -520,25 +528,30 @@

Flame Graph

u(353) u(353) u(290) -f(465,7,15262,3) +f(465,8,15683,24) u(113) u(121) -u(185) +u(34,13) +n(185,11) u(193) u(18) -f(409,5,3,22) -u(370,6) -n(449,12) +f(409,5,11,80) +u(370,20) +n(449,32) u(42) -f(537,6,12,4) +f(529,6,32,28) +u(529) u(537) -u(545) -u(18) -f(537,5,4,5) +u(18,14) +n(169) +u(161) +u(145) +u(34) +f(529,5,14,24) +u(529) u(537) -u(545) -u(18,2) -n(169,3) +u(18,16) +n(169,8) u(161) u(145) u(34) diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/get_3700rs_cpu.html similarity index 52% rename from src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs_cpu.html rename to src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/get_3700rs_cpu.html index 02637c0ab..1b8ee31df 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/get_1000rs_cpu.html +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/get_3700rs_cpu.html @@ -14,7 +14,7 @@ #status {left: 0} #match {right: 0} #reset {cursor: pointer} - #canvas {width: 100%; height: 752px} + #canvas {width: 100%; height: 768px} @@ -32,7 +32,7 @@

Flame Graph

let root, rootLevel, px, pattern; let level0 = 0, left0 = 0, width0 = 0; let reverse = false; - const levels = Array(47); + const levels = Array(48); for (let h = 0; h < levels.length; h++) { levels[h] = []; } @@ -252,46 +252,27 @@

Flame Graph

const cpool = [ 'all', ' /usr/lib/libc.so.6', -' Arena::grow', -' Chunk::next_chop', -'!lassLoaderDataGraph::roots_cld_do', +' AccessInternal::PostRuntimeDispatch, (AccessInternal::BarrierType)0, 548964ul>::oop_access_barrier', +'!rena::grow', +' CallTraceStorage::put', '!odeCache::find_blob', '"mpiledMethod::scope_desc_at', -'$ositeElapsedCounterSource::now', -'"ncurrentGCThread::run', -'#tinuation::is_continuation_enterSpecial', -' Dictionary::lookup', -' G1AllocRegion::retire_internal', -'"BatchedTask::work', -'"CLDScanClosure::do_cld', -'#ollectedHeap::allocate_new_tlab', -'2ttempt_allocation_slow', -'1par_iterate_regions_array', -'1retire_mutator_alloc_region', -')ionSetCandidates::has_more_marking_candidates', -'$ncurrentRefine::adjust_threads_periodically', -'2Thread::run_service', -'"EvacuateRegionsBaseTask::work', -'1Task::evacuate_live_objects', -'7scan_roots', -'"GCParPhaseTimesTracker::G1GCParPhaseTimesTracker', -'"MergeHeapRootsTask::G1CombinedClosure::do_heap_region', -'6work', -'"ParCopyClosure<(G1Barrier)1, false>::do_oop', -'%EvacuateFollowersClosure::do_void', -'%ScanThreadState::steal_and_trim_queue', -'6trim_queue_to_threshold', -'#olicy::revise_young_list_target_length', -'#rimaryConcurrentRefineThread::do_refinement_step', -'"RootProcessor::evacuate_roots', -'1process_java_roots', -'9vm_roots', +' DebugInfoReadStream::read_object_value', +' FlightRecorder::recordEvent', +' G1CollectedHeap::do_collection_pause_at_safepoint', +'Q_helper', +'"GCPhaseTimes::sum_thread_work_items', +'"Policy::record_young_collection_end', +'"YoungCollector::collect', +' HandleMark::pop_and_restore', ' InstanceKlass::allocate_instance', '8objArray', '/get_jmethod_id', ' JavaThread::last_java_vframe', +',pd_last_frame', '$_one_nio_net_NativeSelector_epollWait', '8ocket_read', +'>write', '!vmtiEnv::GetClassSignature', '-StackTrace', '(Base::get_cthread_last_java_vframe', @@ -299,35 +280,34 @@

Flame Graph

'2threadOop_and_JavaThread', '&xport::post_sampled_object_alloc', '%ObjectAllocEventCollector::generate_call_for_allocated', +'@record_allocation', '5Mark::JvmtiObjectAllocEventMark', '%SampledObjectAllocEventCollector::~JvmtiSampledObjectAllocEventCollector', -'%VTMSTransitionDisabler::VTMS_transition_disable_for_all', -' Klass::is_subclass_of', ' MemAllocator::Allocation::notify_allocation_jvmti_sampler', '.allocate', '.mem_allocate_inside_tlab_slow', '"thod::jmethod_id', -'!utatorAllocRegion::retire', -' ObjAllocator::initialize', -'#ectSampler::recordAllocation', +' ObjectSampler::recordAllocation', +'&Value::read_object', '!ptoRuntime::new_array_C', +'7nozero_C', '1instance_C', ' PcDescContainer::find_pc_desc_internal', -'!erfLongVariant::sample', -'#iodicTask::real_time_tick', '!rofiler::recordSample', -' Rdtsc::elapsed_counter', -'!egisterMap::RegisterMap', -'"questHandler0_get.handleRequest', -'.1_put.handleRequest', -'"sourceArea::rollback_to', -' ScopeDesc::decode_body', -'+sender', -'!tatSamplerTask::task', +' RequestHandler0_get.handleRequest', +' ScopeDesc::ScopeDesc', +'+decode_body', +'%Value::read_from', ' Thread::call_run', '&HeapSampler::check_for_sampling', -' WatcherThread::run', -'!orkerThread::run', +'&Shadow::clear_pending_exception', +'&sListHandle::~ThreadsListHandle', +'!ypeArrayKlass::allocate_common', +' VMThread::evaluate_operation', +'*inner_execute', +'*run', +'"_G1CollectForAllocation::doit', +'#Operation::evaluate', ' [vdso]', ' __alloc_skb', '"check_object_size', @@ -336,54 +316,78 @@

Flame Graph

'#o_softirq', '"fdget', '"get_user_8', -'"ip_queue_xmit', +'"inet_lookup_established', +'#p_queue_xmit', +'"kfree_skb', '"local_bh_enable_ip', '"memmove', '"napi_poll', -'#etif_receive_skb_one_core', -'"pthread_mutex_lock', -'#ut_user_nocheck_4', -'"rcu_read_unlock', +'#etif_receive_skb_core.constprop.0', +'4one_core', +')x', +'"put_user_8', +'+nocheck_4', +'38', +'"rcu_read_lock', '#ecv', '#seq_handle_notify_resume', '"schedule', '#k_dst_check', +'%mem_reduce_allocated', '"tcp_push_pending_frames', +'&select_window', '&transmit_skb', +'#ls_get_addr', +'"virt_addr_valid', +'"wake_up_sync_key', '"x64_sys_epoll_wait', '*recvfrom', '*sendto', '!copy_from_iter', +'&to_iter', '!raw_spin_lock_bh', -'*unlock_irq', -'4restore', +'*unlock_irqrestore', '%write_unlock_irq', -' asm_sysvec_apic_timer_interrupt', -' bpf_lsm_socket_sock_rcv_skb', +' asm_common_interrupt', +'$sysvec_apic_timer_interrupt', +'!th10k_htt_txrx_compl_task?[ath10k_core]', +'\'pci_napi_poll?[ath10k_pci]', +' blkcg_maybe_throttle_current', +'!pf_lsm_socket_sock_rcv_skb', +'$skops_write_hdr_opt.isra.0', ' check_bounds', -'!ompiledVFrame::compiledVFrame', +'!ommon_interrupt', +'#piledVFrame::compiledVFrame', '0sender', -'"nsume_stock', +'!ubictcp_cwnd_event', ' dev_hard_start_xmit', '!o_softirq.part.0', '$yscall_64', '!st_release', -' entry_SYSCALL_64_after_hwframe', +' enqueue_to_backlog', +'"try_SYSCALL_64_after_hwframe', '!p_poll', '"oll_wait', '!xit_to_user_mode_prepare', -' frame::is_first_vthread_frame', -'"ee', +' fput', ' inet6_recvmsg', -'!p_finish_output2', +'&sendmsg', +'!p_finish_output', +'02', '#local_deliver_finish', +'#output', '#protocol_deliver_rcu', +'#rcv', +'&_core', +'\'finish_core.isra.0', '"v4_dst_check', '!rq_exit_rcu', -' java/lang/String.charAt', +' java/lang/Class.getSimpleName', +'0reflectionData', +'*String.charAt', '1indexOf', -'*foreign/MemorySegment.copy', -'@get', +'1startsWith', +'*foreign/MemorySegment.get', '@mismatch', '*invoke/VarHandleGuards.guard_LJ_I', 'JJ', @@ -391,10 +395,13 @@

Flame Graph

'IoffsetNoVMAlignCheck', 'CLongs.checkAddress', 'Iget', -'%util/ArrayList$Itr.next', -'*Comparator$$Lambda.0x00007b41eb0ee968.compare', -'Kf2f0.compare', -'4.lambda$comparing$ea9a8b3a$1', +'%util/ArrayList.add', +'4grow', +'*Comparator$$Lambda.0x00007692530e5078.compare', +'L768.compare', +'La00.compare', +'4.lambda$comparing$77a9974f$1', +'Fea9a8b3a$1', 'Flame Graph '8siftDownUsingComparator', 'UsingComparator', -'*concurrent/ConcurrentSkipListMap.cpr', -'KdoGet', -'Kget', -'5locks/AbstractQueuedSynchronizer.acquireShared', -';ReentrantReadWriteLock$ReadLock.lock', -'!dk/internal/foreign/AbstractMemorySegmentImpl.asSlice', +'!byte_disjoint_arraycopy', +'!dk/internal/foreign/AbstractMemorySegmentImpl.', +'OasSlice', 'VNoCheck', 'OcheckAccess', 'TBounds', -'Popy', -'OisAlignedForElement', 'Omismatch', 'OtoArray', -'5MappedMemorySegmentImpl.asSlice', +'5MappedMemorySegmentImpl.', +'MasSlice', 'Mdup', -'6emorySessionImpl.checkValidStateRaw', -'-misc/ScopedMemoryAccess.getByte', -'LInternal', -'HLongUnaligned', -'UInternal', -'!int_arraycopy', -'!ni_SetByteArrayRegion', -'!vmti_Deallocate', -'&GetClassSignature', +'5NativeMemorySegmentImpl.', +'-util/ArraysSupport.mismatch', +'!int_disjoint_arraycopy', +'!ni_GetByteArrayRegion', +'$SetByteArrayRegion', +'!vmti_GetClassSignature', ')StackTrace', -' kmem_cache_alloc_node', -' loopback_xmit', -' malloc', -'!em_cgroup_charge_skmem', -'"thodHandle::~methodHandle', +' kfence_guarded_free', +'!mem_cache_alloc_node', +'+free', +'!time_get', +')_seconds', +' lock_sock_nested', +'"opback_xmit', +' mem_cgroup_charge_skmem', +'+uncharge_skmem', '!od_memcg_state', '$timer', -'!utex_lock', -' napi_consume_skb', -'!et_rx_action', +'!utex_unlock', +' net_rx_action', +'#if_rx_internal', +'&skb_features', +'!f_conntrack_put', ' one/nio/http/HttpServer.handleRequest', '3ssion.handleParsedRequest', '9parseRequest', @@ -444,10 +450,11 @@

Flame Graph

'@Read', '9sendResponse', '9writeResponse', -'-Request.', +'-Request.addHeader', '5getHeader', -'8RequiredParameter', -'/sponse.toBytes', +'8Parameter', +'/sponse.ok', +'6toBytes', '(net/NativeSelector.epollWait', ';select', '3ocket.read', @@ -458,21 +465,28 @@

Flame Graph

'4write', '(server/SelectorThread.run', '(util/ByteArrayBuilder.append', -'-Utf8.length', -'!s::free', +'-URLEncoder.decode', +'.tf8.isAsciiString', +'2read', +'2write', +'!s::javaTimeMillis', +'$malloc', ' process_backlog', -'!thread_mutex_trylock', -' rb_insert_color', -'!ebalance_domains', +'!thread_cond_broadcast', +'(mutex_trylock', +' raw_local_deliver', +'$spin_rq_unlock_irqrestore.part.0', +'!b_insert_color', +'!elease_sock', +'"p_movs_alternative', +'"source_allocate_bytes', '!u/vk/itmo/test/tyapuevdmitrij/ServerImplementation.get', -'Tput', -'?dao/MemorySegmentComparator$$Lambda.0x00007b41eb08d648.compare', +'ThandleRequest', +'?dao/MemorySegmentComparator$$Lambda.0x000076925301e630.compare', 'Z.lambda$getMemorySegmentComparator$0', 'PDao.get', -'Tupsert', -'ErgeIterator$$Lambda.0x00007b41eb08f778.compare', -'QPeekIterator.next', -'^peek', +'ErgeIterator$$Lambda.0x000076925308b7e0.compare', +'QPeekIterator.peek', 'P.', 'QhasNext', 'Qlambda$new$0', @@ -486,627 +500,722 @@

Flame Graph

'NValueByOffset', 'Kiterator', 'Krange', +'"n_rebalance_domains', ' schedule', '(_hrtimeout_range_clock', '!ecurity_sock_rcv_skb', +'-et_recvmsg', +'0sendmsg', '"nd', '!k_filter_trim_cap', -'$orced_mem_schedule', +'#page_frag_refill', '#reset_timer', '"b_attempt_defer_free', '$clone', +')_tx_timestamp', '%opy_datagram_iter', +'$network_protocol', +'$push', '$release_data', '!ock_def_readable', +'%poll', '%recvmsg', -'&free', '!yscall_enter_from_user_mode', ')xit_to_user_mode', +'9_prepare', '#vec_apic_timer_interrupt', ' tcp_ack', '$check_space', +'%leanup_rbuf', +'%urrent_mss', +'$data_ready', +'%o_parse_auth_options', '$event_data_recv', -'$rate_skb_delivered', +'$inbound_hash.constprop.0', +'$mstamp_refresh', +'$options_write', +'$poll', +'%ush', +'$queue_rcv', +'$rate_check_app_limited', +')skb_delivered', '%cv_established', '%ecvmsg', '+_locked', -'$sendmsg', +'&lease_cb', +'$schedule_loss_probe', +'%endmsg', '+_locked', '%tream_alloc_skb', +'$update_skb_after_send', '$v4_do_rcv', '\'fill_cb', '\'rcv', -'$write_xmit', +'\'send_check', +'$wfree', +'%rite_xmit', '!hread_native_entry', -'!ry_charge_memcg', ' unknown_Java', -' vframe::java_sender', +' validate_xmit_skb', +'.xfrm', +'!frame::java_sender', '(new_vframe', '(sender', -'(vframe', -'!oid G1ScanEvacuatedObjClosure::do_oop_work', -'%OopOopIterateBackwardsDispatch::Table::oop_oop_iterate_backwards', -'%WeakProcessor::Task::work' +'(vframe' ]; unpack(cpool); -n(3,5006) -u(11,26) -f(949,2,3,1) -u(933) -u(2013) -f(2139,2,1,22) -u(596) -u(68,1) -u(164) -u(260) -u(156) -u(252) -u(148) -f(612,4,1) -u(516) -u(588) -u(508) -f(620,4,1,20) -u(100,1) -u(987) -f(172,5,1,17) -u(180,15) -u(60,1) -u(651) -u(627) -f(228,7,1,14) -u(236,13) -u(244) -f(2196,10,10,2) -n(2204,1) -u(2196) -f(244,8,1) -f(188,6,1,2) -u(268) -u(276,1) -u(36) +n(3,4122) +u(11,3) +f(925,2,1,1) +u(901) +u(2165) +f(2435,2,1) +u(388) +u(444) +u(436) +u(428) +u(460) +u(452) +u(76) +u(84) u(108) -u(220) -f(284,8,1) -u(196) -u(60) -u(532) -f(212,5,1) -u(132) -u(204) -f(2212,5,1) -f(1649,1,1,4979) -u(1593,20,1,0,0) -f(1585,3,1,19) -u(323,2) -n(963,17) -f(949,5,2,15) -u(933) -f(805,7,1,12) -f(677,8,2,1) -n(957,9) -f(741,9,2,3) -n(861,2) -n(1477,1) -n(1901) -u(1893) -u(773) -f(2005,7,1) -n(2013) -u(973) -u(765) -u(685) -f(1625,2,1,4959,1,0,0) -u(1529,4959,1,0,0) -u(1521,4946) -f(1505,5,3,4941,2,0,0) -u(1497,4941,2,0,0) -u(545,4939) -u(1537,71) -f(1545,9,1,69) -u(1577,4) -f(1658,11,3,1) -u(1666) -u(1042) -f(1641,10,1,65) -u(1641) -u(1617) -u(1609) -f(1915,14,2,63) -f(949,15,5,58) -u(933) -u(821,53) -u(677,2) -n(2085,51) -u(2093) -f(789,20,1,43) -u(2133) -u(797,42) -f(693,23,2,39) -f(781,24,1,1) -u(1029) -f(1005,24,1,37) -f(661,25,3,34) -f(701,26,2,31) -u(925) -f(669,28,1,30) -u(1493) -f(717,30,3,26) -f(1685,31,1,25) -f(725,32,1,23) +u(100) +u(92) +f(1705,1,1,4118) +f(491,2,1,1) +u(467) +f(1649,2,1,78,1,0,0) +u(1641,78,1,0,0) +f(11,4,1,3) +n(163,4) +n(491,2) +u(467) +f(939,4,2,68) +f(925,5,14,54) +u(901) +f(725,7,1,30) +f(517,8,3,2) +n(933,18) +f(613,9,1,1) +n(621) +n(781,4) +n(1509,1) +n(2021,5) +f(2013,10,2,3) +u(653) +f(2149,9,3,5) +f(2277,10,1,4) +f(957,8,4,7) +f(2165,7,7,9) +n(2173,14) +f(949,8,2,7) +f(645,9,3,3) +u(525,2) +n(605,1) +f(821,9,1) +f(2181,8,1,5) +f(1681,2,5,4038,1,0,0) +u(1577,4038,1,0,0) +u(1569,3977) +f(1553,5,2,3961) +u(1849) +u(1545,3961,3,0,0) +f(353,8,3,3958) +f(1275,9,1,1) +n(1585,195) +u(1593,191,5,0,0) +u(1633,4) +f(1714,12,2,2) +u(1746) +u(1746) +u(1074) +f(1697,11,2,187,5,0,0) +u(491,5) +f(467,13,1,4) +f(1697,12,4,182) +u(1673) +u(1665) +f(11,15,1,3) +n(179,4) +f(1387,16,2,2) +u(116,1) +n(843) +f(2051,15,1,174) +f(925,16,13,161) +u(901) +u(741,150) +f(517,19,2,4) +f(629,20,3,1) +f(957,19,1,2) +n(973,1) +n(2045,3) +n(2357,138) +u(1821,2) +u(765,1) +n(2341) +f(2365,20,1,136) +f(485,21,6,1) +u(709) +f(677,21,1,109) +f(2429,22,1,108) +f(693,23,5,100) +f(541,24,5,84) +u(661,1) +u(1045) +f(981,25,1,2) +n(989,80) +f(501,26,4,76) +u(557,65) +f(893,28,1,64) +f(509,29,2,62) +f(1517,30,1,61) +u(573,59) +f(1773,32,1,58) +f(589,33,1,57) +u(581,1) +n(997,54) u(1013) -u(1021) -u(2125) -f(1925,36,2,3) -u(1909) -f(877,38,1,2) -f(2109,36,2,17) -u(2061) -u(941,1) -n(1941) +f(1797,36,1,2) +n(2405,51) +f(533,37,2,2) +n(1541,1) +n(2061,2) +u(2029) +f(829,39,1,1) +f(2253,37,1) +u(2237) +f(2389,37,1,42) +u(2317) +f(909,39,2,1) +n(1453) +n(2077,5) +u(1501) +f(773,41,4,1) +f(2141,39,1,13) +u(717,1) +n(773,12) +f(2197,39,12,10) +f(549,40,7,1) +u(2133) +f(2309,40,1) +n(2349) +f(2205,39,1,2) +n(2229) +n(2245,3) +n(2261,2) +n(2293,1) +f(2397,37,1) +u(565) +f(1021,34,1,2) +u(1029,1) +n(1037) +f(1421,31,1) +u(773) +f(1437,31,1) +f(885,27,1,7) u(1469) -u(853) -f(1981,38,1,4) -f(853,39,1,3) -f(2029,38,3,7) -f(2053,39,6,1) -f(2037,38,1,3) -n(2045,1) -f(2117,36,1) -u(709) -f(845,32,1) -f(1485,30,1) -u(1973) -f(917,26,1) +f(597,29,2,3) +u(1525) +f(917,31,1,2) +f(773,32,1,1) +f(2101,29,1) +n(2421) +f(2453,27,1,3) +f(1533,28,2,1) +u(2117) +f(2461,27,1) +f(1005,25,1) +f(837,24,1,3) +n(877,2) +n(2093,1) +n(2125,2) +n(2269,1) +n(2381) +n(2413) +f(1445,23,1,2) +n(1813,1) +f(749,21,1) +u(1829) +f(2069,21,1,2) +n(2221,1) +n(2285,3) +n(2301,2) +n(2373,11) +f(477,22,1,6) u(1429) -f(1957,23,1) -f(1701,22,1) -f(829,20,1) -n(2101,6) -u(637,3) -f(1421,22,1,2) -f(1445,21,2) -u(1461,1) -n(2149) -u(909) -f(1933,21,1) -f(2005,17,1,5) -f(1561,9,5,1) -f(1569,8,1,2) -n(1713,4866) -f(1313,9,1,2) -f(1314,10,1,1) -u(1058) -u(1290) -u(1298) -u(1298) -f(1745,9,1,4863) -f(1234,10,2,27,17,0,0) -u(1226,27,17,0,0) -f(1217,12,17,10) -u(1730) -u(1738) -u(1074) -u(1074) -u(1306) -f(1729,10,10,1) -u(1737) -u(1073) -u(1074) -u(1306) -f(1793,10,1,67,1,0,0) -f(1817,11,1,66) -u(1185,12,2,0,0) -u(1193,12,2,0,0) -f(1145,14,2,10) -u(1162,10,7,0,0) -u(1762,10,7,0,0) -u(1802,10,7,0,0) -u(1138,9,6,0,0) -u(1154,9,6,0,0) -u(1730,9,6,0,0) -u(1738,9,6,0,0) -f(1066,22,2,4) -u(1082) -u(1098) -u(1106,2) -n(1346) -u(1354) -u(1338) -f(1073,22,2,3) -u(1074) -u(1306) -f(1778,18,3,1) -u(1834) -u(1834) -u(1850) -u(1066) -u(1090) -u(1122) -u(1362) +f(1477,22,6,4) +u(1493) +f(2165,18,4,11) +f(1610,10,11,4,3,0,0) +f(332,11,3,1) +u(124) +u(276) +u(268) +u(260) +u(236) +u(228) +u(300) +u(348) +u(1411) +u(196) +u(212) +u(212) +u(2468) +u(868) +u(2484) +f(1617,9,1,5) +f(1090,10,1,2) u(1370) -u(1338) -f(1825,12,1,54,8,0,0) -f(1138,13,1,2,1,0,0) -u(1154,2,1,0,0) -u(1730,2,1,0,0) -u(1738,2,1,0,0) -u(1066,1) +f(1721,10,2) u(1082) -u(1098) -u(1106) -f(1073,17,1) -u(1074) -u(1306) -f(1170,13,1,3,1,0,0) -u(1178,3,1,0,0) -u(1202,3,1,0,0) -f(1209,16,1,2) -u(1145) -u(1161) -u(1761) -u(1801) -u(1137) -u(1153) -u(1729) -u(1737) -u(1073) -u(1074) -u(1306) -f(1185,13,2,47,4,0,0) -u(1193,47,4,0,0) -f(1145,15,4,43) -f(1162,16,3,40,17,0,0) -u(1762,40,17,0,0) -u(1802,40,17,0,0) -u(1138,36,13,0,0) -u(1154,36,13,0,0) -u(1730,36,13,0,0) -u(1738,36,13,0,0) -f(1066,23,5,9,8,0,0) -u(1082,9,8,0,0) -u(1098,9,8,0,0) -u(1106,3) -n(1346,6,5,0,0) -u(1354,6,5,0,0) -u(1338,6,5,0,0) -f(1073,29,5,1) -f(1073,23,1,22) -f(1074,24,2,20) -u(1306) -f(1778,19,20,4) -u(1834) -u(1834) -u(1850) -u(1066) -u(1090) -u(1122) -u(1362) -u(1370) -u(1338) -f(1770,13,4,1) -f(1809,10,1,4) -u(1809) -u(1169,3) -u(1177) -u(1201) -u(1209) -u(1145) -u(1161) -u(1761) -u(1801) -u(1137,2) -u(1153) -u(1729) -u(1737) -u(1073) -u(1074) -u(1306) -f(1777,20,2,1) -u(1834) -u(1834) -u(1850) -u(1866) +u(1082) +f(1841,9,2,3756,1,0,0) +u(1329,2) +u(1330) +u(1058) u(1066) -u(1090) -u(1122) -u(1362) -u(1370) -u(1338) -f(1769,12,1) -u(1777) -f(1881,10,1,4762) -f(492,11,19,2) -u(292) -u(436) -u(428) -u(404) -u(388) -u(380) -u(476) -u(524) -u(1411) -u(348) -u(364,1) -u(356) -u(44) -f(372,22,1) +f(1625,10,2,1) +u(324) u(420) -f(1785,11,1,659) -f(1129,12,59,1) -u(484) +u(276) +u(268) +u(260) +u(236) +u(228) u(300) -u(436) -u(428) -u(404) -u(388) -u(380) -u(476) -u(524) -u(1411) u(348) -u(364) -u(364) -u(1452) -f(1169,12,1,599,11,0,0) -u(1177,599,11,0,0) -u(1201,599,11,0,0) -f(1209,15,11,588) -f(1145,16,24,564,101,0,0) -f(1161,17,19,545,82,0,0) -u(1761,545,82,0,0) -u(1801,545,82,0,0) -f(1137,20,19,246,63,0,0) -u(1153,246,63,0,0) -u(1729,246,63,0,0) -u(1737,246,63,0,0) -f(1066,24,20,45,43,0,0) -u(1082,45,43,0,0) -u(1098,45,43,0,0) -u(1106,11) -n(1346,34,32,0,0) -u(1354,34,32,0,0) -u(1338,34,32,0,0) -f(1073,30,32,2) -f(1073,24,2,178) -f(1074,25,9,169) -u(1306) -f(1777,24,169,3) -f(1777,20,3,280) -f(1834,21,6,274,265,0,0) -u(1834,274,265,0,0) -u(492,1) -u(292) -u(436) -u(428) -u(404) -u(388) -u(380) -u(476) -u(524) u(1411) -u(348) -u(364) -u(364) -u(2164) -u(900) -u(2180) -f(1850,23,1,273,265,0,0) -u(492,5) +u(196) +u(212) +u(212) u(292) -u(436) -u(428) -u(404) -u(388) -u(380) -u(476) -u(524,4) -u(1411) -u(348) -u(364) -u(364) -u(452,1) -n(2164,3) -u(900) -u(20,2) -n(540,1) -f(1403,32,1) -u(340) -u(11) -f(1066,24,1,13) -u(1090) -u(1122) -u(1362) -u(1370) +u(140) +f(1873,10,1,3753,1,0,0) +f(1857,11,1,1) +u(1865) +u(1105) +u(1106) +u(1322) +f(1905,11,1,40) +u(1929) +u(1241,22,4,0,0) +f(1249,14,1,21,3,0,0) +f(1193,15,3,18) +f(1217,16,2,16,5,0,0) +u(1185,1) +u(1201) +f(1882,17,1,15,5,0,0) +u(1914,15,5,0,0) +u(1177,12,2,0,0) +u(1209,12,2,0,0) +u(1857,12,2,0,0) +u(1865,12,2,0,0) +f(1098,23,1,1) +u(1114) +u(1130) +u(1138) +f(1105,23,1,10) +u(1106) +u(1322) +f(1098,26,4,6) +u(1114) +u(1130) +u(1138) +f(1890,19,6,3) +u(1946) +u(1946) +u(1962) +u(1954) +u(1346) +u(1346) +u(1290) +u(1298) +u(1354) +u(1354) u(1338) -f(1842,24,13,149) -u(1066,138) -u(1090) -u(1122) u(1362) -u(1370) -u(1338) -f(1322,25,138,11) +u(1282) +f(1937,13,3,18,1,0,0) +u(1177,4,1,0,0) +u(1209,4,1,0,0) +u(1857,4,1,0,0) +u(1865,4,1,0,0) +f(1105,18,1,3) +u(1106) u(1322) +f(1098,21,2,1) +u(1114) +u(1130) +u(1138) +f(1225,14,1) +u(1234) u(1258) u(1266) -u(1330) -u(1330) -f(1866,24,11,106,103,0,0) -u(1066,103) -u(1090) -u(1122) -u(1362) -u(1370) +u(1194) +u(1218) +u(1186) +f(1241,14,1,13) +u(1249) +u(1193) +u(1218,13,7,0,0) +u(1882,13,7,0,0) +u(1914,13,7,0,0) +u(1178,10,4,0,0) +u(1210,10,4,0,0) +u(1858,10,4,0,0) +u(1866,10,4,0,0) +f(1098,24,2,2) +u(1114) +u(1130) +u(1138) +f(1105,24,2,6) +u(1106) +u(1322) +f(1098,27,2,4) +u(1114) +u(1130) +u(1138) +f(1890,20,4,3) +u(1946) +u(1946) +u(1962) +u(1954) +u(1346) +u(1346) +u(1290) +u(1298) +u(1354) +u(1354) u(1338) -f(1321,25,103,3) -u(1321) -u(1257) -u(1265) -u(1329) -u(1329) -u(492) -u(292) -u(436) -u(428) -u(404) -u(388) -u(380) -u(476) -u(524) +u(1362) +u(1282) +f(1921,11,3,8) +u(1921) +u(1225) +u(1234,8,3,0,0) +u(1258,8,3,0,0) +u(1266,8,3,0,0) +u(1194,8,3,0,0) +u(1218,8,3,0,0) +u(1882,8,3,0,0) +u(1914,8,3,0,0) +f(1177,21,2,4,1,0,0) +u(1209,4,1,0,0) +u(1857,4,1,0,0) +u(1865,4,1,0,0) +f(1097,25,1,1) +u(1113) +u(1129) +u(1137) +u(1105) +f(1105,25,1,2) +u(1106) +u(1322) +f(1098,28,1,1) +u(1114) +u(1130) +u(1138) +f(1889,21,1,2) +u(1946) +u(1946) +u(1962) +u(1954) +u(1346) +u(1346) +u(1290) +u(1298) +u(1354) +u(1354) +f(1993,11,2,3703) +f(332,12,28,5) +u(124) +u(276) +u(268) +u(260,3) +u(236) +u(228) +u(300) +u(348,2) u(1411) -u(348) +u(196) +u(212) +u(204,1) +u(148) +u(2476) +u(2476) +u(860) +u(52) u(364) +f(212,24,1) +u(2468) +u(868) +u(2484) +u(2476) +u(860) +u(52) u(364) -u(452,2) -u(308) -f(2164,44,2,1) -u(900) -u(2188) -f(1873,11,1,4082,23,0,0) -f(1066,12,11,12) -u(1090) -u(1122) -u(1362) -u(1370) -u(1338) -f(1857,12,12,4059) -f(1066,13,62,2) -u(1090) -u(1122) -u(1362) -u(1370) -u(1338) -f(1730,13,2,2520,1863,0,0) -u(1738,2520,1863,0,0) -f(869,15,1744,2) -u(2021) -u(1037) -u(669) -u(1709) -f(749,20,1,1) -f(1066,15,1,124,117,0,0) -u(1082,124,117,0,0) -u(1098,124,117,0,0) -u(1346,124,117,0,0) -u(1354,124,117,0,0) -u(1338,124,117,0,0) -f(1073,21,117,7) -f(1073,15,7,650) -f(1074,16,28,622) -u(1306) -f(1842,13,622,1475,1440,0,0) -u(1066,1447,1412,0,0) -u(1090,1447,1412,0,0) -u(1122,1447,1412,0,0) -u(1113,35) -u(1273) -u(1281) -u(492,33) -u(292) -u(436,32) -f(11,23,1,1) -n(428,29) -u(404,28) -u(388) u(380) -u(396,1) -n(476,27) -f(11,28,1,1) -n(84) -n(524,21) -f(1411,29,1,20) -u(348) -u(364,17) -u(356,1) +u(60) +u(308) +u(699) +f(1403,20,1) +u(1764) +f(396,16,1,2) +f(1161,12,2,1) +u(1161) +u(1169) +u(1169) u(316) -u(2172) -u(2172) -u(892) +u(132) +u(276) +u(268) +u(260) +u(236) +u(228) +u(300) +u(348) +u(68) +f(1897,12,1,385) +f(1225,13,34,351) +f(1233,14,2,349,91,0,0) +u(1257,349,91,0,0) +u(1265,349,91,0,0) +f(1193,17,5,344,86,0,0) +f(1217,18,19,325,67,0,0) +u(1186,9,7,0,0) +f(1201,20,7,2) +f(1881,19,2,316,60,0,0) +u(1913,316,60,0,0) +f(1177,21,27,136,33,0,0) +u(1209,136,33,0,0) +u(1857,136,33,0,0) +u(1865,136,33,0,0) +f(1098,25,16,19,17,0,0) +u(1114,19,17,0,0) +u(1130,19,17,0,0) +u(1138,19,17,0,0) +f(1105,29,17,2) +f(1105,25,2,99) +f(1106,26,5,94) +u(1322) +f(1098,28,40,54) +u(1114) +u(1130) +u(1138) +f(789,32,53,1) +u(853) +u(1053) +u(509) +u(1517) +u(573) +u(813) +u(805) +f(1889,25,1,2) +f(1889,21,2,153) +f(1946,22,10,143,131,0,0) +u(1946,143,131,0,0) +u(332,6) +u(124) +u(276) +u(268) +u(260) +u(236) +u(228) +u(300) +u(348) +u(36,1) +n(1411,5) +u(196) +u(212,4) +u(212) +u(292,1) +n(2468,3) +u(868) +u(28,2) +n(2484,1) +u(2476) +u(860) u(52) -u(572) -f(364,32,1,16) -u(452,2) -f(308,34,1,1) -f(564,33,1) -u(28) -u(731) -f(2164,33,1,13) -f(900,34,1,11) -u(580,2) -n(2180,9) -f(2172,36,4,5) -u(20,1) -n(892,4) -f(52,38,1,3) -f(500,39,2,1) -f(980,34,1) -u(76) -f(372,31,1) -u(420) -f(412,31,1,2) -f(1395,28,2) -u(987,1) -n(1676) -f(1403,28,1) -u(340) -u(11) -f(604,24,1) -f(444,23,1) -u(116) +u(372) +f(412,35,1) +f(1962,24,1,137,131,0,0) +u(332,2) u(124) -u(460) -u(92) -u(140) -f(468,22,1) -f(1435,20,1) -n(1691) -f(1362,17,1,1412) -u(1370) -u(1338) -f(869,20,1411,1) -u(2021) -u(1037) -u(669) -u(1709) -f(1322,14,1,28) +u(276) +u(268) +u(260,1) +u(236) +u(228) +u(300) +u(348) +u(1411) +u(196) +u(212) +u(212) +u(2468) +u(868) +u(2484) +f(396,29,1) +u(244) +u(20) +f(1954,25,1,131) +u(1346) +u(1346) +u(1290) +u(1298) +u(1354) +u(1354) +f(1977,25,131,4) +u(1345) +u(1345) +u(1289) +u(1297) +u(1353) +u(1353) +u(332) +u(124) +u(276) +u(268) +u(260) +u(236) +u(228) +u(252,1) +n(300,3) +u(348) +u(1411) +u(196) +u(212) +u(212) +u(2468) +f(868,46,1,2) +u(2484,1) +n(2492) +f(1969,12,1) +n(1985,3283,19,0,0) +f(1969,13,19,3264) +f(1858,14,45,3158,2636,0,0) +u(1866,3158,2636,0,0) +f(797,16,2505,2) +u(2189) +u(1053) +u(509) +u(2005) +u(1805) +f(1098,16,2,129) +u(1114) +u(1130) +u(1138) +f(1105,16,129,522) +f(1106,17,35,487) u(1322) -u(1258) -u(1266) -u(1330) -u(1330) -f(554,7,28,2) -f(1722,8,1,1) -u(1754) -u(1250) -u(1242) -f(1513,5,1,2) -f(1554,6,1,1) -u(1050) -u(1050) -u(1379) -f(1633,4,1,13,1,0,0) -f(1601,5,1,12) -u(331,2) -u(1387) -f(883,8,1,1) -f(755,6,1,10) -f(949,7,2,8) -u(933) -u(813,7) -f(1989,10,1,6) -u(997) -u(2069) -u(2077) -f(1949,14,3,1) -u(837) -f(1965,14,1) -u(645) -f(1997,14,1) -f(2005,9,1) -f(2155,1,1) -u(1209) +f(1098,19,263,224) +u(1114) +u(1130) +u(1138) +f(1954,14,224,61,38,0,0) +u(1097,23) +u(1121) +u(1153) +u(1145) +u(1305) +u(1313) +u(11,1) +n(332,21) +u(124) +u(276) +u(268,20) +u(260,17) +u(236) +u(228) +u(300) +u(348,15) +f(36,30,1,1) +n(68) +n(1411,12) +u(196) +u(212,10) +u(204,2) +u(148,1) +u(2476) +u(2476) +u(860) +u(52) +f(156,34,1) +u(44) +f(212,33,1,8) +u(292,2) +f(140,35,1,1) +f(2468,34,1,6) +u(868) +u(2484) +f(2476,37,3,3) +u(860) +u(52) +u(340,1) +n(364,2) +u(380) +u(60) +u(308) +f(380,44,1,1) +u(1835) +f(220,32,1) +n(1779) +f(404,29,1) +n(1403) +u(188) +u(11) +f(396,25,1,3) +f(244,26,2,1) +u(20) +f(284,24,1) +f(1787,21,1) +f(1346,15,1,38) +u(1346) +u(1290) +u(1298) +u(1354) +u(1354) +f(1561,5,38,5,1,0,0) +f(1737,6,1,4) +u(1738) +u(1730) +f(1379,9,3,1) +f(1602,5,1) +u(1275) +f(1610,5,1,8) +f(1689,4,8,61,1,0,0) +f(1657,5,1,60) +u(171,2) +u(1395) +f(116,8,1,1) +f(635,6,1,56) +f(925,7,22,34) +u(901) +f(733,9,1,25) +f(517,10,3,2) +n(2157,20) +u(965,18) +f(2325,12,2,16) +u(1461,2) +f(765,14,1,1) +f(2333,13,1,14) +f(669,14,5,1) +u(1485) +f(2085,14,1,3) +u(765) +f(2109,14,3,4) +f(485,15,1,2) +f(709,16,1,1) +f(757,15,1) +u(1829) +f(2213,14,1) +u(685) +f(2037,11,1,2) +f(2165,9,2,8) +f(1395,6,8,2) +f(2443,1,2,1) +u(1756) search(); diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/put_19600rs_alloc.html similarity index 88% rename from src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs_alloc.html rename to src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/put_19600rs_alloc.html index 5f82fd3a7..79327e9e0 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs_alloc.html +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/put_19600rs_alloc.html @@ -262,14 +262,20 @@

Flame Graph

'1getBytes', '1substring', '0Latin1.newString', +'*Thread.run', +'4With', '*foreign/MemorySegment.ofArray', '*invoke/DirectMethodHandle.allocateInstance', -'1LambdaForm$DMH.0x00007b41eb092800.newInvokeSpecial', -'Flame Graph '5getHeader', '8Parameter', ':th', -'8RequiredParameter', '5trim', '/sponse.', '6toBytes', @@ -300,92 +305,107 @@

Flame Graph

'-Utf8.read', '2toAsciiString', ' ru.vk.itmo.dao.BaseEntry', -'"/vk/itmo/test/tyapuevdmitrij/ServerImplementation.put', -'?dao/MemorySegmentDao.upsert' +'+test.tyapuevdmitrij.dao.State', +'"/vk/itmo/test/tyapuevdmitrij/ServerImplementation.handleRequest', +'Tput', +'?dao/MemorySegmentDao$$Lambda.0x0000769253094000.run', +'S.lambda$flushing$2', +'Tupsert', +'CState.prepareToFlash' ]; unpack(cpool); -n(3,583) -u(361) -u(345) -u(249) -u(241) -u(225,340) -u(217) -u(9,302) -u(257,73) -u(273) -u(337,55) -u(369) -u(18) -f(353,10,55,18) -u(210) -f(313,8,18,28) +n(3,2068) +u(97,1) +u(105) +u(193) +u(201) +u(185) +u(177) +u(465) +u(473) +u(489) +u(442) +f(401,1,1,2067) +u(385) u(297) +u(289) +u(273,1224) +u(449) +u(265) +u(9,1113) +u(305,270) +u(321) +u(377,184) +u(409) +u(18) +f(393,11,184,86) +u(258) +f(345,9,86,104) u(81) u(81) u(89) -u(26,15) -n(129,13) -u(137) +u(26,51) +n(145,53) +u(153) u(18) -f(401,8,13,201) -u(73,14) +f(457,9,53,739) +u(73,53) u(57) u(65) u(18) -f(97,9,14,73) -u(177) -u(170,42) -n(185,31) -u(162) -f(202,9,31,8) -n(329,74) -u(18,30) -n(26,19) -n(34,25) -f(394,9,25,15) -n(409,17) -u(153) -u(145) -u(42,4) -n(50,13) -f(305,7,13,38) +f(113,10,53,321) +u(225) +u(218,192) +n(233,129) +u(210) +f(250,10,129,40) +n(369,194) +u(18,75) +n(26,51) +n(34,68) +f(434,10,68,45) +n(481,86) +u(169) +u(161) +u(42,21) +n(50,65) +f(353,8,65,111) u(81) u(89) -u(26,18) -n(129,20) -u(137) +u(26,50) +n(145,61) +u(153) u(18) -f(233,5,20,111) -u(194,21) -n(281,48) +f(281,5,61,360) +u(242,93) +n(329,149) u(34) -f(377,6,48,42) -u(377) -u(385) -u(18,24) -n(121,18) -u(113) -u(105) +f(417,6,149,118) +u(417) +u(425) +u(18,68) +n(137,50) +u(129) +u(121) u(26) -f(265,5,18,36) +f(313,5,50,114) u(18) -f(289,5,36,28) -u(321) +f(337,5,114,93) +u(361) u(81) u(89) -u(26,11) -n(129,17) -u(137) +u(26,50) +n(145,43) +u(153) u(18) -f(377,5,17,68) -u(377) -u(385) -u(18,38) -n(121,30) -u(113) -u(105) +f(417,5,43,276) +u(417) +u(425) +u(18,178) +n(137,98) +u(129) +u(121) u(26) search(); diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/put_19600rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/put_19600rs_cpu.html new file mode 100644 index 000000000..48b02e623 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage1/put_19600rs_cpu.html @@ -0,0 +1,1405 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_get_18100rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_get_18100rs_alloc.html new file mode 100644 index 000000000..07ce27fc4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_get_18100rs_alloc.html @@ -0,0 +1,589 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_get_18100rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_get_18100rs_cpu.html new file mode 100644 index 000000000..d2d04e38d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_get_18100rs_cpu.html @@ -0,0 +1,2688 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_put_53000rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_put_53000rs_alloc.html new file mode 100644 index 000000000..a909d7319 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_put_53000rs_alloc.html @@ -0,0 +1,476 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_put_53000rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_put_53000rs_cpu.html new file mode 100644 index 000000000..ac961eec6 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/ArrayQueue_sise64_put_53000rs_cpu.html @@ -0,0 +1,2168 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_get_17950rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_get_17950rs_alloc.html new file mode 100644 index 000000000..b43638eb1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_get_17950rs_alloc.html @@ -0,0 +1,612 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_get_17950rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_get_17950rs_cpu.html new file mode 100644 index 000000000..d93f53ec8 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_get_17950rs_cpu.html @@ -0,0 +1,3036 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/stage2/linkedBlockingQueue_size64_put_50050rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_put_50050rs_alloc.html similarity index 100% rename from src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/stage2/linkedBlockingQueue_size64_put_50050rs_alloc.html rename to src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_put_50050rs_alloc.html diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_put_50050rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_put_50050rs_cpu.html new file mode 100644 index 000000000..6e235f3c1 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/linkedBlockingQueue_size64_put_50050rs_cpu.html @@ -0,0 +1,2632 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue__put_2500rs_cpu.html similarity index 55% rename from src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs_cpu.html rename to src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue__put_2500rs_cpu.html index 69146301b..e816a6d47 100644 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/profiler_results/put_5000rs_cpu.html +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue__put_2500rs_cpu.html @@ -252,45 +252,56 @@

Flame Graph

const cpool = [ 'all', ' /usr/lib/libc.so.6', -' Compilation::Compilation', -'-compile_java_method', -'5method', -'-emit_code_body', -'&eBroker::compiler_thread_loop', -'/invoke_compiler_on_method', -'\'r::compile_method', -'"ncurrentGCThread::run', -' G1ConcurrentRefine::adjust_threads_periodically', -'2Thread::run_service', -'"Policy::estimate_used_young_bytes_locked', -'#rimaryConcurrentRefineThread::do_refinement_step', +' AccessInternal::PostRuntimeDispatch, (AccessInternal::BarrierType)3, 286822ul>::oop_access_barrier', +'`544868ul, G1BarrierSet>, (AccessInternal::BarrierType)2, 544868ul>::oop_access_barrier', +'b8964ul, G1BarrierSet>, (AccessInternal::BarrierType)2, 548964ul>::oop_access_barrier', +'!llocTracer::send_allocation_in_new_tlab', +' CodeHeap::find_blob', +' FastThreadsListHandle::FastThreadsListHandle', +' G1Allocator::unsafe_max_tlab_alloc', +'"EvacuateRegionsBaseTask::work', +'1Task::evacuate_live_objects', +'"ParEvacuateFollowersClosure::do_void', +'%ScanThreadState::steal_and_trim_queue', +'6trim_queue_to_threshold', ' HandleMark::pop_and_restore', ' InstanceKlass::allocate_instance', -' JavaFrameAnchor::make_walkable', -'$Thread::last_java_vframe', -',thread_main_inner', +'8objArray', +' JavaThread::threadObj', +'*ParkedState::JavaThreadParkedState', '$_one_nio_net_NativeSelector_epollWait', '8ocket_read', '>write', '!vmtiEnv::GetStackTrace', -'(Base::get_cthread_last_java_vframe', -'2stack_trace', '&xport::post_sampled_object_alloc', '%ObjectAllocEventCollector::generate_call_for_allocated', '%SampledObjectAllocEventCollector::~JvmtiSampledObjectAllocEventCollector', -' LIR_Assembler::logic_op', +' Klass::is_klass', ' MemAllocator::Allocation::notify_allocation_jvmti_sampler', '.allocate', -' ObjectSampler::recordAllocation', -'!ptoRuntime::new_array_C', +'.mem_allocate_inside_tlab_slow', +' ObjectMonitor::TrySpin', +'/enter', +'&Sampler::is_created', +'/recordAllocation', +'\'ynchronizer::enter', +'!ptoRuntime::is_deoptimized_caller_frame', +'-new_array_C', '7nozero_C', '1instance_C', -' Profiler::recordSample', -' RegisterMap::RegisterMap', -'"questHandler1_put.handleRequest', -'"sourceArea::rollback_to', +' Parker::park', +'(unpark', +'!rofiler::getNativeTrace', +'*recordSample', +' ResourceArea::rollback_to', +' SharedRuntime::complete_monitor_locking_C', ' Thread::call_run', +'&BlockInVMPreprocess::~ThreadBlockInVMPreprocess', +'&sListHandle::~ThreadsListHandle', '!ypeArrayKlass::allocate_common', +' Unsafe_Park', +'\'Unpark', +' WorkerThread::run', ' [vdso]', ' __alloc_pages', '(skb', @@ -298,9 +309,14 @@

Flame Graph

'#lock_gettime', '#opy_skb_header', '"dev_queue_xmit', -'#o_softirq', +'#o_fault', +'%softirq', '"fdget', +'#utex_queue', +'(unqueue', +'(wait', '"get_user_8', +'+nocheck_4', '"inet_lookup_established', '#p_local_out', '%queue_xmit', @@ -311,12 +327,14 @@

Flame Graph

'#etif_receive_skb_core.constprop.0', '4one_core', ')x', -'"pthread_mutex_unlock', +'"pthread_mutex_lock', +'0unlock', '#ut_user_nocheck_4', '38', '"rcu_read_lock', '+unlock', '#ecv', +'$serve_bytes?[btrfs]', '#seq_handle_notify_resume', '"schedule', '#k_dst_check', @@ -324,38 +342,47 @@

Flame Graph

'"tcp_push_pending_frames', '&select_window', '&transmit_skb', +'#ls_get_addr', '"usecs_to_jiffies', '"virt_addr_valid', '"wake_up_sync_key', '"x64_sys_epoll_wait', +'*futex', +'*openat', '*recvfrom', '*sendto', '!copy_from_iter', '&to_iter', +'!pthread_cleanup_pop', +'2ush', '!raw_spin_lock', '._bh', -'/irq', -'*unlock_irq', +'/irqsave', +'*unlock_bh', +'1irq', '4restore', '%write_unlock_irq', ' alloc_pages_mpol', '!sm_exc_page_fault', '$sysvec_apic_timer_interrupt', -' bpf_lsm_socket_recvmsg', -'/sock_rcv_skb', +' blkcg_maybe_throttle_current', +'!pf_skops_write_hdr_opt.isra.0', +'!trfs_delalloc_reserve_space?[btrfs]', +'&page_mkwrite?[btrfs]', +'&reserve_data_bytes?[btrfs]', ' check_bounds', '&stack_object', -'!lock_gettime@plt', -'!ompiledVFrame::sender', -'"nsume_stock', +'!lear_page_erms', +'"ock_gettime@plt', '!ubictcp_acked', -')cwnd_event', +')cong_avoid', +'*wnd_event', ' dev_hard_start_xmit', -'!o_anonymous_page', -'#softirq.part.0', +'!o_page_mkwrite', +'#shared_fault', +'$oftirq.part.0', '$yscall_64', '#user_addr_fault', -'!st_release', ' enqueue_to_backlog', '"try_SYSCALL_64_after_hwframe', '!p_done_scan', @@ -364,15 +391,29 @@

Flame Graph

'!th_type_trans', '!xc_page_fault', '"it_to_user_mode_prepare', -' finish_task_switch.isra.0', +' filemap_fault', +'"nish_task_switch.isra.0', +'!olio_alloc', '!put', +'!utex_get_value_locked', +'&hash', +'&q_lock', +'&unqueue', +'&wait', +'(ke', +'*_mark', +' get_futex_key', +'$page_from_freelist', +'#name_flags.part.0', ' handle_mm_fault', -' inet6_recvmsg', +' import_single_range', +'!net6_recvmsg', +'&sendmsg', '$_ehashfn', +'%send_prepare', '!p_finish_output', '02', '#local_deliver_finish', -'#output', '#protocol_deliver_rcu', '#rcv', '&_core', @@ -383,56 +424,70 @@

Flame Graph

'!s_vmalloc_addr', ' java/lang/String.charAt', '1indexOf', -'1regionMatches', '1startsWith', '2ubstring', '0Latin1.newString', -'7regionMatchesCI', '*Thread.run', '4With', '*foreign/MemorySegment.copy', +'@get', '@mismatch', '@ofArray', +'*invoke/DirectMethodHandle$Holder.newInvokeSpecial', +'C.allocateInstance', +'1Invokers$Holder.linkToTargetMethod', +'1VarHandleGuards.guard_LJ_I', +':SegmentAsBytes.get', +'IoffsetNoVMAlignCheck', '%util/Arrays.copyOfRange', -'*concurrent/ConcurrentSkipListMap.addIndices', +'*concurrent/ConcurrentSkipListMap$Iter.', +'Padvance', +'KValueIterator.', +'Ynext', +'Ps.iterator', +'J.addIndices', 'Kcpr', 'KdoPut', 'Kput', '5Executors$RunnableAdapter.call', '5FutureTask.run', +'5LinkedTransferQueue$DualNode.await', +'5SynchronousQueue$Transferer.xferLifo', +'E.offer', +'Ftake', +'Fxfer', '5ThreadPoolExecutor$Worker.run', -'G.runWorker', -'5locks/AbstractQueuedSynchronizer.acquireShared', -'WpparentlyFirstQueuedIsExclusive', -'VreleaseShared', +'G.execute', +'HgetTask', +'HrunWorker', +'5atomic/LongAdder.add', +'Fincrement', +'5locks/AbstractQueuedSynchronizer.releaseShared', 'VsignalNext', -';ReentrantReadWriteLock$NonfairSync.readerShouldBlock', -'RReadLock.lock', -'[unlock', -'RSync.tryAcquireShared', +';LockSupport.park', +'Gunpark', +';ReentrantReadWriteLock$ReadLock.unlock', '!byte_arraycopy', '&disjoint_arraycopy', '!dk/internal/foreign/AbstractMemorySegmentImpl.copy', 'Omismatch', '5HeapMemorySegmentImpl$OfByte.fromArray', '5MemorySessionImpl.checkValidStateRaw', -'GheapSession', '-misc/ScopedMemoryAccess.copyMemory', 'OInternal', +'2Unsafe.park', +'9unpark', '-util/ArraysSupport.mismatch', -'!int_disjoint_arraycopy', '!long_disjoint_arraycopy', '!ni_GetByteArrayRegion', '$SetByteArrayRegion', -'!short_disjoint_arraycopy', '!vmti_GetStackTrace', ' kfree_skbmem', '!mem_cache_alloc_node', '+free', '!time_get', ')_seconds', -' lock_sock_nested', -'"opback_xmit', +' loopback_xmit', ' mem_cgroup_charge_skmem', '+uncharge_skmem', '#move@plt', @@ -443,23 +498,19 @@

Flame Graph

'&unlock', ' napi_consume_skb', '!et_rx_action', +'#dev_core_pick_tx', '#if_rx_internal', -'&skb_features', '!f_conntrack_put', -' one/nio/http/HttpServer.handleRequest', -'3ssion.handleParsedRequest', +' one/nio/http/HttpSession.handleParsedRequest', '9parseRequest', ':rocessHttpBuffer', '@Read', '9sendResponse', '9writeResponse', -'-Request.', -'5getHeader', +'-Request.getHeader', '8Parameter', -'8RequiredParameter', '5trim', -'/sponse.', -'6toBytes', +'/sponse.toBytes', '(net/NativeSelector.epollWait', ';select', '3ocket.read', @@ -468,27 +519,46 @@

Flame Graph

'3.process', '4read', '4write', -'(server/SelectorThread.run', +'(server/PayloadThread.run', +'/SelectorThread.run', +'1rver.incRequestsProcessed', '(util/ByteArrayBuilder.append', '-URLEncoder.decode', '.tf8.isAsciiString', -'2length', '2read', '2startsWith', +'2toAsciiString', +'2write', '!opDesc* JNIHandles::resolve_impl<0ul, false>', +'!pen', '!s::javaTimeMillis', -' process_backlog', +' page_cache_ra_unbounded', +'!list_add', +'&del', +'!ost_alloc_hook', +'!rocess_backlog', +'!thread_cond_broadcast', +'-signal', +'-wait', +'(mutex_trylock', +'5@plt', +'.unlock@plt', ' raw_local_deliver', '$spin_rq_unlock_irqrestore.part.0', -'!b_insert_color', +'!b_erase', +'#insert_color', '!ead_tsc', +'"balance_domains', '"fill_stock', '"lease_sock', '"p_movs_alternative', -'!u/vk/itmo/test/tyapuevdmitrij/ServerImplementation.put', -'?dao/MemorySegmentComparator$$Lambda.0x00007b41eb08d648.compare', +'!u/vk/itmo/test/tyapuevdmitrij/ServerImplementation$$Lambda.0x000072ae44086a00.run', +'S.handleRequest', +'Tlambda$handleRequest$0', +'Tput', +'?dao/MemorySegmentComparator$$Lambda.0x000072ae4401c7f8.compare', 'Z.lambda$getMemorySegmentComparator$0', -'PDao$$Lambda.0x00007b41eb08e9f0.run', +'PDao$$Lambda.0x000072ae4408ee78.run', 'S.lambda$flushing$2', 'Tupsert', 'CStorage.save', @@ -496,60 +566,62 @@

Flame Graph

'JHelper.getSsTableDataByteSize', '"n_rebalance_domains', ' schedule', -'(_hrtimeout_range', -'8_clock', -'!ecurity_sock_rcv_skb', -'-et_recvmsg', +'(_hrtimeout_range_clock', +'!ecurity_socket_recvmsg', '0sendmsg', '"nd', +'$@plt', '!k_filter_trim_cap', -'$ree', +'$orced_mem_schedule', '#page_frag_refill', '#reset_timer', '"b_attempt_defer_free', '$clone', '%opy_datagram_iter', -'$network_protocol', -'$push', +'$free_head', '$release_data', '!lab_post_alloc_hook.constprop.0', '!ock_def_readable', '%poll', +'&ut', '%recvmsg', '&free', +'!trncpy_from_user', '!yscall_enter_from_user_mode', ')xit_to_user_mode', '9_prepare', '#vec_apic_timer_interrupt', ' tcp_ack', '$check_space', -'&rono_start', -'%leanup_rbuf', +'&rono_stop', '%urrent_mss', -'$do_parse_auth_options', +'$data_ready', +'%o_parse_auth_options', +'$event_data_recv', '$inbound_hash.constprop.0', '$mstamp_refresh', '$newly_delivered', -'$options_write', '$poll', '$queue_rcv', '$rack_advance', ')update_reo_wnd', '&te_check_app_limited', -')skb_delivered', +')gen', +')skb_sent', '%btree_insert', '%cv_established', '(space_adjust', -'%ecvmsg', +'%earm_rto', +'&cvmsg', '+_locked', '&lease_cb', '$schedule_loss_probe', '%endmsg', '+_locked', '%kb_entail', +'%mall_queue_check.isra.0', '%tream_alloc_skb', '+memory_free', -'$update_skb_after_send', '$v4_do_rcv', '\'fill_cb', '\'rcv', @@ -558,452 +630,569 @@

Flame Graph

'%rite_xmit', '!hread_native_entry', '!ry_charge_memcg', -' unknown_Java', +'$to_wake_up', +' unknown', +'\'_Java', '"safe_arraycopy', ' validate_xmit_skb', -'!frame::java_sender', -'(new_vframe', -'!ma_alloc_folio', -'!oid AccessInternal::arraycopy_conjoint' +'!oid AccessInternal::arraycopy_conjoint', +'%G1ScanEvacuatedObjClosure::do_oop_work', +'%OopOopIterateBackwardsDispatch::Table::oop_oop_iterate_backwards', +' wake_up_q' ]; unpack(cpool); -n(3,1427) -u(11,5) -f(861,2,1,1) -u(829) -u(2149) -u(909) -u(541) -u(405) -f(2451,2,1,3) -u(316) -u(76,1) -u(92) -u(108) +n(3,1783) +u(11,11) +f(3027,2,3,8) +u(364) +u(412) +u(76) u(84) +u(92) u(100) -f(148,4,1) -u(52) -u(60) -u(68) -u(20) -u(36) -u(28) -u(44) -u(228) -f(491,4,1) -f(1105,1,1,12) -u(1113) -u(1201) -u(1209) -u(1193) -u(1185) -u(1905) -u(1913) -u(1929) -u(1937,8) -u(1122,6) -u(1122) -u(1298) -u(1338) -u(1346) -u(1322) -f(1291,11,6,1) -n(1395) -f(1945,10,1,4) -f(1745,1,4,1409) -f(363,2,3,2) -u(331) -f(1689,2,2,255,1,0,0) -u(1681,255,1,0,0) -f(11,4,11,4) -n(155,7) -n(331,5) -n(363,14) -f(331,5,9,5) -f(763,4,5,2) -n(883,212) -f(861,5,48,164) -u(829) -f(621,7,1,137) -f(397,8,9,6) -n(877,122) -f(501,9,25,2) -n(509,1) -n(701,3) -n(869,2) -n(1517,7) -n(1525,4) -n(1973,3) -n(1981,53) -f(1965,10,9,44) -f(549,11,5,39) -f(917,12,3,36) -f(2117,9,36,22) -f(2253,10,10,12) -f(2389,11,11,1) -f(2141,7,1,12) -n(2149,14) -u(909,12) -f(541,9,4,8) -f(405,10,3,5) -f(2157,8,5,2) -f(1721,2,2,1149,20,0,0) -u(1601,1149,20,0,0) -f(1593,4,1,912) -f(1577,5,15,851,64,0,0) -u(1569,851,64,0,0) -u(297,851,64,0,0) -f(1283,8,34,3) -n(1291,6) -n(1371,2) -n(1609,594) -f(1617,9,1,591,15,0,0) -f(1673,10,2,22) -f(1754,11,10,12) +u(108) +f(3100,10,6,2) +u(3092) +f(1417,1,2,10) +u(1425) +u(1649) +u(1673) +u(1601) +u(1593) +u(2465) +u(2473) +u(2489) +u(2497,7) +u(1434,5) +u(1434) +u(1754) +u(1786) +u(1794) u(1778) -u(1050) -f(1737,10,12,567,13,0,0) -f(331,11,3,1) -n(363,4) -n(763,1) -n(1737,558,4,0,0) -f(1713,12,2,556,2,0,0) -f(1705,13,1,555,1,0,0) -f(171,14,11,7) -f(1379,15,1,5) -f(116,16,3,1) -n(747) -u(308) -f(2516,15,1) -f(2011,14,1,537) -f(861,15,32,505) -u(829) -f(637,17,2,483) -f(397,18,4,3) -n(925,1) -n(2005,3) -n(2357,472) -f(445,19,1,1) -n(1869,2) -f(669,20,1,1) -f(2365,19,1,468) -f(357,20,11,3) -f(605,21,2,1) -f(573,20,1,404) -f(2445,21,1,403) -f(589,22,25,368) -f(429,23,10,343) -f(421,24,5,1) -n(525) -n(557,6) -u(1021) -f(957,24,6,7) -n(965,322) -f(381,25,5,317) -f(445,26,14,268) -f(821,27,2,266) -f(389,28,4,262) -f(1541,29,9,253) -f(461,30,7,227) -u(1821) -u(477,223) -f(469,33,1,13) -n(973,197) -f(525,34,1,1) -n(989,195) -f(1829,35,3,5) -f(517,36,4,1) -f(2421,35,1,187) -f(413,36,13,16) -f(949,37,12,4) -f(1565,36,4,1) -n(2021,10) -f(1989,37,4,6) -f(741,38,3,3) -f(2221,36,3,4) -f(2213,37,2,2) -f(2405,36,2,137) -f(2309,37,1,136) -f(845,38,6,1) -n(1445,2) -n(2045,6) -u(1509) -f(693,40,4,2) -f(2109,38,2,66) -f(613,39,2,1) -n(693,63) -f(2173,38,63,44) -f(437,39,23,4) -u(2093) -f(597,39,4,2) -n(789,3) -n(1413,1) -n(2237,3) -n(2269,2) -n(2277) -n(2293,3) -n(2349,1) -f(2181,38,1,4) -n(2229,2) -u(1437) -f(1853,40,1,1) -f(2261,38,1,5) -f(2413,36,5,6) -f(453,37,4,2) -f(997,33,2,12) -f(1005,34,2,4) -n(1013,6) -f(525,32,6,1) -n(677) -n(685,2) -f(661,30,2,1) -n(1413,5) -n(1429,4) -n(1533,9) -f(2093,31,2,7) -f(1429,32,5,2) -f(805,26,2,21) -f(1461,27,6,15) -f(485,28,3,8) -u(1549) -f(853,30,1,7) -f(693,31,1,6) -f(893,28,6,2) -n(2029,1) -n(2437) -f(2485,26,1,14) -f(1557,27,10,4) -f(2077,28,3,1) -f(981,24,1) -f(581,23,1) -n(797) -n(2061,7) -f(373,24,5,2) -f(2085,23,2,1) -n(2245,2) -n(2397,1) -n(2429,2) -f(597,22,2,1) -n(1437,3) -f(1853,23,2,1) -f(1845,22,1,2) -n(2045,3) -u(1509) -f(2301,22,3,1) -f(645,20,1,2) -f(1877,21,1,1) -f(2037,20,1,2) -n(2205,11) -f(1029,21,8,3) -f(2285,20,3,2) -n(2373,3) -f(2189,21,2,1) -f(2381,20,1,30) -f(349,21,2,22) -f(1421,22,3,19) -f(2101,23,14,5) -u(1493) -f(1469,21,5,6) -f(1501,22,2,3) -n(2461,1) -u(781) -f(2141,17,1,17) -n(2149,3) -u(2157) -f(1633,9,3,2) -f(1066,10,1,1) -u(1098) -f(1649,8,1,24) -f(1642,9,2,22,17,0,0) -f(1074,10,3,7) -u(1354) -f(725,12,6,1) -u(2165) +f(1554,11,5,1) +u(1538) +u(1522) +u(909) +u(1117) +u(1061) +u(1245) +u(1037) +u(477) +u(1133) +u(2261) +u(1149) +u(901) +u(429) +u(1229) +u(2285) +u(981) +f(1747,11,1) +u(909) +u(1117) +u(1061) +u(1245) u(1037) -u(389) -u(1957) -u(1837) -f(1082,10,1,3) -u(1082) -u(1090) -f(1291,10,3,4) -n(1761,5) -u(1058) -u(1058) -f(1881,8,5,187,18,0,0) -u(1137,1) -u(1313) -u(1329) +u(1029) +u(949) +u(941) +u(957) +u(677) +f(2505,10,1,3) +f(1546,11,1,2) +u(1530) +f(2153,1,2,1140) +u(1417) +u(1425) +u(1649) +u(1673) +u(1665,357) +u(1633) +u(1641) +f(1617,9,1,356,13,0,0) +f(1609,10,4,350,7,0,0) +f(1713,11,5,345,2,0,0) +f(1801,12,2,343) +f(11,13,10,135) +f(1077,14,53,82) +u(1053) +f(781,16,1,57) +f(1197,17,7,50) +f(517,18,5,45) +f(501,19,4,1) +u(2269) +f(1165,19,1,3) +f(533,20,1,2) +f(1181,19,2,5) +f(845,20,3,1) +n(1173) +f(1189,19,1,4) +n(2525,28) +f(693,20,1,27) +f(1141,21,2,25) +f(2701,16,25,14) +n(2709,10) +u(1125,8) +f(685,18,1,7) +f(525,19,4,3) +f(2717,17,3,2) +f(36,13,2,6) +n(140,2) +n(395,177) +f(11,14,5,133) +f(1077,15,43,90) +u(1053) +f(781,17,2,18) +f(1205,18,12,6) +f(1173,19,5,1) +f(2701,17,1,70) +f(20,14,70,12) +n(140,1) +n(148,2) +n(316,10) +f(28,15,5,2) +n(372,3) +f(627,14,3,7) +n(2315,1) +n(2323,4) +n(2331,1) +n(2339) +f(827,13,1) +n(835,3) +n(2315,7) +n(2323,2) +f(1722,10,2) +f(2417,6,2,783,9,0,0) +f(1641,7,9,1) +n(2433,773) +f(1827,8,6,7) +n(2041,575) +f(2049,9,1,557,8,0,0) +u(2081,19) +f(2178,11,9,10) +u(2226) +u(2226) +u(1378) +f(2145,10,10,538,8,0,0) +u(2145,538,8,0,0) +f(2121,12,1,537,7,0,0) +f(2113,13,6,531,1,0,0) +f(11,14,7,5) +n(171,17) +f(11,15,4,5) +n(1835,6) +f(963,16,4,2) +u(348) +f(1923,15,2,1) +n(2236) +f(2555,14,1,500) +f(1077,15,29,471) +u(1053) +f(805,17,3,458) +f(493,18,9,15) +f(653,19,14,1) +f(1253,18,1) +n(1269,4) +f(1285,19,1,3) +f(2549,18,3,2) +n(2933,427) +u(573,2) +n(869,1) +n(2405,3) +f(853,20,1,1) +n(2917) +f(2941,19,1,421) +f(445,20,14,7) +f(757,21,1,5) +n(1373,1) +f(717,20,1,347) +f(3021,21,3,344) +f(733,22,20,312) +f(557,23,12,284) +f(549,24,5,3) +n(661,1) +n(701,2) +f(1349,25,1,1) +f(1293,24,1,8) +n(1301,265) +f(469,25,18,247) +f(573,26,13,214) +f(1045,27,3,211) +f(485,28,6,205) +f(1981,29,6,199) +f(589,30,7,177) +f(2293,31,2,175) +f(605,32,1,173) +f(597,33,2,12) +n(1309,150) +f(661,34,1,1) +n(1317,148) +f(2349,35,3,1) +n(2997,144) +f(541,36,6,13) +f(1277,37,9,4) +f(2005,36,4,1) +n(2573,7) +f(661,37,6,1) +f(2669,36,1,2) +n(2789,3) +f(2773,37,1,2) +f(2981,36,2,107) +u(1349,1) +n(2877,106) +f(661,38,4,1) +n(1893,2) +n(2597,8) +u(1949) +f(861,40,1,1) +n(885,6) +f(2653,38,6,50) +u(765,2) +f(861,40,1,1) +f(885,39,1,48) +f(2733,38,48,29) +f(565,39,11,2) +f(2637,40,1,1) +f(749,39,1) +n(997,3) +n(1005) +n(2365,1) +n(2749) +n(2805,2) +n(2829,1) +n(2837,2) +n(2853,1) +n(2925) +f(2741,38,1,5) +n(2765,1) +n(2781,4) +n(2821,2) +f(2989,36,2,5) +f(581,37,1,4) +f(1325,33,4,9) +f(1333,34,1,1) +n(1341,7) +f(877,32,7,1) +f(749,30,1) +n(845) +n(1861) +n(1877,2) +n(1973,10) +f(2637,31,2,8) +f(1877,32,5,2) +n(2629,1) +f(1021,26,1,18) +f(1901,27,1,17) +f(613,28,5,7) +u(1997) +f(1069,30,1,6) +f(861,31,2,1) +n(885,3) +f(1109,28,3) +n(3013,2) +f(1989,26,2,1) +n(3077) +f(725,23,1,2) +n(933,1) +n(1013,4) +n(2613,7) +f(461,24,4,3) +f(2861,23,3,1) +n(3005) +f(749,22,1) +n(2373) +n(2597) +u(1949) +f(2741,22,1,2) +n(2869,1) +n(2893) +n(2957,5) +f(813,20,5,3) +n(2589) +f(917,21,2,1) +u(2725) +u(1365) +u(485) +u(2389) +f(2757,20,1,9) +f(1357,21,5,4) +f(2845,20,4,1) +n(2949,2) +n(2965,34) +f(437,21,1,22) +f(1869,22,1,21) +f(2645,23,14,7) +f(1933,24,3,4) +f(1909,21,4,9) +f(1941,22,1,7) +n(3037,1) +f(2581,21,1,2) +f(2973,20,2,1) +f(2701,17,1,10) +f(2563,14,10,2) +f(2058,9,2,4) +n(2170,13,12,0,0) +u(1690,13,12,0,0) +u(1682,13,12,0,0) +f(356,12,2,1) u(276) -u(124) -u(244) -u(236) -u(220) -u(212) -f(1665,9,1) -u(260) -u(324) +u(252) u(244) -u(236) +f(419,12,1,6) +n(451,3) +f(419,13,2,1) +f(987,12,1) +f(2065,8,1,20) +f(1386,9,6,1) +n(1394) +u(1818) +f(1402,9,1) +u(1402) +u(1410) +f(1739,9,1) +n(1747,5) +n(2185) +f(1386,10,2,3) +u(1386) +f(2441,8,3,165) +u(1457,1) +u(1769) +u(308) +u(124) +u(228) u(220) -u(212) u(204) -u(252) -u(284) -u(1403) -u(180) u(196) u(188) -u(140) -u(2500) -f(1921,9,1,185,18,0,0) -u(1177,168,1,0,0) -f(1169,11,1,167) -f(1153,12,72,18) -f(1153,13,4,5) -u(1153,2) -u(1153,1) -u(1153) -u(1162) -u(1890) -u(1898) -u(1130) -u(1130) -u(1306) -f(1162,15,1) -u(1890) -u(1898) -u(1130) -u(1130) -u(1306) -f(1162,14,1,3) -u(1890) -u(1898) -u(1130) -u(1130) -u(1306) -f(1162,13,3,9) -u(1890) -u(1898) -u(1130) -u(1130) -u(1306) -f(1161,12,9,77) -f(1890,13,1,76) -u(1898) -u(1130) -u(1130) -u(1306) -f(1258,10,76,16) -u(1218) -u(1274) -u(1250) -u(1226) -f(1266,10,16,1) -u(1234) -u(1242) -f(2475,8,1) -f(1585,5,1,25) -f(1626,6,5,11) -u(1058) -u(1058) -f(1283,9,4,1) -n(1363,2) -n(2475,4) -f(1794,6,4,9) -f(1633,5,9,19) -f(1066,6,5,8) -u(1098) -f(1658,6,8,6,5,0,0) -u(1050,5) -f(1291,8,3,2) -f(1081,7,2,1) -u(1089) -u(1145) u(268) -u(324) -u(244) -u(236) +u(340) +u(1851) +u(180) +u(2299) +f(2481,9,1,164,21,0,0) +u(1585,161,18,0,0) +f(1577,11,15,143) +f(1561,12,6,16) +u(1561,3) +f(1561,14,1,1) +u(1569) +u(2449) +u(2457) +u(1449) +u(1450) +u(1762) +f(1570,14,1) +u(2450) +u(2458) +f(1570,13,1,13,8,0,0) +u(2450,13,8,0,0) +u(2458,13,8,0,0) +f(1442,16,4,4) +u(1490) +u(1498) +u(1506) +f(1449,16,4,5) +u(1450) +u(1762) +f(1570,12,5,121,76,0,0) +u(2450,121,76,0,0) +u(2458,121,76,0,0) +f(1442,15,69,7) +u(1490) +u(1498) +u(1506) +f(1449,15,7,45) +u(1450) +u(1762) +f(1827,11,45,3) +f(1730,10,3) +u(1698) +u(1706) +f(1747,13,2,1) +f(2161,1,1,612) +f(419,2,3,1) +n(451,3) +f(419,3,2,1) +f(2097,2,1,259,1,0,0) +u(2089,259,1,0,0) +f(11,4,14,17) +n(155,7) +n(419,3) +n(451,28) +f(419,5,22,6) +f(987,4,6,5) +n(1099,185) +f(917,5,64,1) +u(2725) +u(1365) +u(485) +u(2517) +u(2357) +f(1077,5,1,120) +u(1053) +f(773,7,4,88) +f(493,8,6,1) +n(1093,80) +f(637,9,16,4) +n(645,1) +n(893,5) +n(1085,2) +n(1957,3) +n(1965,2) +n(2533,33) +f(2525,10,2,31) +f(693,11,3,28) +f(1141,12,1,27) +f(2661,9,27,14) +f(2813,10,7,7) +f(2973,11,5,2) +f(1157,8,2,1) +f(2701,7,1,15) +n(2709,13) +f(1125,8,1,12) +f(685,9,1,8) +f(525,10,3,5) +f(925,9,5,3) +f(2129,2,3,346,23,0,0) +u(2033,346,23,0,0) +u(2025,175) +f(2009,5,6,130,4,0,0) +u(2425,130,4,0,0) +u(1481,1) +u(1465) +u(1473) +u(292) +u(132) +u(228) u(220) -u(212) u(204) -u(252) -u(284) -u(1403) -u(180) -u(196) u(196) -u(2492) -u(772) -u(292) -u(717) -u(901) -u(837) -u(933) -u(813) -u(2509) -u(709) -u(341) -f(1786,5,1,2) -u(1786) -u(1770) -f(1729,4,2,236,19,0,0) -f(1697,5,19,217) +u(188) +u(268) +u(332) +f(1657,7,1,129,4,0,0) +u(1625,129,4,0,0) +f(1641,9,1,125) +u(1617,125,10,0,0) +f(1721,11,4,121,6,0,0) +f(1809,12,5,116,1,0,0) +f(60,13,3,1) +n(324,2) +n(380,1) +n(403,21) +f(11,14,1,4) +n(60,2) +n(380,7) +n(619,4) +n(739,3) +f(2307,13,3,88) +f(1077,14,24,64) +u(1053) +u(781,53) +f(1205,17,3,50) +f(1213,18,9,6) +f(509,19,3,3) +f(2277,20,1,2) +f(1221,18,2,3) +n(3109,32) +f(3045,19,2,30) +u(885) +f(2701,16,30,11) +f(1747,9,11,3) +f(2018,5,3,6,4,0,0) +f(308,6,1,1) +u(124) +u(228) +u(236) +u(68) +f(2202,6,1,2,1,0,0) +u(2202,2,1,0,0) +u(2194,1) +n(2217) +u(308) +u(124) +u(228) +u(44) +u(260) +f(2210,6,1,2) +f(2058,5,2,30,25,0,0) +f(1739,6,18,2) +n(1747,3) +n(2073,5) +f(1402,7,2,3,1,0,0) +u(1409,2) +u(1513) +u(300) +u(284,1) +u(52) +f(388,11,1) +u(228) +u(236) +u(68) +f(1739,8,1) +f(3067,6,1,2) +f(2202,5,2,3) +u(2202) +u(2194) +f(2137,4,3,171,23,0,0) +f(2105,5,23,148) f(11,6,6,3) -n(163,31) -f(11,7,2,8) +n(163,23) +u(11,7) n(116,1) -n(1387,15) -f(116,8,5,1) -n(132) -n(747,8) -f(308,9,3,5) -f(1483,7,5,3) -n(1804,2) -f(531,6,2,173) -f(861,7,37,136) -u(829) -f(629,9,2,101) -f(397,10,6,11) -f(517,11,9,2) -f(925,10,2,1) -n(2125,83) -f(941,11,2,78) -f(2325,12,4,74) -f(445,13,1,1) -n(1453) -u(669) -f(1869,13,1,3) -f(2341,14,2,1) -f(2333,13,1,68) -f(565,14,14,10) -f(1477,15,2,6) -n(1861,2) -f(2053,14,2,3) -f(669,15,2,1) -f(2069,14,1,29) -f(357,15,6,18) -f(605,16,9,5) -n(757,1) -n(1045,3) -f(653,15,3,5) -f(1877,16,2,3) -f(2133,14,3) -n(2197) -n(2317,6) -f(2229,15,1,5) -f(1437,16,2,3) -f(1853,17,1,2) -f(1997,11,2,3) -f(733,12,2,1) -f(2141,9,1,32) -n(2149,1) -u(2157) -f(1387,6,1,4) -f(2467,1,4,1) -u(1812) +n(1843,13) +f(116,8,2,2) +n(348) +n(963,7) +f(348,9,1,6) +f(3084,7,6,2) +f(667,6,2,116) +f(1077,7,20,96) +u(1053) +u(797,74) +f(493,10,7,10) +n(1253,2) +n(2677,55) +f(1261,11,2,51) +f(2901,12,1,50) +f(573,13,1,1) +n(869,2) +n(2405,1) +n(2909,45) +f(709,14,8,8) +u(1917,7) +n(2397,1) +f(2605,14,1) +u(853) +f(2621,14,1,22) +f(445,15,4,12) +f(757,16,1,7) +n(973,2) +n(1373) +f(821,15,2,6) +f(2413,16,3,3) +f(2685,14,3) +n(2885) +u(2797) +u(1885) +f(2381,17,2,1) +f(2541,11,1,2) +f(2701,9,2,22) +f(3051,1,22,1) +u(2243) +u(1077) +u(1053) +u(789) +u(1237) +u(2693) +f(3059,1,1,9) +u(2089,2) +n(2252) +n(2441) +n(3051,3) +u(212) +u(3051) +u(2252) search(); diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2000rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2000rs_alloc.html new file mode 100644 index 000000000..68cf19aaa --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2000rs_alloc.html @@ -0,0 +1,604 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2000rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2000rs_cpu.html new file mode 100644 index 000000000..b32af111c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2000rs_cpu.html @@ -0,0 +1,1649 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2500rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2500rs_alloc.html new file mode 100644 index 000000000..e44cbe19a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage2/synchronousQueue_put_2500rs_alloc.html @@ -0,0 +1,419 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_alloc.html new file mode 100644 index 000000000..4187d8dcd --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_alloc.html @@ -0,0 +1,3352 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_cpu.html new file mode 100644 index 000000000..5ab41db56 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_cpu.html @@ -0,0 +1,11619 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_lock.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_lock.html new file mode 100644 index 000000000..c4843ce9c --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/get_16100rs_lock.html @@ -0,0 +1,1012 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_alloc.html new file mode 100644 index 000000000..7b597b420 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_alloc.html @@ -0,0 +1,3238 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_cpu.html new file mode 100644 index 000000000..2d442d95f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_cpu.html @@ -0,0 +1,10302 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_lock.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_lock.html new file mode 100644 index 000000000..c0d6da365 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage3/put_16600rs_lock.html @@ -0,0 +1,959 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_alloc.html new file mode 100644 index 000000000..8cb3e9605 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_alloc.html @@ -0,0 +1,3634 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_cpu.html new file mode 100644 index 000000000..6c111d4b3 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_cpu.html @@ -0,0 +1,8936 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_lock.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_lock.html new file mode 100644 index 000000000..cc011ba55 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/get_5400rs_1min_lock.html @@ -0,0 +1,1064 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_alloc.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_alloc.html new file mode 100644 index 000000000..96f724f91 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_alloc.html @@ -0,0 +1,3321 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_cpu.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_cpu.html new file mode 100644 index 000000000..bb0cc6111 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_cpu.html @@ -0,0 +1,8494 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_lock.html b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_lock.html new file mode 100644 index 000000000..cae1e3714 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/profiler_results/stage4/put_6800rs_1min_lock.html @@ -0,0 +1,952 @@ + + + + + + + +

Flame Graph

+
  
+
Produced by async-profiler
+ +
+

+

Matched:

+ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/get_16100rs_1min.png b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/get_16100rs_1min.png new file mode 100644 index 000000000..a839df119 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/get_16100rs_1min.png differ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/put_16600rs_1min.png b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/put_16600rs_1min.png new file mode 100644 index 000000000..a23c1e3c6 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/put_16600rs_1min.png differ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/report.md b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/report.md new file mode 100644 index 000000000..8c767ad42 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage3/report.md @@ -0,0 +1,119 @@ +## put + +1. 16600 запросов в секунду в течение 1 минуты: + Найдена точка разладки. Latency всех запросов не превышает 100 мс. + На последнем персентиль наблюдается резкое возрастание latency. + ![put_16600rs_1min.png](put_16600rs_1min.png) + +2. 16700 запросов в секунду в течение 1 минуты: + +``` +90.000% 47.20ms +99.000% 148.48ms +99.900% 199.93ms +99.990% 230.27ms +99.999% 240.77ms +100.000% 245.12ms +``` +На последних 10 персентиль наблюдается скачек latency, время выполнения всех запросов уже превышает 100мс. + +### cpu profiling + +На обработку перенаправления запроса уходит 20% ресурсов процессора, 11 % на SelectorManager и 39% на работу сетевых +потоков. +![cpu](../profiler_results/stage3/put_16600rs_cpu.html) + +### alloc profiling + +![alloc](../profiler_results/stage3/put_16600rs_alloc.html) +Аллокации из-за поиска необходимой ноды занимают менее 1% (конкатенация строк). +Появились 54 процента новых аллокаций из-за перенаправления запросов по http и работы httpClient, 26% - работа сетевых +потоков и 4% selectorManager. Итого 85% новых аллокаций. + +### lock profiling + +![lock](../profiler_results/stage3/put_16600rs_lock.html) +Большую часть находятся в ожидании сетевые потоки (SequentialScheduler) - 50% и selectorManager - 14% + +## get + +1. 16100 запросов в секунду в течение 1 минуты: + Найдена точка разладки. Latency всех запросов не превышает 100 мс. + На последнем персентиль наблюдается резкое возрастание latency. + ![get_16100rs_1min.png](get_16100rs_1min.png) + +2. 16200 запросов в секунду в течение 1 минуты: + +``` +75.000% 5.92ms +90.000% 152.32ms +99.000% 544.26ms +99.900% 602.11ms +99.990% 618.49ms +99.999% 623.10ms +100.000% 625.66ms +``` +На последних 20 персентиль наблюдается резкий скачёк latency, время выполнения всех запросов уже значительно +превышает 100мс. + +### cpu profiling + +![cpu](../profiler_results/stage3/get_16100rs_cpu.html) +На обработку перенаправления запроса уходит 15% ресурсов процессора, 9 % на SelectorManager и 28% на работу сетевых +потоков. + +### alloc profiling + +![alloc](../profiler_results/stage3/get_16100rs_alloc.html) +Аллокации из-за поиска необходимой ноды занимают менее 0.5% (конкатенация строк). +Появились 23 процента новых аллокаций из-за перенаправления запросов по http и работы httpClient, 10% - работа сетевых +потоков и 2% selectorManager. Итого 35% новых аллокаций. + +### lock profiling + +![lock](../profiler_results/stage3/get_16100rs_lock.html) +Большую часть находятся в ожидании сетевые потоки (SequentialScheduler) - 52% и selectorManager - 15% + +## Сравнение с прошлым этапом + +### PUT +Точка разладки сместилась на 4000 запросов в секунду в меньшую сторону по сравнению с прошлым этапом. +Возьмем и замерим результаты реализации прошлого этапа в точке разладки этого: + +``` + было стало +50.000 1.24ms 1.97ms +75.000 1.71ms 2.79ms +90.000 2.18ms 5.06ms +99.000 7.31ms 21.85ms +99.900 11.80ms 48.90ms +99.990 15.29ms 69.69ms +99.999 19.26ms 76.86ms +100.000 22.14ms 79.04ms +``` + + +### GET +Точка разладки сместилась на 2000 запросов в секунду в меньшую сторону по сравнению с прошлым этапом. +Возьмем и замерим результаты реализации прошлого этапа в точке разладки этого: + +``` + было стало +50.000 1.53ms 2.48ms +75.000 2.13ms 4.03ms +90.000 2.74ms 7.20ms +99.000 3.72ms 49.53ms +99.900 20.13ms 83.84ms +99.990% 29.90ms 88.45ms +99.999% 35.42ms 90.69ms +100.000% 38.65ms 92.16ms +``` +## Выводы +Текущая реализация стала держать меньше get и put-запросов и стала медленнее. Это можно объяснить появлением большого +количества новых аллокаций и блокировок из-за сетевого взаимодействия нод, что приводит к необходимости тратить как +минимум треть ресурсов процессора на это. Использование рандеву-хэширования не вносит явных замедлений в систему. +Использование хэш-функции murmur3 помогает относительно равномерно распределять данные по нодам в отличие от +стандартного hashcode() (с ним одна из трех нод всегда пустая). + + + diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/get_5400rs_1min.png b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/get_5400rs_1min.png new file mode 100644 index 000000000..09a5bb68c Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/get_5400rs_1min.png differ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/put_6800rs_1min.png b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/put_6800rs_1min.png new file mode 100644 index 000000000..2bef46115 Binary files /dev/null and b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/put_6800rs_1min.png differ diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/report.md b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/report.md new file mode 100644 index 000000000..dbeea0c7a --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/reports/stage4/report.md @@ -0,0 +1,127 @@ +## Описание исследований +Все исследования проводились для кластера из 3 нод. На целевую ноду поступали запросы без указания ack и from. +Таким образом ack = 2 и from = Это симулирует ситуацию, когда пользователю максимально важно сохранить данные и получить +самые свежие из них + +## put + +1. 6800 запросов в секунду в течение 1 минуты: + Найдена точка разладки. Latency всех запросов не превышает 100 мс. + На последних 10 персентиль наблюдается резкое возрастание latency. + ![put_6800rs_1min.png](put_6800rs_1min.png) + +2. 6900 запросов в секунду в течение 1 минуты: + +``` + 50.000% 6.01ms + 75.000% 35.74ms + 90.000% 60.51ms + 99.000% 89.41ms + 99.900% 102.14ms + 99.990% 109.25ms + 99.999% 115.78ms +100.000% 117.44ms +``` +На последнем персентиль наблюдается скачек latency, время выполнения всех запросов уже превышает 100мс. + +### cpu profiling + +На обработку перенаправления запроса уходит 20% ресурсов процессора, 11 % на SelectorManager и 39% на работу сетевых +потоков. +![cpu](../profiler_results/stage4/put_6800rs_1min_cpu.html) + +### alloc profiling + +![alloc](../profiler_results/stage4/put_6800rs_1min_alloc.html) +Аллокации из-за поиска необходимых нод по максимальному хэшу занимают менее 1% (конкатенация строк и создание treeMap). +Аллокации из-за перенаправления запросов по http и работы httpClient и сборке всех ответов - 58% , 27% - работа сетевых +потоков и 5% selectorManager. + +### lock profiling + +![lock](../profiler_results/stage4/put_6800rs_1min_lock.html) +Большую часть находятся в ожидании сетевые потоки (SequentialScheduler) - 52% и selectorManager - 14% + +## get + +1. 5400 запросов в секунду в течение 1 минуты: + Найдена точка разладки. Latency всех запросов не превышает 100 мс. + На последних 10 персентиль наблюдается резкое возрастание latency. + ![get_5400rs_1min.png](get_5400rs_1min.png) + +2. 5500 запросов в секунду в течение 1 минуты: + +``` + 50.000% 91.78ms + 75.000% 143.87ms + 90.000% 205.70ms + 99.000% 255.23ms + 99.900% 281.09ms + 99.990% 291.58ms + 99.999% 294.91ms +100.000% 295.68ms +``` +На последних 25 персентиль наблюдается резкий скачёк latency, время выполнения всех запросов уже значительно +превышает 100мс. + +### cpu profiling + +![cpu](../profiler_results/stage4/get_5400rs_1min_cpu.html) +На обработку перенаправления запроса уходит 17% ресурсов процессора при этом 25% тратится на сбор всех ответов на +целевой ноде, 10 % на SelectorManager и 30% на работу сетевых потоков. + +### alloc profiling + +![alloc](../profiler_results/stage4/get_5400rs_1min_alloc.html) +Аллокации из-за поиска необходимой ноды занимают менее 0.5% (конкатенация строк). +Появились 46 процентов новых аллокаций из-за ожидания выполнения всех запросов, 13% - работа сетевых +потоков и 2% selectorManager. Итого 35% новых аллокаций. Обработка одного запроса для целевой ноды теперь приводит +к числу семплов аллокаций в 3 раза большему, чем в прошлом этапе. + +### lock profiling + +![lock](../profiler_results/stage4/get_5400rs_1min_lock.html) +Большую часть находятся в ожидании сетевые потоки (SequentialScheduler) - 52% и selectorManager - 15% + +## Сравнение с прошлым этапом + +### PUT +Точка разладки сместилась на 9800 запросов в секунду в меньшую сторону по сравнению с прошлым этапом +(была 16600 стала 6800). +Возьмем и замерим результаты реализации прошлого этапа в точке разладки этого: + +``` + было стало +50.000 1.60ms 8.32ms +75.000 2.27ms 30.13ms +90.000 3.03ms 48.10ms +99.000 10.38ms 72.45ms +99.900 18.54ms 84.03ms +99.990 22.93ms 90.37ms +99.999 26.40ms 94.27ms +100.000 26.40ms 96.13ms +``` + +### GET +Точка разладки сместилась на 10700 запросов в секунду в меньшую сторону по сравнению с прошлым этапом +(была 16100 стала 5400). +Возьмем и замерим результаты реализации прошлого этапа в точке разладки этого: + +``` + было стало +50.000 4.01ms 5.11ms +75.000 4.91ms 6.94ms +90.000 5.71ms 9.53ms +99.000 7.33ms 39.78ms +99.900 9.51ms 45.66ms +99.990% 12.38ms 48.70ms +99.999% 14.26ms 50.72ms +100.000% 15.08ms 51.20m +``` + +## Выводы +Текущая реализация стала держать меньше get и put-запросов и стала медленнее. Из проофилей виднно, что хоть и +процентное соотношение всех семплов осталось приблизительно таким же, как в прошлом этапе (кроме участка, где требуется +ожидание ответов от всех нод на целевой), но каждый из участков стал занимать значительно больше сэмплов. Для решения +данной проблемы требуется ввести асинхронные опрос нод и агрегацию ответов. + diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_10000rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_10000rs_1min.txt deleted file mode 100644 index dbf1cd51c..000000000 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_10000rs_1min.txt +++ /dev/null @@ -1,69 +0,0 @@ - Value Percentile TotalCount 1/(1-Percentile) - - 6791.167 0.000000 17 1.00 - 10133.503 0.100000 16568 1.11 - 13484.031 0.200000 33070 1.25 - 16859.135 0.300000 49603 1.43 - 20234.239 0.400000 66123 1.67 - 23609.343 0.500000 82714 2.00 - 25264.127 0.550000 90905 2.22 - 26918.911 0.600000 99174 2.50 - 28590.079 0.650000 107489 2.86 - 30277.631 0.700000 115779 3.33 - 31932.415 0.750000 123989 4.00 - 32784.383 0.775000 128155 4.44 - 33652.735 0.800000 132374 5.00 - 34439.167 0.825000 136465 5.71 - 35258.367 0.850000 140568 6.67 - 36110.335 0.875000 144729 8.00 - 36536.319 0.887500 146820 8.89 - 36929.535 0.900000 148758 10.00 - 37355.519 0.912500 150876 11.43 - 37781.503 0.925000 152951 13.33 - 38207.487 0.937500 155083 16.00 - 38404.095 0.943750 156020 17.78 - 38633.471 0.950000 157133 20.00 - 38830.079 0.956250 158125 22.86 - 39026.687 0.962500 159113 26.67 - 39256.063 0.968750 160245 32.00 - 39354.367 0.971875 160727 35.56 - 39452.671 0.975000 161213 40.00 - 39550.975 0.978125 161710 45.71 - 39649.279 0.981250 162191 53.33 - 39780.351 0.984375 162850 64.00 - 39813.119 0.985938 163022 71.11 - 39878.655 0.987500 163366 80.00 - 39911.423 0.989062 163529 91.43 - 39976.959 0.990625 163860 106.67 - 40009.727 0.992188 164032 128.00 - 40042.495 0.992969 164208 142.22 - 40075.263 0.993750 164380 160.00 - 40075.263 0.994531 164380 182.86 - 40108.031 0.995313 164542 213.33 - 40140.799 0.996094 164712 256.00 - 40140.799 0.996484 164712 284.44 - 40173.567 0.996875 164883 320.00 - 40173.567 0.997266 164883 365.71 - 40206.335 0.997656 165055 426.67 - 40206.335 0.998047 165055 512.00 - 40206.335 0.998242 165055 568.89 - 40206.335 0.998437 165055 640.00 - 40206.335 0.998633 165055 731.43 - 40239.103 0.998828 165206 853.33 - 40239.103 0.999023 165206 1024.00 - 40239.103 0.999121 165206 1137.78 - 40239.103 0.999219 165206 1280.00 - 40239.103 0.999316 165206 1462.86 - 40239.103 0.999414 165206 1706.67 - 40239.103 0.999512 165206 2048.00 - 40239.103 0.999561 165206 2275.56 - 40271.871 0.999609 165276 2560.00 - 40271.871 1.000000 165276 inf -#[Mean = 23550.947, StdDeviation = 9670.107] -#[Max = 40239.104, Total count = 165276] -#[Buckets = 27, SubBuckets = 2048] ----------------------------------------------------------- - 197455 requests in 1.00m, 18.13MB read - Non-2xx or 3xx responses: 26509 -Requests/sec: 3290.94 -Transfer/sec: 309.49KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_1000rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_1000rs_1min.txt deleted file mode 100644 index d166ba3be..000000000 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_1000rs_1min.txt +++ /dev/null @@ -1,89 +0,0 @@ - 0.075 0.000000 1 1.00 - 0.722 0.100000 5034 1.11 - 0.864 0.200000 10013 1.25 - 0.998 0.300000 15036 1.43 - 1.130 0.400000 19999 1.67 - 1.252 0.500000 25023 2.00 - 1.297 0.550000 27510 2.22 - 1.377 0.600000 30026 2.50 - 1.437 0.650000 32537 2.86 - 1.496 0.700000 35046 3.33 - 1.553 0.750000 37498 4.00 - 1.584 0.775000 38776 4.44 - 1.614 0.800000 40003 5.00 - 1.645 0.825000 41274 5.71 - 1.676 0.850000 42515 6.67 - 1.713 0.875000 43765 8.00 - 1.732 0.887500 44405 8.89 - 1.753 0.900000 45001 10.00 - 1.774 0.912500 45638 11.43 - 1.796 0.925000 46256 13.33 - 1.821 0.937500 46903 16.00 - 1.833 0.943750 47191 17.78 - 1.849 0.950000 47496 20.00 - 1.874 0.956250 47819 22.86 - 1.906 0.962500 48125 26.67 - 1.947 0.968750 48438 32.00 - 1.975 0.971875 48591 35.56 - 2.012 0.975000 48750 40.00 - 2.051 0.978125 48906 45.71 - 2.111 0.981250 49063 53.33 - 2.163 0.984375 49216 64.00 - 2.185 0.985938 49300 71.11 - 2.207 0.987500 49373 80.00 - 2.245 0.989062 49451 91.43 - 2.265 0.990625 49530 106.67 - 2.281 0.992188 49605 128.00 - 2.287 0.992969 49645 142.22 - 2.299 0.993750 49683 160.00 - 2.321 0.994531 49723 182.86 - 2.341 0.995313 49763 213.33 - 2.363 0.996094 49800 256.00 - 2.379 0.996484 49821 284.44 - 2.425 0.996875 49839 320.00 - 2.501 0.997266 49860 365.71 - 2.559 0.997656 49878 426.67 - 2.617 0.998047 49898 512.00 - 2.649 0.998242 49908 568.89 - 2.713 0.998437 49917 640.00 - 2.767 0.998633 49927 731.43 - 2.875 0.998828 49937 853.33 - 3.131 0.999023 49947 1024.00 - 3.253 0.999121 49952 1137.78 - 3.451 0.999219 49956 1280.00 - 3.699 0.999316 49961 1462.86 - 3.879 0.999414 49966 1706.67 - 4.017 0.999512 49971 2048.00 - 4.049 0.999561 49974 2275.56 - 4.057 0.999609 49976 2560.00 - 4.069 0.999658 49978 2925.71 - 4.111 0.999707 49981 3413.33 - 4.131 0.999756 49985 4096.00 - 4.131 0.999780 49985 4551.11 - 4.295 0.999805 49986 5120.00 - 4.359 0.999829 49987 5851.43 - 4.439 0.999854 49988 6826.67 - 4.523 0.999878 49989 8192.00 - 4.591 0.999890 49990 9102.22 - 4.627 0.999902 49991 10240.00 - 4.627 0.999915 49991 11702.86 - 4.639 0.999927 49992 13653.33 - 4.639 0.999939 49992 16384.00 - 4.723 0.999945 49993 18204.44 - 4.723 0.999951 49993 20480.00 - 4.723 0.999957 49993 23405.71 - 4.783 0.999963 49994 27306.67 - 4.783 0.999969 49994 32768.00 - 4.783 0.999973 49994 36408.89 - 4.783 0.999976 49994 40960.00 - 4.783 0.999979 49994 46811.43 - 4.811 0.999982 49995 54613.33 - 4.811 1.000000 49995 inf -#[Mean = 1.251, StdDeviation = 0.408] -#[Max = 4.808, Total count = 49995] -#[Buckets = 27, SubBuckets = 2048] ----------------------------------------------------------- - 60000 requests in 1.00m, 5.50MB read - Non-2xx or 3xx responses: 8210 -Requests/sec: 999.98 -Transfer/sec: 93.90KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_3000rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_3000rs_1min.txt deleted file mode 100644 index 792f4d740..000000000 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_3000rs_1min.txt +++ /dev/null @@ -1,99 +0,0 @@ - Value Percentile TotalCount 1/(1-Percentile) - - 0.039 0.000000 1 1.00 - 0.461 0.100000 15039 1.11 - 0.629 0.200000 30040 1.25 - 0.782 0.300000 45002 1.43 - 0.947 0.400000 60053 1.67 - 1.092 0.500000 75073 2.00 - 1.168 0.550000 82577 2.22 - 1.244 0.600000 90005 2.50 - 1.313 0.650000 97523 2.86 - 1.382 0.700000 105085 3.33 - 1.472 0.750000 112535 4.00 - 1.521 0.775000 116282 4.44 - 1.586 0.800000 119995 5.00 - 1.681 0.825000 123764 5.71 - 1.922 0.850000 127486 6.67 - 2.375 0.875000 131247 8.00 - 2.721 0.887500 133114 8.89 - 3.373 0.900000 134987 10.00 - 4.061 0.912500 136860 11.43 - 4.771 0.925000 138738 13.33 - 5.583 0.937500 140615 16.00 - 5.935 0.943750 141545 17.78 - 6.299 0.950000 142484 20.00 - 6.639 0.956250 143433 22.86 - 6.951 0.962500 144361 26.67 - 7.275 0.968750 145300 32.00 - 7.435 0.971875 145768 35.56 - 7.651 0.975000 146232 40.00 - 8.011 0.978125 146703 45.71 - 8.559 0.981250 147169 53.33 - 9.087 0.984375 147647 64.00 - 9.431 0.985938 147878 71.11 - 9.887 0.987500 148107 80.00 - 10.863 0.989062 148342 91.43 - 12.335 0.990625 148576 106.67 - 13.959 0.992188 148810 128.00 - 14.703 0.992969 148928 142.22 - 14.991 0.993750 149044 160.00 - 15.623 0.994531 149161 182.86 - 16.927 0.995313 149280 213.33 - 18.511 0.996094 149397 256.00 - 19.023 0.996484 149455 284.44 - 19.647 0.996875 149513 320.00 - 20.335 0.997266 149571 365.71 - 20.751 0.997656 149630 426.67 - 21.023 0.998047 149699 512.00 - 21.055 0.998242 149720 568.89 - 21.311 0.998437 149747 640.00 - 23.263 0.998633 149776 731.43 - 24.431 0.998828 149806 853.33 - 25.199 0.999023 149836 1024.00 - 25.583 0.999121 149851 1137.78 - 26.031 0.999219 149865 1280.00 - 26.687 0.999316 149879 1462.86 - 27.151 0.999414 149895 1706.67 - 27.487 0.999512 149908 2048.00 - 27.647 0.999561 149916 2275.56 - 27.951 0.999609 149923 2560.00 - 28.079 0.999658 149930 2925.71 - 28.255 0.999707 149938 3413.33 - 28.367 0.999756 149945 4096.00 - 28.415 0.999780 149949 4551.11 - 28.447 0.999805 149952 5120.00 - 28.527 0.999829 149956 5851.43 - 28.559 0.999854 149960 6826.67 - 28.623 0.999878 149964 8192.00 - 28.655 0.999890 149965 9102.22 - 28.719 0.999902 149968 10240.00 - 28.735 0.999915 149969 11702.86 - 28.783 0.999927 149972 13653.33 - 28.783 0.999939 149972 16384.00 - 28.799 0.999945 149974 18204.44 - 28.799 0.999951 149974 20480.00 - 28.831 0.999957 149976 23405.71 - 28.831 0.999963 149976 27306.67 - 28.879 0.999969 149979 32768.00 - 28.879 0.999973 149979 36408.89 - 28.879 0.999976 149979 40960.00 - 28.879 0.999979 149979 46811.43 - 28.879 0.999982 149979 54613.33 - 28.879 0.999985 149979 65536.00 - 28.879 0.999986 149979 72817.78 - 28.911 0.999988 149980 81920.00 - 28.911 0.999989 149980 93622.86 - 28.911 0.999991 149980 109226.67 - 28.911 0.999992 149980 131072.00 - 28.911 0.999993 149980 145635.56 - 28.927 0.999994 149981 163840.00 - 28.927 1.000000 149981 inf -#[Mean = 1.679, StdDeviation = 2.320] -#[Max = 28.912, Total count = 149981] -#[Buckets = 27, SubBuckets = 2048] ----------------------------------------------------------- - 179993 requests in 1.00m, 16.53MB read - Non-2xx or 3xx responses: 24130 -Requests/sec: 2999.90 -Transfer/sec: 282.04KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_4000rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_4000rs_1min.txt deleted file mode 100644 index 1f2cc8200..000000000 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/get_4000rs_1min.txt +++ /dev/null @@ -1,67 +0,0 @@ - Value Percentile TotalCount 1/(1-Percentile) - - 1694.719 0.000000 15 1.00 - 2498.559 0.100000 16809 1.11 - 3184.639 0.200000 33587 1.25 - 4018.175 0.300000 50372 1.43 - 4853.759 0.400000 67154 1.67 - 5701.631 0.500000 84007 2.00 - 6103.039 0.550000 92331 2.22 - 6504.447 0.600000 100740 2.50 - 6905.855 0.650000 109176 2.86 - 7340.031 0.700000 117536 3.33 - 7741.439 0.750000 125962 4.00 - 7917.567 0.775000 130127 4.44 - 8085.503 0.800000 134323 5.00 - 8249.343 0.825000 138542 5.71 - 8462.335 0.850000 142740 6.67 - 8683.519 0.875000 146984 8.00 - 8798.207 0.887500 149089 8.89 - 8896.511 0.900000 151146 10.00 - 8994.815 0.912500 153259 11.43 - 9109.503 0.925000 155350 13.33 - 9224.191 0.937500 157398 16.00 - 9281.535 0.943750 158493 17.78 - 9338.879 0.950000 159571 20.00 - 9388.031 0.956250 160562 22.86 - 9445.375 0.962500 161698 26.67 - 9494.527 0.968750 162673 32.00 - 9519.103 0.971875 163138 35.56 - 9551.871 0.975000 163683 40.00 - 9584.639 0.978125 164372 45.71 - 9601.023 0.981250 164749 53.33 - 9625.599 0.984375 165418 64.00 - 9633.791 0.985938 165622 71.11 - 9641.983 0.987500 165831 80.00 - 9650.175 0.989062 166043 91.43 - 9666.559 0.990625 166447 106.67 - 9674.751 0.992188 166642 128.00 - 9682.943 0.992969 166843 142.22 - 9682.943 0.993750 166843 160.00 - 9691.135 0.994531 167049 182.86 - 9699.327 0.995313 167272 213.33 - 9699.327 0.996094 167272 256.00 - 9699.327 0.996484 167272 284.44 - 9707.519 0.996875 167425 320.00 - 9707.519 0.997266 167425 365.71 - 9715.711 0.997656 167568 426.67 - 9715.711 0.998047 167568 512.00 - 9715.711 0.998242 167568 568.89 - 9723.903 0.998437 167746 640.00 - 9723.903 0.998633 167746 731.43 - 9723.903 0.998828 167746 853.33 - 9723.903 0.999023 167746 1024.00 - 9723.903 0.999121 167746 1137.78 - 9723.903 0.999219 167746 1280.00 - 9723.903 0.999316 167746 1462.86 - 9723.903 0.999414 167746 1706.67 - 9732.095 0.999512 167839 2048.00 - 9732.095 1.000000 167839 inf -#[Mean = 5678.582, StdDeviation = 2333.851] -#[Max = 9723.904, Total count = 167839] -#[Buckets = 27, SubBuckets = 2048] ----------------------------------------------------------- - 201084 requests in 1.00m, 18.46MB read - Non-2xx or 3xx responses: 27115 -Requests/sec: 3351.41 -Transfer/sec: 315.05KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_10000rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_10000rs_1min.txt deleted file mode 100644 index 64382cb2a..000000000 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_10000rs_1min.txt +++ /dev/null @@ -1,99 +0,0 @@ - Value Percentile TotalCount 1/(1-Percentile) - - 0.025 0.000000 5 1.00 - 0.226 0.100000 50110 1.11 - 0.399 0.200000 100298 1.25 - 0.569 0.300000 150136 1.43 - 0.737 0.400000 199986 1.67 - 0.903 0.500000 250243 2.00 - 0.981 0.550000 275105 2.22 - 1.049 0.600000 299984 2.50 - 1.109 0.650000 325078 2.86 - 1.214 0.700000 350095 3.33 - 1.390 0.750000 374994 4.00 - 1.500 0.775000 387568 4.44 - 1.619 0.800000 400008 5.00 - 1.747 0.825000 412550 5.71 - 1.881 0.850000 424976 6.67 - 2.018 0.875000 437509 8.00 - 2.081 0.887500 443825 8.89 - 2.139 0.900000 450084 10.00 - 2.197 0.912500 456377 11.43 - 2.261 0.925000 462659 13.33 - 2.317 0.937500 468716 16.00 - 2.347 0.943750 472008 17.78 - 2.375 0.950000 475031 20.00 - 2.415 0.956250 478158 22.86 - 2.463 0.962500 481244 26.67 - 2.529 0.968750 484338 32.00 - 2.575 0.971875 485953 35.56 - 2.641 0.975000 487491 40.00 - 2.803 0.978125 489033 45.71 - 3.393 0.981250 490583 53.33 - 4.771 0.984375 492148 64.00 - 5.887 0.985938 492928 71.11 - 7.063 0.987500 493710 80.00 - 8.807 0.989062 494493 91.43 - 10.743 0.990625 495275 106.67 - 14.199 0.992188 496053 128.00 - 16.111 0.992969 496442 142.22 - 18.095 0.993750 496834 160.00 - 20.543 0.994531 497223 182.86 - 23.439 0.995313 497614 213.33 - 26.383 0.996094 498005 256.00 - 28.143 0.996484 498200 284.44 - 29.311 0.996875 498396 320.00 - 30.495 0.997266 498590 365.71 - 33.727 0.997656 498787 426.67 - 38.079 0.998047 498981 512.00 - 40.223 0.998242 499080 568.89 - 43.711 0.998437 499176 640.00 - 48.479 0.998633 499274 731.43 - 50.559 0.998828 499373 853.33 - 53.727 0.999023 499469 1024.00 - 55.263 0.999121 499519 1137.78 - 56.191 0.999219 499567 1280.00 - 57.183 0.999316 499617 1462.86 - 58.559 0.999414 499665 1706.67 - 60.383 0.999512 499714 2048.00 - 61.439 0.999561 499738 2275.56 - 62.367 0.999609 499762 2560.00 - 63.167 0.999658 499788 2925.71 - 63.967 0.999707 499811 3413.33 - 64.607 0.999756 499835 4096.00 - 64.863 0.999780 499849 4551.11 - 65.055 0.999805 499861 5120.00 - 65.247 0.999829 499872 5851.43 - 65.407 0.999854 499885 6826.67 - 65.599 0.999878 499900 8192.00 - 65.663 0.999890 499904 9102.22 - 65.727 0.999902 499909 10240.00 - 65.791 0.999915 499915 11702.86 - 65.855 0.999927 499923 13653.33 - 65.919 0.999939 499928 16384.00 - 65.983 0.999945 499936 18204.44 - 65.983 0.999951 499936 20480.00 - 65.983 0.999957 499936 23405.71 - 66.047 0.999963 499941 27306.67 - 66.111 0.999969 499948 32768.00 - 66.111 0.999973 499948 36408.89 - 66.111 0.999976 499948 40960.00 - 66.111 0.999979 499948 46811.43 - 66.111 0.999982 499948 54613.33 - 66.175 0.999985 499954 65536.00 - 66.175 0.999986 499954 72817.78 - 66.175 0.999988 499954 81920.00 - 66.175 0.999989 499954 93622.86 - 66.175 0.999991 499954 109226.67 - 66.175 0.999992 499954 131072.00 - 66.175 0.999993 499954 145635.56 - 66.175 0.999994 499954 163840.00 - 66.239 0.999995 499957 187245.71 - 66.239 1.000000 499957 inf -#[Mean = 1.289, StdDeviation = 3.068] -#[Max = 66.176, Total count = 499957] -#[Buckets = 27, SubBuckets = 2048] ----------------------------------------------------------- - 599980 requests in 1.00m, 38.34MB read -Requests/sec: 9999.62 -Transfer/sec: 654.27KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_30000rs_1min b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_30000rs_1min deleted file mode 100644 index a67e670e4..000000000 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_30000rs_1min +++ /dev/null @@ -1,68 +0,0 @@ - Value Percentile TotalCount 1/(1-Percentile) - - 988.159 0.000000 109 1.00 - 1499.135 0.100000 133127 1.11 - 2033.663 0.200000 266437 1.25 - 2848.767 0.300000 400046 1.43 - 3672.063 0.400000 532334 1.67 - 4222.975 0.500000 665821 2.00 - 4874.239 0.550000 733572 2.22 - 5017.599 0.600000 798626 2.50 - 5455.871 0.650000 865072 2.86 - 5746.687 0.700000 931630 3.33 - 5865.471 0.750000 999402 4.00 - 5943.295 0.775000 1032089 4.44 - 6230.015 0.800000 1065194 5.00 - 6324.223 0.825000 1099263 5.71 - 6422.527 0.850000 1131430 6.67 - 6541.311 0.875000 1164776 8.00 - 6606.847 0.887500 1181083 8.89 - 6672.383 0.900000 1197783 10.00 - 6868.991 0.912500 1214449 11.43 - 6946.815 0.925000 1231691 13.33 - 7000.063 0.937500 1248166 16.00 - 7020.543 0.943750 1257511 17.78 - 7032.831 0.950000 1264498 20.00 - 7045.119 0.956250 1273235 22.86 - 7057.407 0.962500 1281632 26.67 - 7073.791 0.968750 1290057 32.00 - 7081.983 0.971875 1295954 35.56 - 7086.079 0.975000 1297921 40.00 - 7094.271 0.978125 1302080 45.71 - 7102.463 0.981250 1307071 53.33 - 7106.559 0.984375 1309955 64.00 - 7110.655 0.985938 1312309 71.11 - 7114.751 0.987500 1314935 80.00 - 7118.847 0.989062 1317041 91.43 - 7122.943 0.990625 1318561 106.67 - 7135.231 0.992188 1320991 128.00 - 7139.327 0.992969 1321791 142.22 - 7143.423 0.993750 1322610 160.00 - 7151.615 0.994531 1324193 182.86 - 7155.711 0.995313 1324983 213.33 - 7159.807 0.996094 1325765 256.00 - 7163.903 0.996484 1326528 284.44 - 7167.999 0.996875 1327402 320.00 - 7167.999 0.997266 1327402 365.71 - 7172.095 0.997656 1328272 426.67 - 7172.095 0.998047 1328272 512.00 - 7176.191 0.998242 1329242 568.89 - 7176.191 0.998437 1329242 640.00 - 7176.191 0.998633 1329242 731.43 - 7176.191 0.998828 1329242 853.33 - 7180.287 0.999023 1330180 1024.00 - 7180.287 0.999121 1330180 1137.78 - 7180.287 0.999219 1330180 1280.00 - 7180.287 0.999316 1330180 1462.86 - 7180.287 0.999414 1330180 1706.67 - 7180.287 0.999512 1330180 2048.00 - 7180.287 0.999561 1330180 2275.56 - 7184.383 0.999609 1330728 2560.00 - 7184.383 1.000000 1330728 inf -#[Mean = 4240.484, StdDeviation = 1909.991] -#[Max = 7180.288, Total count = 1330728] -#[Buckets = 27, SubBuckets = 2048] ----------------------------------------------------------- - 1594140 requests in 1.00m, 101.86MB read -Requests/sec: 26569.23 -Transfer/sec: 1.70MB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_3000rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_3000rs_1min.txt deleted file mode 100644 index d7b00d8cc..000000000 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_3000rs_1min.txt +++ /dev/null @@ -1,98 +0,0 @@ - Value Percentile TotalCount 1/(1-Percentile) - - 0.032 0.000000 1 1.00 - 0.402 0.100000 15059 1.11 - 0.592 0.200000 30049 1.25 - 0.741 0.300000 45016 1.43 - 0.904 0.400000 60050 1.67 - 1.036 0.500000 75062 2.00 - 1.086 0.550000 82603 2.22 - 1.135 0.600000 90037 2.50 - 1.210 0.650000 97533 2.86 - 1.269 0.700000 105107 3.33 - 1.328 0.750000 112540 4.00 - 1.363 0.775000 116325 4.44 - 1.391 0.800000 120116 5.00 - 1.418 0.825000 123733 5.71 - 1.447 0.850000 127624 6.67 - 1.484 0.875000 131276 8.00 - 1.520 0.887500 133143 8.89 - 1.567 0.900000 134996 10.00 - 1.602 0.912500 136877 11.43 - 1.647 0.925000 138745 13.33 - 1.741 0.937500 140614 16.00 - 1.797 0.943750 141546 17.78 - 1.870 0.950000 142487 20.00 - 1.934 0.956250 143422 22.86 - 1.992 0.962500 144353 26.67 - 2.053 0.968750 145333 32.00 - 2.073 0.971875 145766 35.56 - 2.107 0.975000 146263 40.00 - 2.131 0.978125 146706 45.71 - 2.171 0.981250 147171 53.33 - 2.227 0.984375 147650 64.00 - 2.247 0.985938 147874 71.11 - 2.265 0.987500 148104 80.00 - 2.283 0.989062 148340 91.43 - 2.301 0.990625 148585 106.67 - 2.323 0.992188 148809 128.00 - 2.337 0.992969 148932 142.22 - 2.353 0.993750 149040 160.00 - 2.369 0.994531 149171 182.86 - 2.383 0.995313 149279 213.33 - 2.395 0.996094 149395 256.00 - 2.403 0.996484 149480 284.44 - 2.407 0.996875 149512 320.00 - 2.419 0.997266 149567 365.71 - 2.431 0.997656 149631 426.67 - 2.439 0.998047 149688 512.00 - 2.443 0.998242 149716 568.89 - 2.449 0.998437 149752 640.00 - 2.455 0.998633 149776 731.43 - 2.461 0.998828 149807 853.33 - 2.469 0.999023 149831 1024.00 - 2.477 0.999121 149851 1137.78 - 2.483 0.999219 149861 1280.00 - 2.499 0.999316 149878 1462.86 - 2.539 0.999414 149890 1706.67 - 2.569 0.999512 149904 2048.00 - 2.587 0.999561 149912 2275.56 - 2.601 0.999609 149920 2560.00 - 2.627 0.999658 149926 2925.71 - 2.643 0.999707 149934 3413.33 - 2.697 0.999756 149941 4096.00 - 2.817 0.999780 149945 4551.11 - 3.025 0.999805 149948 5120.00 - 3.291 0.999829 149952 5851.43 - 3.581 0.999854 149956 6826.67 - 3.851 0.999878 149959 8192.00 - 3.997 0.999890 149961 9102.22 - 4.143 0.999902 149963 10240.00 - 4.259 0.999915 149965 11702.86 - 4.399 0.999927 149967 13653.33 - 4.467 0.999939 149968 16384.00 - 4.503 0.999945 149969 18204.44 - 4.523 0.999951 149970 20480.00 - 4.559 0.999957 149971 23405.71 - 4.611 0.999963 149972 27306.67 - 4.647 0.999969 149973 32768.00 - 4.647 0.999973 149973 36408.89 - 4.739 0.999976 149974 40960.00 - 4.739 0.999979 149974 46811.43 - 4.867 0.999982 149975 54613.33 - 4.867 0.999985 149975 65536.00 - 4.867 0.999986 149975 72817.78 - 4.979 0.999988 149976 81920.00 - 4.979 0.999989 149976 93622.86 - 4.979 0.999991 149976 109226.67 - 4.979 0.999992 149976 131072.00 - 4.979 0.999993 149976 145635.56 - 5.051 0.999994 149977 163840.00 - 5.051 1.000000 149977 inf -#[Mean = 1.024, StdDeviation = 0.465] -#[Max = 5.048, Total count = 149977] -#[Buckets = 27, SubBuckets = 2048] ----------------------------------------------------------- - 179990 requests in 1.00m, 11.50MB read -Requests/sec: 2999.86 -Transfer/sec: 196.28KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_5000rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_5000rs_1min.txt deleted file mode 100644 index d7b00d8cc..000000000 --- a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/put_5000rs_1min.txt +++ /dev/null @@ -1,98 +0,0 @@ - Value Percentile TotalCount 1/(1-Percentile) - - 0.032 0.000000 1 1.00 - 0.402 0.100000 15059 1.11 - 0.592 0.200000 30049 1.25 - 0.741 0.300000 45016 1.43 - 0.904 0.400000 60050 1.67 - 1.036 0.500000 75062 2.00 - 1.086 0.550000 82603 2.22 - 1.135 0.600000 90037 2.50 - 1.210 0.650000 97533 2.86 - 1.269 0.700000 105107 3.33 - 1.328 0.750000 112540 4.00 - 1.363 0.775000 116325 4.44 - 1.391 0.800000 120116 5.00 - 1.418 0.825000 123733 5.71 - 1.447 0.850000 127624 6.67 - 1.484 0.875000 131276 8.00 - 1.520 0.887500 133143 8.89 - 1.567 0.900000 134996 10.00 - 1.602 0.912500 136877 11.43 - 1.647 0.925000 138745 13.33 - 1.741 0.937500 140614 16.00 - 1.797 0.943750 141546 17.78 - 1.870 0.950000 142487 20.00 - 1.934 0.956250 143422 22.86 - 1.992 0.962500 144353 26.67 - 2.053 0.968750 145333 32.00 - 2.073 0.971875 145766 35.56 - 2.107 0.975000 146263 40.00 - 2.131 0.978125 146706 45.71 - 2.171 0.981250 147171 53.33 - 2.227 0.984375 147650 64.00 - 2.247 0.985938 147874 71.11 - 2.265 0.987500 148104 80.00 - 2.283 0.989062 148340 91.43 - 2.301 0.990625 148585 106.67 - 2.323 0.992188 148809 128.00 - 2.337 0.992969 148932 142.22 - 2.353 0.993750 149040 160.00 - 2.369 0.994531 149171 182.86 - 2.383 0.995313 149279 213.33 - 2.395 0.996094 149395 256.00 - 2.403 0.996484 149480 284.44 - 2.407 0.996875 149512 320.00 - 2.419 0.997266 149567 365.71 - 2.431 0.997656 149631 426.67 - 2.439 0.998047 149688 512.00 - 2.443 0.998242 149716 568.89 - 2.449 0.998437 149752 640.00 - 2.455 0.998633 149776 731.43 - 2.461 0.998828 149807 853.33 - 2.469 0.999023 149831 1024.00 - 2.477 0.999121 149851 1137.78 - 2.483 0.999219 149861 1280.00 - 2.499 0.999316 149878 1462.86 - 2.539 0.999414 149890 1706.67 - 2.569 0.999512 149904 2048.00 - 2.587 0.999561 149912 2275.56 - 2.601 0.999609 149920 2560.00 - 2.627 0.999658 149926 2925.71 - 2.643 0.999707 149934 3413.33 - 2.697 0.999756 149941 4096.00 - 2.817 0.999780 149945 4551.11 - 3.025 0.999805 149948 5120.00 - 3.291 0.999829 149952 5851.43 - 3.581 0.999854 149956 6826.67 - 3.851 0.999878 149959 8192.00 - 3.997 0.999890 149961 9102.22 - 4.143 0.999902 149963 10240.00 - 4.259 0.999915 149965 11702.86 - 4.399 0.999927 149967 13653.33 - 4.467 0.999939 149968 16384.00 - 4.503 0.999945 149969 18204.44 - 4.523 0.999951 149970 20480.00 - 4.559 0.999957 149971 23405.71 - 4.611 0.999963 149972 27306.67 - 4.647 0.999969 149973 32768.00 - 4.647 0.999973 149973 36408.89 - 4.739 0.999976 149974 40960.00 - 4.739 0.999979 149974 46811.43 - 4.867 0.999982 149975 54613.33 - 4.867 0.999985 149975 65536.00 - 4.867 0.999986 149975 72817.78 - 4.979 0.999988 149976 81920.00 - 4.979 0.999989 149976 93622.86 - 4.979 0.999991 149976 109226.67 - 4.979 0.999992 149976 131072.00 - 4.979 0.999993 149976 145635.56 - 5.051 0.999994 149977 163840.00 - 5.051 1.000000 149977 inf -#[Mean = 1.024, StdDeviation = 0.465] -#[Max = 5.048, Total count = 149977] -#[Buckets = 27, SubBuckets = 2048] ----------------------------------------------------------- - 179990 requests in 1.00m, 11.50MB read -Requests/sec: 2999.86 -Transfer/sec: 196.28KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/get_16100rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/get_16100rs_1min.txt new file mode 100644 index 000000000..1713a7656 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/get_16100rs_1min.txt @@ -0,0 +1,129 @@ +dmitrij@MY-PC ~/Д/J/w/wrk2 (master)> ./wrk -d 60 -t 1 -c 64 -R 16100 -L -s /home/dmitrij/Документы/JavaProjects/highload/2024-highload-dht/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_scripts/get.lua http://localhost:8080 +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 2.299ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 4.34ms 8.10ms 92.10ms 94.60% + Req/Sec 17.00k 1.65k 33.00k 74.89% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 2.48ms + 75.000% 4.03ms + 90.000% 7.20ms + 99.000% 49.53ms + 99.900% 83.84ms + 99.990% 88.45ms + 99.999% 90.69ms +100.000% 92.16ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.188 0.000000 1 1.00 + 0.953 0.100000 79992 1.11 + 1.268 0.200000 159988 1.25 + 1.665 0.300000 239948 1.43 + 2.079 0.400000 319980 1.67 + 2.477 0.500000 400122 2.00 + 2.691 0.550000 439921 2.22 + 2.951 0.600000 480012 2.50 + 3.263 0.650000 519891 2.86 + 3.611 0.700000 559882 3.33 + 4.027 0.750000 599879 4.00 + 4.275 0.775000 619901 4.44 + 4.551 0.800000 639953 5.00 + 4.899 0.825000 659939 5.71 + 5.391 0.850000 679949 6.67 + 6.095 0.875000 699862 8.00 + 6.579 0.887500 709830 8.89 + 7.199 0.900000 719859 10.00 + 8.035 0.912500 729841 11.43 + 9.175 0.925000 739866 13.33 + 10.863 0.937500 749831 16.00 + 11.999 0.943750 754834 17.78 + 13.311 0.950000 759829 20.00 + 14.895 0.956250 764828 22.86 + 16.783 0.962500 769830 26.67 + 18.719 0.968750 774812 32.00 + 19.791 0.971875 777313 35.56 + 21.039 0.975000 779824 40.00 + 22.671 0.978125 782304 45.71 + 24.959 0.981250 784804 53.33 + 28.159 0.984375 787304 64.00 + 30.863 0.985938 788555 71.11 + 35.711 0.987500 789810 80.00 + 43.807 0.989062 791055 91.43 + 52.863 0.990625 792302 106.67 + 60.447 0.992188 793552 128.00 + 64.031 0.992969 794179 142.22 + 67.903 0.993750 794803 160.00 + 71.551 0.994531 795431 182.86 + 74.495 0.995313 796056 213.33 + 76.735 0.996094 796677 256.00 + 77.567 0.996484 797002 284.44 + 78.335 0.996875 797318 320.00 + 79.231 0.997266 797630 365.71 + 80.191 0.997656 797928 426.67 + 81.151 0.998047 798238 512.00 + 81.791 0.998242 798406 568.89 + 82.303 0.998437 798561 640.00 + 82.815 0.998633 798711 731.43 + 83.391 0.998828 798862 853.33 + 83.967 0.999023 799033 1024.00 + 84.223 0.999121 799111 1137.78 + 84.479 0.999219 799179 1280.00 + 84.799 0.999316 799253 1462.86 + 85.247 0.999414 799334 1706.67 + 85.567 0.999512 799409 2048.00 + 85.759 0.999561 799452 2275.56 + 85.951 0.999609 799488 2560.00 + 86.271 0.999658 799531 2925.71 + 86.591 0.999707 799571 3413.33 + 86.911 0.999756 799610 4096.00 + 87.103 0.999780 799628 4551.11 + 87.295 0.999805 799645 5120.00 + 87.487 0.999829 799667 5851.43 + 87.743 0.999854 799686 6826.67 + 88.063 0.999878 799704 8192.00 + 88.255 0.999890 799712 9102.22 + 88.447 0.999902 799722 10240.00 + 88.703 0.999915 799735 11702.86 + 88.831 0.999927 799744 13653.33 + 89.023 0.999939 799751 16384.00 + 89.215 0.999945 799757 18204.44 + 89.407 0.999951 799761 20480.00 + 89.471 0.999957 799765 23405.71 + 89.535 0.999963 799770 27306.67 + 89.855 0.999969 799776 32768.00 + 89.983 0.999973 799778 36408.89 + 90.047 0.999976 799781 40960.00 + 90.111 0.999979 799782 46811.43 + 90.239 0.999982 799785 54613.33 + 90.431 0.999985 799787 65536.00 + 90.623 0.999986 799790 72817.78 + 90.623 0.999988 799790 81920.00 + 90.687 0.999989 799791 93622.86 + 90.815 0.999991 799792 109226.67 + 90.943 0.999992 799793 131072.00 + 91.263 0.999993 799794 145635.56 + 91.327 0.999994 799796 163840.00 + 91.327 0.999995 799796 187245.71 + 91.327 0.999995 799796 218453.33 + 91.327 0.999996 799796 262144.00 + 91.647 0.999997 799797 291271.11 + 91.647 0.999997 799797 327680.00 + 91.647 0.999997 799797 374491.43 + 91.903 0.999998 799798 436906.67 + 91.903 0.999998 799798 524288.00 + 91.903 0.999998 799798 582542.22 + 91.903 0.999998 799798 655360.00 + 91.903 0.999999 799798 748982.86 + 92.159 0.999999 799799 873813.33 + 92.159 1.000000 799799 inf +#[Mean = 4.338, StdDeviation = 8.095] +#[Max = 92.096, Total count = 799799] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 963419 requests in 1.00m, 86.54MB read + Non-2xx or 3xx responses: 192558 +Requests/sec: 16057.00 +Transfer/sec: 1.44MB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/get_16200rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/get_16200rs_1min.txt new file mode 100644 index 000000000..6e66f8ee4 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/get_16200rs_1min.txt @@ -0,0 +1,124 @@ +dmitrij@MY-PC ~/Д/J/w/wrk2 (master)> ./wrk -d 60 -t 1 -c 64 -R 16200 -L -s /home/dmitrij/Документы/JavaProjects/highload/2024-highload-dht/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_scripts/get.lua http://localhost:8080 +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 51.375ms, rate sampling interval: 505ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 45.66ms 116.92ms 625.15ms 90.23% + Req/Sec 16.35k 1.78k 28.61k 85.71% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 2.94ms + 75.000% 5.92ms + 90.000% 152.32ms + 99.000% 544.26ms + 99.900% 602.11ms + 99.990% 618.49ms + 99.999% 623.10ms +100.000% 625.66ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.158 0.000000 1 1.00 + 1.017 0.100000 81337 1.11 + 1.399 0.200000 162386 1.25 + 1.922 0.300000 243378 1.43 + 2.391 0.400000 324517 1.67 + 2.945 0.500000 405724 2.00 + 3.313 0.550000 446325 2.22 + 3.693 0.600000 486913 2.50 + 4.143 0.650000 527626 2.86 + 4.731 0.700000 567948 3.33 + 5.919 0.750000 608487 4.00 + 7.211 0.775000 628715 4.44 + 10.975 0.800000 648991 5.00 + 22.143 0.825000 669272 5.71 + 45.119 0.850000 689549 6.67 + 104.959 0.875000 709819 8.00 + 133.119 0.887500 719968 8.89 + 152.319 0.900000 730114 10.00 + 223.103 0.912500 740248 11.43 + 278.527 0.925000 750427 13.33 + 343.039 0.937500 760551 16.00 + 370.943 0.943750 765606 17.78 + 387.839 0.950000 770779 20.00 + 396.287 0.956250 775808 22.86 + 405.247 0.962500 780872 26.67 + 417.279 0.968750 785871 32.00 + 428.543 0.971875 788419 35.56 + 442.111 0.975000 790953 40.00 + 458.239 0.978125 793488 45.71 + 475.391 0.981250 796021 53.33 + 492.543 0.984375 798549 64.00 + 502.015 0.985938 799844 71.11 + 513.535 0.987500 801083 80.00 + 532.991 0.989062 802381 91.43 + 549.375 0.990625 803616 106.67 + 560.127 0.992188 804917 128.00 + 564.735 0.992969 805554 142.22 + 569.343 0.993750 806180 160.00 + 573.439 0.994531 806844 182.86 + 577.023 0.995313 807442 213.33 + 581.119 0.996094 808097 256.00 + 583.167 0.996484 808392 284.44 + 585.727 0.996875 808733 320.00 + 587.775 0.997266 809010 365.71 + 590.335 0.997656 809348 426.67 + 592.895 0.998047 809653 512.00 + 594.431 0.998242 809807 568.89 + 595.967 0.998437 809971 640.00 + 597.503 0.998633 810111 731.43 + 600.063 0.998828 810287 853.33 + 602.623 0.999023 810440 1024.00 + 603.647 0.999121 810508 1137.78 + 605.183 0.999219 810604 1280.00 + 606.207 0.999316 810672 1462.86 + 607.743 0.999414 810772 1706.67 + 608.767 0.999512 810832 2048.00 + 609.279 0.999561 810865 2275.56 + 610.303 0.999609 810929 2560.00 + 610.815 0.999658 810948 2925.71 + 611.839 0.999707 810993 3413.33 + 612.863 0.999756 811037 4096.00 + 613.375 0.999780 811053 4551.11 + 613.887 0.999805 811070 5120.00 + 614.911 0.999829 811082 5851.43 + 616.447 0.999854 811107 6826.67 + 617.471 0.999878 811128 8192.00 + 617.983 0.999890 811136 9102.22 + 618.495 0.999902 811142 10240.00 + 619.007 0.999915 811152 11702.86 + 619.519 0.999927 811163 13653.33 + 620.031 0.999939 811172 16384.00 + 620.543 0.999945 811179 18204.44 + 621.055 0.999951 811187 20480.00 + 621.055 0.999957 811187 23405.71 + 621.567 0.999963 811194 27306.67 + 622.079 0.999969 811203 32768.00 + 622.079 0.999973 811203 36408.89 + 622.079 0.999976 811203 40960.00 + 622.079 0.999979 811203 46811.43 + 622.591 0.999982 811208 54613.33 + 622.591 0.999985 811208 65536.00 + 623.103 0.999986 811212 72817.78 + 623.103 0.999988 811212 81920.00 + 623.103 0.999989 811212 93622.86 + 623.615 0.999991 811213 109226.67 + 624.127 0.999992 811214 131072.00 + 624.639 0.999993 811215 145635.56 + 625.151 0.999994 811218 163840.00 + 625.151 0.999995 811218 187245.71 + 625.151 0.999995 811218 218453.33 + 625.151 0.999996 811218 262144.00 + 625.151 0.999997 811218 291271.11 + 625.151 0.999997 811218 327680.00 + 625.151 0.999997 811218 374491.43 + 625.663 0.999998 811220 436906.67 + 625.663 1.000000 811220 inf +#[Mean = 45.661, StdDeviation = 116.924] +#[Max = 625.152, Total count = 811220] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 969419 requests in 1.00m, 87.07MB read + Non-2xx or 3xx responses: 194168 +Requests/sec: 16157.00 +Transfer/sec: 1.45MB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/put_16600rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/put_16600rs_1min.txt new file mode 100644 index 000000000..9cb576964 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/put_16600rs_1min.txt @@ -0,0 +1,128 @@ + ./wrk -d 60 -t 1 -c 64 -R 16600 -L -s /home/dmitrij/Документы/JavaProjects/highload/2024-highload-dht/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_scripts/put.lua http://localhost:8080 +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 1.749ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 2.88ms 4.03ms 78.98ms 94.39% + Req/Sec 17.46k 1.63k 31.44k 76.58% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.97ms + 75.000% 2.79ms + 90.000% 5.06ms + 99.000% 21.85ms + 99.900% 48.90ms + 99.990% 69.69ms + 99.999% 76.86ms +100.000% 79.04ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.073 0.000000 1 1.00 + 0.945 0.100000 82612 1.11 + 1.221 0.200000 165031 1.25 + 1.464 0.300000 247432 1.43 + 1.709 0.400000 330086 1.67 + 1.968 0.500000 412497 2.00 + 2.105 0.550000 453762 2.22 + 2.251 0.600000 495189 2.50 + 2.405 0.650000 536099 2.86 + 2.581 0.700000 577651 3.33 + 2.787 0.750000 618625 4.00 + 2.915 0.775000 639359 4.44 + 3.069 0.800000 659808 5.00 + 3.285 0.825000 680442 5.71 + 3.661 0.850000 700969 6.67 + 4.291 0.875000 721578 8.00 + 4.667 0.887500 731903 8.89 + 5.059 0.900000 742258 10.00 + 5.479 0.912500 752570 11.43 + 5.955 0.925000 762858 13.33 + 6.519 0.937500 773116 16.00 + 6.899 0.943750 778296 17.78 + 7.411 0.950000 783422 20.00 + 8.099 0.956250 788573 22.86 + 9.063 0.962500 793746 26.67 + 10.375 0.968750 798894 32.00 + 11.207 0.971875 801460 35.56 + 12.255 0.975000 804036 40.00 + 13.511 0.978125 806617 45.71 + 15.055 0.981250 809192 53.33 + 16.991 0.984375 811770 64.00 + 18.159 0.985938 813067 71.11 + 19.471 0.987500 814342 80.00 + 20.927 0.989062 815630 91.43 + 22.559 0.990625 816923 106.67 + 24.415 0.992188 818218 128.00 + 25.487 0.992969 818856 142.22 + 26.671 0.993750 819495 160.00 + 28.079 0.994531 820140 182.86 + 29.871 0.995313 820784 213.33 + 32.143 0.996094 821427 256.00 + 33.567 0.996484 821749 284.44 + 35.071 0.996875 822070 320.00 + 36.895 0.997266 822394 365.71 + 39.039 0.997656 822718 426.67 + 41.183 0.998047 823040 512.00 + 42.431 0.998242 823199 568.89 + 43.967 0.998437 823362 640.00 + 45.471 0.998633 823520 731.43 + 47.263 0.998828 823683 853.33 + 49.151 0.999023 823844 1024.00 + 50.303 0.999121 823927 1137.78 + 51.583 0.999219 824005 1280.00 + 53.087 0.999316 824084 1462.86 + 54.815 0.999414 824164 1706.67 + 57.023 0.999512 824246 2048.00 + 58.367 0.999561 824286 2275.56 + 60.287 0.999609 824326 2560.00 + 61.791 0.999658 824366 2925.71 + 63.199 0.999707 824406 3413.33 + 64.735 0.999756 824448 4096.00 + 65.215 0.999780 824466 4551.11 + 66.047 0.999805 824486 5120.00 + 67.135 0.999829 824508 5851.43 + 68.031 0.999854 824527 6826.67 + 68.991 0.999878 824547 8192.00 + 69.439 0.999890 824557 9102.22 + 69.759 0.999902 824567 10240.00 + 70.719 0.999915 824577 11702.86 + 71.487 0.999927 824587 13653.33 + 72.255 0.999939 824597 16384.00 + 72.959 0.999945 824602 18204.44 + 73.663 0.999951 824608 20480.00 + 74.047 0.999957 824612 23405.71 + 74.751 0.999963 824617 27306.67 + 75.007 0.999969 824622 32768.00 + 75.263 0.999973 824625 36408.89 + 75.519 0.999976 824627 40960.00 + 75.839 0.999979 824630 46811.43 + 76.031 0.999982 824633 54613.33 + 76.543 0.999985 824636 65536.00 + 76.543 0.999986 824636 72817.78 + 76.607 0.999988 824637 81920.00 + 76.863 0.999989 824639 93622.86 + 77.055 0.999991 824640 109226.67 + 77.183 0.999992 824641 131072.00 + 77.503 0.999993 824642 145635.56 + 77.503 0.999994 824642 163840.00 + 77.695 0.999995 824643 187245.71 + 78.143 0.999995 824644 218453.33 + 78.143 0.999996 824644 262144.00 + 78.335 0.999997 824645 291271.11 + 78.335 0.999997 824645 327680.00 + 78.335 0.999997 824645 374491.43 + 78.591 0.999998 824646 436906.67 + 78.591 0.999998 824646 524288.00 + 78.591 0.999998 824646 582542.22 + 78.591 0.999998 824646 655360.00 + 78.591 0.999999 824646 748982.86 + 79.039 0.999999 824647 873813.33 + 79.039 1.000000 824647 inf +#[Mean = 2.880, StdDeviation = 4.026] +#[Max = 78.976, Total count = 824647] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 979326 requests in 1.00m, 62.58MB read +Requests/sec: 16321.83 +Transfer/sec: 1.04MB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/put_16700rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/put_16700rs_1min.txt new file mode 100644 index 000000000..792b69d7f --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage3/put_16700rs_1min.txt @@ -0,0 +1,128 @@ +dmitrij@MY-PC ~/Д/J/w/wrk2 (master)> ./wrk -d 60 -t 1 -c 64 -R 16700 -L -s /home/dmitrij/Документы/JavaProjects/highload/2024-highload-dht/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_scripts/put.lua http://localhost:8080 +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 1.692ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 14.95ms 30.25ms 244.99ms 89.61% + Req/Sec 17.56k 2.28k 40.33k 85.14% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 2.42ms + 75.000% 10.94ms + 90.000% 47.20ms + 99.000% 148.48ms + 99.900% 199.93ms + 99.990% 230.27ms + 99.999% 240.77ms +100.000% 245.12ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.095 0.000000 1 1.00 + 0.997 0.100000 83032 1.11 + 1.320 0.200000 165991 1.25 + 1.639 0.300000 249092 1.43 + 1.998 0.400000 331853 1.67 + 2.417 0.500000 415147 2.00 + 2.675 0.550000 456311 2.22 + 3.033 0.600000 497907 2.50 + 4.139 0.650000 539301 2.86 + 6.831 0.700000 580753 3.33 + 10.935 0.750000 622247 4.00 + 13.855 0.775000 642966 4.44 + 17.599 0.800000 663713 5.00 + 22.143 0.825000 684510 5.71 + 27.999 0.850000 705236 6.67 + 36.191 0.875000 725938 8.00 + 41.279 0.887500 736322 8.89 + 47.199 0.900000 746670 10.00 + 54.207 0.912500 757059 11.43 + 63.007 0.925000 767441 13.33 + 73.279 0.937500 777814 16.00 + 79.359 0.943750 783006 17.78 + 86.143 0.950000 788170 20.00 + 93.055 0.956250 793382 22.86 + 100.159 0.962500 798550 26.67 + 107.711 0.968750 803718 32.00 + 111.743 0.971875 806308 35.56 + 116.159 0.975000 808896 40.00 + 121.791 0.978125 811489 45.71 + 128.191 0.981250 814082 53.33 + 134.783 0.984375 816674 64.00 + 138.111 0.985938 817967 71.11 + 141.567 0.987500 819296 80.00 + 145.791 0.989062 820567 91.43 + 150.143 0.990625 821869 106.67 + 154.879 0.992188 823153 128.00 + 157.695 0.992969 823811 142.22 + 160.895 0.993750 824456 160.00 + 165.247 0.994531 825113 182.86 + 170.111 0.995313 825751 213.33 + 175.103 0.996094 826398 256.00 + 177.535 0.996484 826719 284.44 + 180.479 0.996875 827045 320.00 + 183.935 0.997266 827365 365.71 + 187.391 0.997656 827703 426.67 + 190.335 0.998047 828026 512.00 + 191.999 0.998242 828181 568.89 + 194.047 0.998437 828340 640.00 + 196.095 0.998633 828504 731.43 + 198.015 0.998828 828662 853.33 + 200.319 0.999023 828823 1024.00 + 201.855 0.999121 828903 1137.78 + 203.519 0.999219 828990 1280.00 + 205.183 0.999316 829066 1462.86 + 207.359 0.999414 829146 1706.67 + 210.047 0.999512 829228 2048.00 + 211.711 0.999561 829269 2275.56 + 213.759 0.999609 829310 2560.00 + 216.319 0.999658 829349 2925.71 + 217.983 0.999707 829393 3413.33 + 220.159 0.999756 829430 4096.00 + 222.591 0.999780 829451 4551.11 + 224.639 0.999805 829470 5120.00 + 226.175 0.999829 829492 5851.43 + 227.327 0.999854 829513 6826.67 + 228.863 0.999878 829531 8192.00 + 229.631 0.999890 829543 9102.22 + 230.527 0.999902 829551 10240.00 + 231.679 0.999915 829563 11702.86 + 232.831 0.999927 829573 13653.33 + 234.111 0.999939 829582 16384.00 + 235.391 0.999945 829587 18204.44 + 235.903 0.999951 829592 20480.00 + 236.287 0.999957 829598 23405.71 + 236.543 0.999963 829602 27306.67 + 236.799 0.999969 829608 32768.00 + 237.311 0.999973 829611 36408.89 + 237.695 0.999976 829612 40960.00 + 238.207 0.999979 829615 46811.43 + 238.719 0.999982 829617 54613.33 + 239.487 0.999985 829620 65536.00 + 239.743 0.999986 829621 72817.78 + 240.511 0.999988 829622 81920.00 + 240.767 0.999989 829624 93622.86 + 241.023 0.999991 829625 109226.67 + 241.791 0.999992 829626 131072.00 + 243.199 0.999993 829627 145635.56 + 243.199 0.999994 829627 163840.00 + 243.455 0.999995 829628 187245.71 + 244.479 0.999995 829629 218453.33 + 244.479 0.999996 829629 262144.00 + 244.607 0.999997 829630 291271.11 + 244.607 0.999997 829630 327680.00 + 244.607 0.999997 829630 374491.43 + 244.863 0.999998 829631 436906.67 + 244.863 0.999998 829631 524288.00 + 244.863 0.999998 829631 582542.22 + 244.863 0.999998 829631 655360.00 + 244.863 0.999999 829631 748982.86 + 245.119 0.999999 829632 873813.33 + 245.119 1.000000 829632 inf +#[Mean = 14.948, StdDeviation = 30.251] +#[Max = 244.992, Total count = 829632] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 985252 requests in 1.00m, 62.95MB read +Requests/sec: 16420.88 +Transfer/sec: 1.05MB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/get_5400rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/get_5400rs_1min.txt new file mode 100644 index 000000000..7cb470bd5 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/get_5400rs_1min.txt @@ -0,0 +1,121 @@ +dmitrij@PC ~/D/j/wrk2 (master)> ./wrk -d 60 -t 1 -c 64 -R 5400 -L -s /home/dmitrij/Documents/javaProjects/2024-highload-dht/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_scripts/get.lua http://localhost:8080 +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 3.150ms, rate sampling interval: 12ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 6.48ms 6.77ms 51.17ms 93.35% + Req/Sec 5.64k 353.80 8.18k 66.73% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 5.11ms + 75.000% 6.94ms + 90.000% 9.53ms + 99.000% 39.78ms + 99.900% 45.66ms + 99.990% 48.70ms + 99.999% 50.72ms +100.000% 51.20ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.442 0.000000 1 1.00 + 1.811 0.100000 26854 1.11 + 2.731 0.200000 53669 1.25 + 3.607 0.300000 80534 1.43 + 4.399 0.400000 107397 1.67 + 5.111 0.500000 134295 2.00 + 5.451 0.550000 147566 2.22 + 5.803 0.600000 161046 2.50 + 6.163 0.650000 174492 2.86 + 6.535 0.700000 187806 3.33 + 6.943 0.750000 201244 4.00 + 7.171 0.775000 207946 4.44 + 7.427 0.800000 214685 5.00 + 7.723 0.825000 221370 5.71 + 8.091 0.850000 228069 6.67 + 8.591 0.875000 234752 8.00 + 8.975 0.887500 238111 8.89 + 9.535 0.900000 241446 10.00 + 10.359 0.912500 244796 11.43 + 11.831 0.925000 248131 13.33 + 14.183 0.937500 251493 16.00 + 15.703 0.943750 253164 17.78 + 18.623 0.950000 254839 20.00 + 23.055 0.956250 256518 22.86 + 26.575 0.962500 258197 26.67 + 29.855 0.968750 259869 32.00 + 31.423 0.971875 260710 35.56 + 32.799 0.975000 261554 40.00 + 34.143 0.978125 262402 45.71 + 35.583 0.981250 263234 53.33 + 37.247 0.984375 264065 64.00 + 38.015 0.985938 264488 71.11 + 38.751 0.987500 264911 80.00 + 39.423 0.989062 265333 91.43 + 40.031 0.990625 265751 106.67 + 40.735 0.992188 266163 128.00 + 41.087 0.992969 266377 142.22 + 41.471 0.993750 266580 160.00 + 41.951 0.994531 266785 182.86 + 42.463 0.995313 266995 213.33 + 42.943 0.996094 267208 256.00 + 43.231 0.996484 267313 284.44 + 43.519 0.996875 267420 320.00 + 43.775 0.997266 267518 365.71 + 44.191 0.997656 267627 426.67 + 44.543 0.998047 267730 512.00 + 44.799 0.998242 267791 568.89 + 44.959 0.998437 267834 640.00 + 45.215 0.998633 267890 731.43 + 45.439 0.998828 267936 853.33 + 45.695 0.999023 267993 1024.00 + 45.855 0.999121 268016 1137.78 + 46.015 0.999219 268041 1280.00 + 46.175 0.999316 268068 1462.86 + 46.399 0.999414 268093 1706.67 + 46.623 0.999512 268124 2048.00 + 46.687 0.999561 268132 2275.56 + 46.815 0.999609 268147 2560.00 + 46.975 0.999658 268158 2925.71 + 47.231 0.999707 268171 3413.33 + 47.455 0.999756 268184 4096.00 + 47.743 0.999780 268192 4551.11 + 47.807 0.999805 268197 5120.00 + 48.127 0.999829 268204 5851.43 + 48.383 0.999854 268210 6826.67 + 48.639 0.999878 268217 8192.00 + 48.671 0.999890 268220 9102.22 + 48.735 0.999902 268223 10240.00 + 49.183 0.999915 268227 11702.86 + 49.247 0.999927 268230 13653.33 + 49.503 0.999939 268233 16384.00 + 49.663 0.999945 268235 18204.44 + 49.727 0.999951 268236 20480.00 + 49.791 0.999957 268238 23405.71 + 49.855 0.999963 268240 27306.67 + 50.111 0.999969 268241 32768.00 + 50.207 0.999973 268242 36408.89 + 50.271 0.999976 268243 40960.00 + 50.335 0.999979 268244 46811.43 + 50.655 0.999982 268245 54613.33 + 50.655 0.999985 268245 65536.00 + 50.719 0.999986 268246 72817.78 + 50.719 0.999988 268246 81920.00 + 51.103 0.999989 268247 93622.86 + 51.103 0.999991 268247 109226.67 + 51.103 0.999992 268247 131072.00 + 51.167 0.999993 268248 145635.56 + 51.167 0.999994 268248 163840.00 + 51.167 0.999995 268248 187245.71 + 51.167 0.999995 268248 218453.33 + 51.167 0.999996 268248 262144.00 + 51.199 0.999997 268249 291271.11 + 51.199 1.000000 268249 inf +#[Mean = 6.477, StdDeviation = 6.765] +#[Max = 51.168, Total count = 268249] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 323136 requests in 1.00m, 26.64MB read + Non-2xx or 3xx responses: 213631 +Requests/sec: 5385.63 +Transfer/sec: 454.58KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/get_5500rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/get_5500rs_1min.txt new file mode 100644 index 000000000..887bd3f52 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/get_5500rs_1min.txt @@ -0,0 +1,121 @@ +dmitrij@PC ~/D/j/wrk2 (master)> ./wrk -d 60 -t 1 -c 64 -R 5500 -L -s /home/dmitrij/Documents/javaProjects/2024-highload-dht/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_scripts/get.lua http://localhost:8080 +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 21.435ms, rate sampling interval: 111ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 99.95ms 68.31ms 295.42ms 62.70% + Req/Sec 5.50k 327.71 9.96k 97.99% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 91.78ms + 75.000% 143.87ms + 90.000% 205.70ms + 99.000% 255.23ms + 99.900% 281.09ms + 99.990% 291.58ms + 99.999% 294.91ms +100.000% 295.68ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.415 0.000000 1 1.00 + 17.567 0.100000 27193 1.11 + 32.927 0.200000 54390 1.25 + 53.183 0.300000 81582 1.43 + 72.127 0.400000 108789 1.67 + 91.775 0.500000 135992 2.00 + 101.119 0.550000 149581 2.22 + 110.719 0.600000 163153 2.50 + 120.959 0.650000 176718 2.86 + 131.711 0.700000 190337 3.33 + 143.871 0.750000 203976 4.00 + 151.167 0.775000 210793 4.44 + 159.999 0.800000 217568 5.00 + 170.495 0.825000 224344 5.71 + 181.631 0.850000 231129 6.67 + 193.535 0.875000 237917 8.00 + 199.295 0.887500 241287 8.89 + 205.695 0.900000 244710 10.00 + 212.095 0.912500 248107 11.43 + 218.495 0.925000 251523 13.33 + 224.895 0.937500 254944 16.00 + 227.583 0.943750 256593 17.78 + 230.271 0.950000 258376 20.00 + 232.703 0.956250 260008 22.86 + 235.647 0.962500 261730 26.67 + 238.847 0.968750 263406 32.00 + 240.639 0.971875 264280 35.56 + 242.559 0.975000 265125 40.00 + 244.735 0.978125 265961 45.71 + 247.423 0.981250 266808 53.33 + 249.855 0.984375 267663 64.00 + 251.007 0.985938 268084 71.11 + 252.287 0.987500 268501 80.00 + 253.951 0.989062 268903 91.43 + 256.127 0.990625 269327 106.67 + 258.687 0.992188 269771 128.00 + 259.967 0.992969 269960 142.22 + 262.399 0.993750 270183 160.00 + 264.191 0.994531 270400 182.86 + 265.983 0.995313 270603 213.33 + 267.775 0.996094 270826 256.00 + 268.543 0.996484 270925 284.44 + 269.823 0.996875 271027 320.00 + 271.103 0.997266 271132 365.71 + 273.151 0.997656 271237 426.67 + 274.943 0.998047 271340 512.00 + 276.223 0.998242 271400 568.89 + 278.015 0.998437 271451 640.00 + 279.295 0.998633 271508 731.43 + 280.319 0.998828 271558 853.33 + 281.343 0.999023 271621 1024.00 + 281.599 0.999121 271635 1137.78 + 282.111 0.999219 271659 1280.00 + 283.135 0.999316 271685 1462.86 + 284.671 0.999414 271717 1706.67 + 285.439 0.999512 271741 2048.00 + 285.695 0.999561 271750 2275.56 + 286.719 0.999609 271765 2560.00 + 287.487 0.999658 271779 2925.71 + 288.255 0.999707 271795 3413.33 + 288.511 0.999756 271803 4096.00 + 289.023 0.999780 271813 4551.11 + 289.791 0.999805 271820 5120.00 + 290.047 0.999829 271824 5851.43 + 290.559 0.999854 271832 6826.67 + 290.815 0.999878 271836 8192.00 + 291.583 0.999890 271844 9102.22 + 291.583 0.999902 271844 10240.00 + 291.839 0.999915 271847 11702.86 + 292.351 0.999927 271850 13653.33 + 293.375 0.999939 271854 16384.00 + 293.631 0.999945 271856 18204.44 + 293.631 0.999951 271856 20480.00 + 294.399 0.999957 271859 23405.71 + 294.911 0.999963 271866 27306.67 + 294.911 0.999969 271866 32768.00 + 294.911 0.999973 271866 36408.89 + 294.911 0.999976 271866 40960.00 + 294.911 0.999979 271866 46811.43 + 294.911 0.999982 271866 54613.33 + 294.911 0.999985 271866 65536.00 + 294.911 0.999986 271866 72817.78 + 294.911 0.999988 271866 81920.00 + 295.423 0.999989 271868 93622.86 + 295.423 0.999991 271868 109226.67 + 295.423 0.999992 271868 131072.00 + 295.423 0.999993 271868 145635.56 + 295.423 0.999994 271868 163840.00 + 295.423 0.999995 271868 187245.71 + 295.423 0.999995 271868 218453.33 + 295.423 0.999996 271868 262144.00 + 295.679 0.999997 271869 291271.11 + 295.679 1.000000 271869 inf +#[Mean = 99.947, StdDeviation = 68.306] +#[Max = 295.424, Total count = 271869] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 327783 requests in 1.00m, 26.99MB read + Non-2xx or 3xx responses: 217252 +Requests/sec: 5463.02 +Transfer/sec: 460.65KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/put_6800rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/put_6800rs_1min.txt new file mode 100644 index 000000000..2f5806a6d --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/put_6800rs_1min.txt @@ -0,0 +1,122 @@ +dmitrij@PC ~/D/j/wrk2 (master)> ./wrk -d 60 -t 1 -c 64 -R 6800 -L -s /home/dmitrij/Documents/javaProjects/2024-highload-dht/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_scripts/put.lua http://localhost:8080 +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 9.699ms, rate sampling interval: 65ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 17.64ms 19.49ms 96.06ms 80.56% + Req/Sec 6.85k 661.81 12.12k 73.92% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 8.32ms + 75.000% 30.13ms + 90.000% 48.10ms + 99.000% 72.45ms + 99.900% 84.03ms + 99.990% 90.37ms + 99.999% 94.27ms +100.000% 96.13ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.372 0.000000 1 1.00 + 1.237 0.100000 33785 1.11 + 1.632 0.200000 67577 1.25 + 2.129 0.300000 101399 1.43 + 2.961 0.400000 135132 1.67 + 8.319 0.500000 168926 2.00 + 12.391 0.550000 185804 2.22 + 16.431 0.600000 202740 2.50 + 20.559 0.650000 219606 2.86 + 24.751 0.700000 236487 3.33 + 30.127 0.750000 253366 4.00 + 33.375 0.775000 261855 4.44 + 36.415 0.800000 270260 5.00 + 39.583 0.825000 278779 5.71 + 42.335 0.850000 287159 6.67 + 44.991 0.875000 295654 8.00 + 46.495 0.887500 299908 8.89 + 48.095 0.900000 304064 10.00 + 49.855 0.912500 308331 11.43 + 51.711 0.925000 312497 13.33 + 53.887 0.937500 316731 16.00 + 55.167 0.943750 318857 17.78 + 56.607 0.950000 320942 20.00 + 58.079 0.956250 323071 22.86 + 59.711 0.962500 325166 26.67 + 61.823 0.968750 327275 32.00 + 63.007 0.971875 328339 35.56 + 64.223 0.975000 329387 40.00 + 65.503 0.978125 330457 45.71 + 66.943 0.981250 331490 53.33 + 68.671 0.984375 332563 64.00 + 69.567 0.985938 333091 71.11 + 70.591 0.987500 333620 80.00 + 71.679 0.989062 334144 91.43 + 72.959 0.990625 334665 106.67 + 74.367 0.992188 335190 128.00 + 75.071 0.992969 335451 142.22 + 75.839 0.993750 335713 160.00 + 76.607 0.994531 335991 182.86 + 77.375 0.995313 336243 213.33 + 78.271 0.996094 336508 256.00 + 78.847 0.996484 336641 284.44 + 79.423 0.996875 336774 320.00 + 79.999 0.997266 336914 365.71 + 80.575 0.997656 337035 426.67 + 81.343 0.998047 337162 512.00 + 81.791 0.998242 337231 568.89 + 82.367 0.998437 337300 640.00 + 82.815 0.998633 337361 731.43 + 83.391 0.998828 337426 853.33 + 84.095 0.999023 337492 1024.00 + 84.607 0.999121 337529 1137.78 + 84.927 0.999219 337557 1280.00 + 85.631 0.999316 337597 1462.86 + 86.143 0.999414 337624 1706.67 + 86.719 0.999512 337659 2048.00 + 86.911 0.999561 337678 2275.56 + 87.167 0.999609 337689 2560.00 + 87.487 0.999658 337707 2925.71 + 87.999 0.999707 337723 3413.33 + 88.447 0.999756 337741 4096.00 + 88.575 0.999780 337748 4551.11 + 88.831 0.999805 337755 5120.00 + 89.215 0.999829 337763 5851.43 + 89.663 0.999854 337774 6826.67 + 89.983 0.999878 337779 8192.00 + 90.175 0.999890 337785 9102.22 + 90.431 0.999902 337788 10240.00 + 90.559 0.999915 337792 11702.86 + 91.071 0.999927 337796 13653.33 + 91.327 0.999939 337800 16384.00 + 91.647 0.999945 337802 18204.44 + 91.967 0.999951 337804 20480.00 + 92.287 0.999957 337806 23405.71 + 92.671 0.999963 337808 27306.67 + 92.799 0.999969 337810 32768.00 + 92.863 0.999973 337811 36408.89 + 92.991 0.999976 337813 40960.00 + 92.991 0.999979 337813 46811.43 + 93.375 0.999982 337814 54613.33 + 93.695 0.999985 337815 65536.00 + 94.079 0.999986 337816 72817.78 + 94.079 0.999988 337816 81920.00 + 94.271 0.999989 337817 93622.86 + 94.271 0.999991 337817 109226.67 + 94.719 0.999992 337818 131072.00 + 94.719 0.999993 337818 145635.56 + 94.719 0.999994 337818 163840.00 + 96.063 0.999995 337819 187245.71 + 96.063 0.999995 337819 218453.33 + 96.063 0.999996 337819 262144.00 + 96.063 0.999997 337819 291271.11 + 96.063 0.999997 337819 327680.00 + 96.127 0.999997 337820 374491.43 + 96.127 1.000000 337820 inf +#[Mean = 17.640, StdDeviation = 19.486] +#[Max = 96.064, Total count = 337820] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 406945 requests in 1.00m, 26.00MB read +Requests/sec: 6782.39 +Transfer/sec: 443.77KB diff --git a/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/put_6900rs_1min.txt b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/put_6900rs_1min.txt new file mode 100644 index 000000000..22cfc67aa --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_results/stage4/put_6900rs_1min.txt @@ -0,0 +1,122 @@ +dmitrij@PC ~/D/j/wrk2 (master)> ./wrk -d 60 -t 1 -c 64 -R 6900 -L -s /home/dmitrij/Documents/javaProjects/2024-highload-dht/src/main/java/ru/vk/itmo/test/tyapuevdmitrij/wrk2_scripts/put.lua http://localhost:8080 +Running 1m test @ http://localhost:8080 + 1 threads and 64 connections + Thread calibration: mean lat.: 9.926ms, rate sampling interval: 63ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 20.81ms 24.33ms 117.38ms 80.17% + Req/Sec 6.94k 753.00 13.40k 78.93% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 6.01ms + 75.000% 35.74ms + 90.000% 60.51ms + 99.000% 89.41ms + 99.900% 102.14ms + 99.990% 109.25ms + 99.999% 115.78ms +100.000% 117.44ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.327 0.000000 1 1.00 + 1.485 0.100000 34240 1.11 + 2.018 0.200000 68440 1.25 + 2.595 0.300000 102722 1.43 + 3.633 0.400000 136896 1.67 + 6.007 0.500000 171113 2.00 + 9.823 0.550000 188219 2.22 + 16.879 0.600000 205320 2.50 + 23.407 0.650000 222446 2.86 + 28.735 0.700000 239549 3.33 + 35.743 0.750000 256634 4.00 + 40.511 0.775000 265222 4.44 + 44.895 0.800000 273757 5.00 + 48.895 0.825000 282300 5.71 + 52.831 0.850000 290887 6.67 + 56.639 0.875000 299461 8.00 + 58.623 0.887500 303706 8.89 + 60.511 0.900000 307956 10.00 + 62.495 0.912500 312237 11.43 + 64.511 0.925000 316528 13.33 + 66.751 0.937500 320864 16.00 + 67.967 0.943750 323006 17.78 + 69.311 0.950000 325122 20.00 + 70.911 0.956250 327264 22.86 + 72.895 0.962500 329360 26.67 + 75.519 0.968750 331479 32.00 + 77.119 0.971875 332553 35.56 + 78.975 0.975000 333618 40.00 + 81.151 0.978125 334699 45.71 + 83.263 0.981250 335758 53.33 + 85.375 0.984375 336853 64.00 + 86.335 0.985938 337377 71.11 + 87.487 0.987500 337896 80.00 + 88.703 0.989062 338442 91.43 + 89.855 0.990625 338972 106.67 + 91.135 0.992188 339505 128.00 + 91.903 0.992969 339779 142.22 + 92.607 0.993750 340033 160.00 + 93.567 0.994531 340323 182.86 + 94.399 0.995313 340589 213.33 + 95.295 0.996094 340843 256.00 + 95.935 0.996484 340988 284.44 + 96.575 0.996875 341103 320.00 + 97.343 0.997266 341247 365.71 + 98.239 0.997656 341377 426.67 + 99.135 0.998047 341511 512.00 + 99.519 0.998242 341570 568.89 + 100.159 0.998437 341643 640.00 + 100.735 0.998633 341707 731.43 + 101.439 0.998828 341774 853.33 + 102.207 0.999023 341838 1024.00 + 102.655 0.999121 341876 1137.78 + 103.039 0.999219 341906 1280.00 + 103.551 0.999316 341944 1462.86 + 104.063 0.999414 341974 1706.67 + 104.575 0.999512 342005 2048.00 + 104.831 0.999561 342021 2275.56 + 105.215 0.999609 342039 2560.00 + 105.791 0.999658 342056 2925.71 + 106.111 0.999707 342073 3413.33 + 106.431 0.999756 342088 4096.00 + 106.815 0.999780 342096 4551.11 + 107.071 0.999805 342105 5120.00 + 107.519 0.999829 342113 5851.43 + 107.967 0.999854 342121 6826.67 + 108.863 0.999878 342130 8192.00 + 109.119 0.999890 342134 9102.22 + 109.311 0.999902 342138 10240.00 + 110.079 0.999915 342142 11702.86 + 110.783 0.999927 342148 13653.33 + 110.975 0.999939 342152 16384.00 + 111.103 0.999945 342153 18204.44 + 111.807 0.999951 342155 20480.00 + 111.999 0.999957 342157 23405.71 + 112.767 0.999963 342159 27306.67 + 113.215 0.999969 342161 32768.00 + 114.047 0.999973 342162 36408.89 + 114.623 0.999976 342163 40960.00 + 115.071 0.999979 342164 46811.43 + 115.199 0.999982 342165 54613.33 + 115.391 0.999985 342166 65536.00 + 115.711 0.999986 342167 72817.78 + 115.711 0.999988 342167 81920.00 + 115.775 0.999989 342168 93622.86 + 115.775 0.999991 342168 109226.67 + 115.903 0.999992 342169 131072.00 + 115.903 0.999993 342169 145635.56 + 115.903 0.999994 342169 163840.00 + 116.351 0.999995 342170 187245.71 + 116.351 0.999995 342170 218453.33 + 116.351 0.999996 342170 262144.00 + 116.351 0.999997 342170 291271.11 + 116.351 0.999997 342170 327680.00 + 117.439 0.999997 342171 374491.43 + 117.439 1.000000 342171 inf +#[Mean = 20.809, StdDeviation = 24.330] +#[Max = 117.376, Total count = 342171] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 406473 requests in 1.00m, 25.97MB read +Requests/sec: 6774.61 +Transfer/sec: 443.26KB diff --git a/src/main/resources/smirnovdmitrii/application.properties b/src/main/resources/smirnovdmitrii/application.properties index 429bcd31d..c574dda55 100644 --- a/src/main/resources/smirnovdmitrii/application.properties +++ b/src/main/resources/smirnovdmitrii/application.properties @@ -23,3 +23,5 @@ local.ports=8080,8081,8082 local.hosts=localhost,localhost,localhost server.virtualThreads.enable=false + +server.use.async=false diff --git a/src/test/java/ru/vk/itmo/ServiceInfo.java b/src/test/java/ru/vk/itmo/ServiceInfo.java index edd829172..fe140344c 100644 --- a/src/test/java/ru/vk/itmo/ServiceInfo.java +++ b/src/test/java/ru/vk/itmo/ServiceInfo.java @@ -48,6 +48,13 @@ public HttpResponse get(String key, int ack, int from) throws Exception ); } + public HttpResponse range(String start, String end) throws Exception { + return client.send( + requestForRange(start, end).GET().build(), + HttpResponse.BodyHandlers.ofByteArray() + ); + } + public HttpResponse delete(String key) throws Exception { return client.send( requestForKey(key).DELETE().build(), @@ -94,4 +101,8 @@ private HttpRequest.Builder requestForKey(String key) { private HttpRequest.Builder requestForKey(String key, int ack, int from) { return request(STR."/v0/entity?id=\{key}&from=\{from}&ack=\{ack}"); } + + private HttpRequest.Builder requestForRange(String start, String end) { + return request(STR."/v0/entities?start=\{start}\{end == null ? "" : (STR."&end=\{end}")}"); + } } diff --git a/src/test/java/ru/vk/itmo/ServiceTest.java b/src/test/java/ru/vk/itmo/ServiceTest.java index c57190806..63819b1dd 100644 --- a/src/test/java/ru/vk/itmo/ServiceTest.java +++ b/src/test/java/ru/vk/itmo/ServiceTest.java @@ -41,7 +41,7 @@ @ParameterizedTest @ArgumentsSource(ServiceTest.ServiceList.class) @ExtendWith(ServiceTest.ServiceList.class) -@Timeout(value = 1, unit = TimeUnit.MINUTES) +@Timeout(value = 1, unit = TimeUnit.HOURS) public @interface ServiceTest { int stage(); int clusterSize() default 1; diff --git a/src/test/java/ru/vk/itmo/SingleRangeTest.java b/src/test/java/ru/vk/itmo/SingleRangeTest.java new file mode 100644 index 000000000..e433606a6 --- /dev/null +++ b/src/test/java/ru/vk/itmo/SingleRangeTest.java @@ -0,0 +1,149 @@ +package ru.vk.itmo; + +import java.net.HttpURLConnection; +import java.net.http.HttpResponse; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Unit tests for single node range API. + * + * @author Vadim Tsesko + */ +class SingleRangeTest extends TestBase { + private static byte[] chunkOf( + String key, + String value) { + return (key + '\n' + value).getBytes(UTF_8); + } + + @ServiceTest(stage = 6) + void emptyKey(ServiceInfo service) throws Exception { + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, service.range("", "").statusCode()); + assertEquals(HttpURLConnection.HTTP_BAD_REQUEST, service.upsert("", new byte[]{0}).statusCode()); + } + + @ServiceTest(stage = 6) + void absentParameterRequest(ServiceInfo service) throws Exception { + assertEquals( + HttpURLConnection.HTTP_BAD_REQUEST, + client.send(service.request("/v0/entities").GET().build(), + HttpResponse.BodyHandlers.ofByteArray()).statusCode() + ); + assertEquals( + HttpURLConnection.HTTP_BAD_REQUEST, + client.send(service.request("/v0/entities?end=end").GET().build(), + HttpResponse.BodyHandlers.ofByteArray()).statusCode() + ); + } + + @ServiceTest(stage = 6) + void getAbsent(ServiceInfo service) throws Exception { + HttpResponse response = service.range("absent0", "absent1"); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertEquals(0, response.body().length); + } + + @ServiceTest(stage = 6) + void single(ServiceInfo service) throws Exception { + String prefix = "single"; + String key = prefix + 1; + String value = "value1"; + + // Insert + assertEquals(HttpURLConnection.HTTP_CREATED, service.upsert(key, value.getBytes(UTF_8)).statusCode()); + + // Check + { + HttpResponse response = service.range(key, prefix + 2); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertArrayEquals(chunkOf(key, value), response.body()); + } + + // Excluding the key + { + HttpResponse response = service.range("a", key); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertEquals(0, response.body().length); + } + + // After the key + { + HttpResponse response = service.range(prefix + 2, prefix + 3); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertEquals(0, response.body().length); + } + } + + @ServiceTest(stage = 6) + void triple(ServiceInfo service) throws Exception { + String prefix = "triple"; + String value1 = "value1"; + String value2 = ""; + String value3 = "value3"; + + // Insert reversed + assertEquals(HttpURLConnection.HTTP_CREATED, service.upsert(prefix + 3, value3.getBytes(UTF_8)).statusCode()); + assertEquals(HttpURLConnection.HTTP_CREATED, service.upsert(prefix + 2, value2.getBytes(UTF_8)).statusCode()); + assertEquals(HttpURLConnection.HTTP_CREATED, service.upsert(prefix + 1, value1.getBytes(UTF_8)).statusCode()); + + // Check all + { + byte[] chunk1 = chunkOf(prefix + 1, value1); + byte[] chunk2 = chunkOf(prefix + 2, value2); + byte[] chunk3 = chunkOf(prefix + 3, value3); + byte[] expected = new byte[chunk1.length + chunk2.length + chunk3.length]; + System.arraycopy(chunk1, 0, expected, 0, chunk1.length); + System.arraycopy(chunk2, 0, expected, chunk1.length, chunk2.length); + System.arraycopy(chunk3, 0, expected, expected.length - chunk3.length, chunk3.length); + + HttpResponse response = service.range(prefix + 1, prefix + 4); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertArrayEquals(expected, response.body()); + } + + // To the left + { + HttpResponse response = service.range(prefix + 0, prefix + 1); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertEquals(0, response.body().length); + } + + // First left + { + HttpResponse response = service.range(prefix + 0, prefix + 2); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertArrayEquals(chunkOf(prefix + 1, value1), response.body()); + } + + // First point + { + HttpResponse response = service.range(prefix + 1, prefix + 2); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertArrayEquals(chunkOf(prefix + 1, value1), response.body()); + } + + // Second point + { + HttpResponse response = service.range(prefix + 2, prefix + 3); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertArrayEquals(chunkOf(prefix + 2, value2), response.body()); + } + + // Third point + { + HttpResponse response = service.range(prefix + 3, prefix + 4); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertArrayEquals(chunkOf(prefix + 3, value3), response.body()); + } + + // To the right + { + HttpResponse response = service.range(prefix + 4, null); + assertEquals(HttpURLConnection.HTTP_OK, response.statusCode()); + assertEquals(0, response.body().length); + } + } +}