diff --git a/.DS_Store b/.DS_Store index db68ea8..6de347d 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index 3cdb695..e7bbb62 100644 --- a/.gitignore +++ b/.gitignore @@ -198,3 +198,5 @@ go/cmd/gcsobjtable/gcsobjtable python/babyray/rayclient_pb2_grpc.py-e go/cmd/localobjstore/localobjstore go/cmd/localscheduler/localscheduler +gcsobjtable.db +.DS_Store diff --git a/Dockerfile b/Dockerfile index 168934b..8f29f81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,15 @@ ARG CONFIG=base # Base stage for common setup FROM golang as base -RUN apt update -RUN apt install -y protobuf-compiler +# Install gcc and other necessary tools +# we need gcc for Golang SQLite bc uses C code +RUN apt-get update && \ + apt-get install -y gcc && \ + apt-get install -y protobuf-compiler && \ + apt-get clean + +# Set CGO_ENABLED=1 , we need this for Golang SQLite bc uses C code +ENV CGO_ENABLED=1 # Set the Current Working Directory inside the container WORKDIR /app @@ -28,6 +35,10 @@ RUN python3 -m pip install grpcio --break-system-packages RUN python3 -m pip install grpcio-tools --break-system-packages +# For figure10b +# RUN python3 -m pip install matplotlib + +# RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 diff --git a/Makefile b/Makefile index fc0e0cf..3a40457 100644 --- a/Makefile +++ b/Makefile @@ -25,27 +25,27 @@ servers: gcsfunctable gcsobjtable globalscheduler localobjstore localscheduler w gcsfunctable: @echo "Building GCS Function Table Server..." - cd go && go build -o bin/gcsfunctable cmd/gcsfunctable/main.go + cd go && go build -o bin/gcsfunctable cmd/gcsfunctable/*.go gcsobjtable: @echo "Building GCS Object Table Server..." - cd go && go build -o bin/gcsobjtable cmd/gcsobjtable/main.go + cd go && go build -o bin/gcsobjtable cmd/gcsobjtable/*.go globalscheduler: @echo "Building Global Scheduler Server..." - cd go && go build -o bin/globalscheduler cmd/globalscheduler/main.go + cd go && go build -o bin/globalscheduler cmd/globalscheduler/*.go localobjstore: @echo "Building Local Object Store Server..." - cd go && go build -o bin/localobjstore cmd/localobjstore/main.go + cd go && go build -o bin/localobjstore cmd/localobjstore/*.go localscheduler: @echo "Building Local Scheduler Server..." - cd go && go build -o bin/localscheduler cmd/localscheduler/main.go + cd go && go build -o bin/localscheduler cmd/localscheduler/*.go worker: @echo "Building Worker Server..." - cd go && go build -o bin/worker cmd/worker/main.go + cd go && go build -o bin/worker cmd/worker/*.go docker: docker build --build-arg CONFIG=base -t ray-node:base . diff --git a/analysis/README.md b/analysis/README.md new file mode 100644 index 0000000..116e292 --- /dev/null +++ b/analysis/README.md @@ -0,0 +1,9 @@ +# analysis +`output_10b_run1.txt`: Results of 10b run1, aka no caching, 2GB memory limit + +* After runs 1-5, realized we've been measuring memory on worker and not the GCS. need to change that!! + +* Run 6 is flush to disk every 1s using AOF and garbage collection +* Run 7 is the new baseline, don't flush to disk. + +run 6 vs run 8 https://chatgpt.com/c/76bd6aef-f711-4067-b626-31446db56773 \ No newline at end of file diff --git a/analysis/fig10b_graphs/attempt2_blurry.png b/analysis/fig10b_graphs/attempt2_blurry.png new file mode 100644 index 0000000..38dd305 Binary files /dev/null and b/analysis/fig10b_graphs/attempt2_blurry.png differ diff --git a/analysis/fig10b_graphs/attempt2_sharp.png b/analysis/fig10b_graphs/attempt2_sharp.png new file mode 100644 index 0000000..2b89c75 Binary files /dev/null and b/analysis/fig10b_graphs/attempt2_sharp.png differ diff --git a/analysis/fig10b_graphs/attempt_Figure_1.png b/analysis/fig10b_graphs/attempt_Figure_1.png new file mode 100644 index 0000000..18dd30f Binary files /dev/null and b/analysis/fig10b_graphs/attempt_Figure_1.png differ diff --git a/analysis/fig10b_graphs/output a(1).png b/analysis/fig10b_graphs/output a(1).png new file mode 100644 index 0000000..a7309eb Binary files /dev/null and b/analysis/fig10b_graphs/output a(1).png differ diff --git a/analysis/fig10b_graphs/outputa(2).png b/analysis/fig10b_graphs/outputa(2).png new file mode 100644 index 0000000..a061c27 Binary files /dev/null and b/analysis/fig10b_graphs/outputa(2).png differ diff --git a/analysis/fig10b_graphs/outputa.png b/analysis/fig10b_graphs/outputa.png new file mode 100644 index 0000000..6e207e6 Binary files /dev/null and b/analysis/fig10b_graphs/outputa.png differ diff --git a/analysis/fig10b_graphs/run_6_vs_run8_1.png b/analysis/fig10b_graphs/run_6_vs_run8_1.png new file mode 100644 index 0000000..32dfb2c Binary files /dev/null and b/analysis/fig10b_graphs/run_6_vs_run8_1.png differ diff --git a/analysis/fig10b_graphs/run_6_vs_run8_2.png b/analysis/fig10b_graphs/run_6_vs_run8_2.png new file mode 100644 index 0000000..31cd3f2 Binary files /dev/null and b/analysis/fig10b_graphs/run_6_vs_run8_2.png differ diff --git a/analysis/fig10b_graphs/run_6_vs_run8_3.png b/analysis/fig10b_graphs/run_6_vs_run8_3.png new file mode 100644 index 0000000..b2b7fbd Binary files /dev/null and b/analysis/fig10b_graphs/run_6_vs_run8_3.png differ diff --git a/analysis/output_10b_run1.txt b/analysis/output_10b_run1.txt new file mode 100644 index 0000000..ec90c45 --- /dev/null +++ b/analysis/output_10b_run1.txt @@ -0,0 +1,406 @@ +worker1-1 | 2024/06/04 01:44:07.510 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 01:44:07.524 FIG10b: Starting tasks submission without GCS flushing... +worker1-1 | 2024/06/04 01:44:08.529 FIG10b: Memory usage without flush after 123 tasks: 970.90234375 MB +worker1-1 | 2024/06/04 01:44:09.530 FIG10b: Memory usage without flush after 245 tasks: 977.0 MB +worker1-1 | 2024/06/04 01:44:10.533 FIG10b: Memory usage without flush after 373 tasks: 986.859375 MB +worker1-1 | 2024/06/04 01:44:11.533 FIG10b: Memory usage without flush after 500 tasks: 987.625 MB +worker1-1 | 2024/06/04 01:44:12.538 FIG10b: Memory usage without flush after 622 tasks: 989.30078125 MB +worker1-1 | 2024/06/04 01:44:13.542 [worker]2024/06/04 01:44:13.544 FIG10b: Memory usage without flush after 750 tasks: 990.41015625 MB +worker1-1 | 2024/06/04 01:44:14.551 FIG10b: Memory usage without flush after 862 tasks: 991.5859375 MB +worker1-1 | 2024/06/04 01:44:15.558 FIG10b: Memory usage without flush after 980 tasks: 991.37109375 MB +worker1-1 | 2024/06/04 01:44:16.561 2024/06/04 01:44:16.562 FIG10b: Memory usage without flush after 1106 tasks: 997.9453125 MB +worker1-1 | stored... 2024/06/04 01:44:17.564 FIG10b: Memory usage without flush after 1222 tasks: 998.30859375 MB +worker1-1 | 2024/06/04 01:44:18.565 FIG10b: Memory usage without flush after 1339 tasks: 999.69140625 MB +worker1-1 | 2024/06/04 01:44:19.571 FIG10b: Memory usage without flush after 1449 tasks: 999.6328125 MB +worker1-1 | 2024/06/04 01:44:20.578 [worker] 2024/06/04 01:44:20.578 FIG10b: Memory usage without flush after 1574 tasks: 999.58203125 MB +worker1-1 | 2024/06/04 01:44:21.582 FIG10b: Memory usage without flush after 1697 tasks: 1001.9453125 MB +worker1-1 | 2024/06/04 01:44:22.593 2024/06/04 01:44:22.593[worker] FIG10b: Memory usage without flush after 1829 tasks: 1002.6328125 MB +worker1-1 | 2024/06/04 01:44:23.603 FIG10b: Memory usage without flush after 1961 tasks: 1002.921875 MB +worker1-1 | 2024/06/04 01:44:24.613 2024/06/04 01:44:24.614 FIG10b: Memory usage without flush after 2088 tasks: 1003.82421875 MB +worker1-1 | FIG10b: Memory usage without flush after 2213 tasks: 1003.9140625 MB +worker1-1 | 2024/06/04 01:44:26.620 FIG10b: Memory usage without flush after 2332 tasks: 1003.6015625 MB +worker1-1 | 2024/06/04 01:44:27.620 FIG10b: Memory usage without flush after 2460 tasks: 1005.21875 MB +worker1-1 | 2024/06/04 01:44:28.620 FIG10b: Memory usage without flush after 2586 tasks: 1004.91015625 MB +worker1-1 | 2024/06/04 01:44:29.622in Run() rn FIG10b: Memory usage without flush after 2715 tasks: 1006.0546875 MB +worker1-1 | 2024/06/04 01:44:30.623 2024/06/04 01:44:30.623 FIG10b: Memory usage without flush after 2838 tasks: 1007.6953125 MB +worker1-1 | 2024/06/04 01:44:31.6322024/06/04 01:44:31.632 FIG10b: Memory usage without flush after 2965 tasks: 1006.2734375 MB +worker1-1 | 2024/06/04 01:44:32.635 FIG10b: Memory usage without flush after 3088 tasks: 1010.0390625 MB +worker1-1 | [worker] 2024/06/04 01:44:33.639 FIG10b: Memory usage without flush after 3141 tasks: 1014.9296875 MB +worker1-1 | 2024/06/04 01:44:34.655 FIG10b: Memory usage without flush after 3205 tasks: 1013.94140625 MB +worker1-1 | FIG10b: Memory usage without flush after 3296 tasks: 1013.73828125 MB +worker1-1 | 2024/06/04 01:44:36.6602024/06/04 01:44:36.660 FIG10b: Memory usage without flush after 3420 tasks: 1013.67578125 MB +worker1-1 | [worker] 2024/06/04 01:44:37.662 2024/06/04 01:44:37.662 FIG10b: Memory usage without flush after 3548 tasks: 1014.49609375 MB +worker1-1 | 2024/06/04 01:44:38.734 FIG10b: Memory usage without flush after 3603 tasks: 1018.734375 MB +worker1-1 | 2024/06/04 01:44:39.971 FIG10b: Memory usage without flush after 3649 tasks: 1018.75390625 MB +worker1-1 | 2024/06/04 01:44:40.893 FIG10b: Memory usage without flush after 3658 tasks: 1018.98828125 MB +worker1-1 | 2024/06/04 01:44:41.889 FIG10b: Memory usage without flush after 3728 tasks: 1019.6015625 MB +worker1-1 | 2024/06/04 01:44:42.891 FIG10b: Memory usage without flush after 3778 tasks: 1023.62109375 MB +worker1-1 | 2024/06/04 01:44:43.896 FIG10b: Memory usage without flush after 3852 tasks: 1023.48046875 MB +worker1-1 | 2024/06/04 01:44:44.901 FIG10b: Memory usage without flush after 3965 tasks: 1023.890625 MB +worker1-1 | 2024/06/04 01:44:45.907 FIG10b: Memory usage without flush after 4067 tasks: 1024.66015625 MB +worker1-1 | 2024/06/04 01:44:46.908 FIG10b: Memory usage without flush after 4182 tasks: 1025.2265625 MB +worker1-1 | 2024/06/04 01:44:47.913 FIG10b: Memory usage without flush after 4292 tasks: 1024.94140625 MB +worker1-1 | 2024/06/04 01:44:48.9162024/06/04 01:44:48.916 FIG10b: Memory usage without flush after 4402 tasks: 1025.671875 MB +worker1-1 | 2024/06/04 01:44:49.917 FIG10b: Memory usage without flush after 4514 tasks: 1026.1484375 MB +worker1-1 | 2024/06/04 01:44:50.929 FIG10b: Memory usage without flush after 4618 tasks: 1025.8984375 MB +worker1-1 | FIG10b: Memory usage without flush after 4729 tasks: 1025.9140625 MB +worker1-1 | 2024/06/04 01:44:52.9382024/06/04 01:44:52.938 FIG10b: Memory usage without flush after 4842 tasks: 1017.90234375 MB +worker1-1 | 2024/06/04 01:44:53.946 FIG10b: Memory usage without flush after 4971 tasks: 1022.25 MB +worker1-1 | 2024/06/04 01:44:54.955 FIG10b: Memory usage without flush after 5089 tasks: 1021.984375 MB +worker1-1 | 2024/06/04 01:44:55.959 FIG10b: Memory usage without flush after 5203 tasks: 1021.9375 MB +worker1-1 | 2024/06/04 01:44:56.961 FIG10b: Memory usage without flush after 5329 tasks: 1022.0546875 MB +worker1-1 | 2024/06/04 01:44:57.967 FIG10b: Memory usage without flush after 5450 tasks: 1022.19140625 MB +worker1-1 | FIG10b: Memory usage without flush after 5544 tasks: 1021.75390625 MB +worker1-1 | [worker]2024/06/04 01:44:59.970 FIG10b: Memory usage without flush after 5669 tasks: 1021.70703125 MB +worker1-1 | 2024/06/04 01:45:00.974 FIG10b: Memory usage without flush after 5772 tasks: 1021.69921875 MB +worker1-1 | 2024/06/04 01:45:01.975 [worker] took this many seconds: 0.009632024/06/04 01:45:01.976 FIG10b: Memory usage without flush after 5887 tasks: 1021.63671875 MB +worker1-1 | 2024/06/04 01:45:02.982 FIG10b: Memory usage without flush after 6011 tasks: 1021.86328125 MB +worker1-1 | 2024/06/04 01:45:03.990 FIG10b: Memory usage without flush after 6144 tasks: 1021.4609375 MB +worker1-1 | 2024/06/04 01:45:04.9962024/06/04 01:45:04.996 FIG10b: Memory usage without flush after 6274 tasks: 1022.5234375 MB +worker1-1 | 2024/06/04 01:45:06.003 FIG10b: Memory usage without flush after 6402 tasks: 1022.73828125 MB +worker1-1 | took this many seconds: 0.01812024/06/04 01:45:07.004 FIG10b: Memory usage without flush after 6523 tasks: 1023.6640625 MB +worker1-1 | 2024/06/04 01:45:08.0142024/06/04 01:45:08.014[worker] FIG10b: Memory usage without flush after 6644 tasks: 1023.75390625 MB +worker1-1 | 2024/06/04 01:45:09.0182024/06/04 01:45:09.018 FIG10b: Memory usage without flush after 6766 tasks: 1026.2109375 MB +worker1-1 | 2024/06/04 01:45:10.023 FIG10b: Memory usage without flush after 6894 tasks: 1026.3515625 MB +worker1-1 | 2024/06/04 01:45:11.028 FIG10b: Memory usage without flush after 7002 tasks: 1025.765625 MB +worker1-1 | 2024/06/04 01:45:12.038 FIG10b: Memory usage without flush after 7112 tasks: 1025.53125 MB +worker1-1 | [worker] 2024/06/04 01:45:13.046 FIG10b: Memory usage without flush after 7230 tasks: 1025.96484375 MB +worker1-1 | 2024/06/04 01:45:14.047 FIG10b: Memory usage without flush after 7336 tasks: 1026.4296875 MB +worker1-1 | 2024/06/04 01:45:15.051 FIG10b: Memory usage without flush after 7453 tasks: 1026.5234375 MB +worker1-1 | FIG10b: Memory usage without flush after 7576 tasks: 1026.00390625 MB +worker1-1 | 2024/06/04 01:45:17.055 FIG10b: Memory usage without flush after 7687 tasks: 1026.36328125 MB +worker1-1 | 2024/06/04 01:45:18.063 FIG10b: Memory usage without flush after 7794 tasks: 1026.046875 MB +worker1-1 | [worker][worker] 2024/06/04 01:45:19.069 FIG10b: Memory usage without flush after 7928 tasks: 1026.15234375 MB +worker1-1 | FIG10b: Memory usage without flush after 8054 tasks: 1025.4609375 MB +worker1-1 | FIG10b: Memory usage without flush after 8169 tasks: 1025.1171875 MB +worker1-1 | 2024/06/04 01:45:22.081 FIG10b: Memory usage without flush after 8293 tasks: 1026.33203125 MB +worker1-1 | 2024/06/04 01:45:23.084 FIG10b: Memory usage without flush after 8417 tasks: 1026.7421875 MB +worker1-1 | 2024/06/04 01:45:24.0862024/06/04 01:45:24.086 FIG10b: Memory usage without flush after 8547 tasks: 1025.98828125 MB +worker1-1 | 2024/06/04 01:45:25.088 [worker]2024/06/04 01:45:25.089 FIG10b: Memory usage without flush after 8674 tasks: 1027.140625 MB +worker1-1 | gonna store now2024/06/04 01:45:26.097 FIG10b: Memory usage without flush after 8815 tasks: 1026.8828125 MB +worker1-1 | 2024/06/04 01:45:27.102 FIG10b: Memory usage without flush after 8951 tasks: 1028.72265625 MB +worker1-1 | FIG10b: Memory usage without flush after 9083 tasks: 1028.1796875 MB +worker1-1 | 2024/06/04 01:45:29.121 [worker] 2024/06/04 01:45:29.122stored... FIG10b: Memory usage without flush after 9208 tasks: 1028.1796875 MB +worker1-1 | 2024/06/04 01:45:30.134 FIG10b: Memory usage without flush after 9344 tasks: 1028.61328125 MB +worker1-1 | 2024/06/04 01:45:31.138 FIG10b: Memory usage without flush after 9481 tasks: 1029.0859375 MB +worker1-1 | 2024/06/04 01:45:32.145 FIG10b: Memory usage without flush after 9616 tasks: 1029.4609375 MB +worker1-1 | 2024/06/04 01:45:33.153 FIG10b: Memory usage without flush after 9751 tasks: 1029.68359375 MB +worker1-1 | 2024/06/04 01:45:34.154 FIG10b: Memory usage without flush after 9886 tasks: 1030.1015625 MB +worker1-1 | 2024/06/04 01:45:35.163 FIG10b: Memory usage without flush after 10020 tasks: 1030.99609375 MB +worker1-1 | 2024/06/04 01:45:36.171 FIG10b: Memory usage without flush after 10153 tasks: 1029.703125 MB +worker1-1 | 2024/06/04 01:45:37.176 FIG10b: Memory usage without flush after 10285 tasks: 1029.75 MB +worker1-1 | [worker]2024/06/04 01:45:38.178 FIG10b: Memory usage without flush after 10407 tasks: 1029.6171875 MB +worker1-1 | 2024/06/04 01:45:39.180 FIG10b: Memory usage without flush after 10540 tasks: 1029.24609375 MB +worker1-1 | stored...2024/06/04 01:45:40.186 FIG10b: Memory usage without flush after 10661 tasks: 1030.0703125 MB +worker1-1 | stored...2024/06/04 01:45:41.188 FIG10b: Memory usage without flush after 10788 tasks: 1030.15234375 MB +worker1-1 | 2024/06/04 01:45:42.189 FIG10b: Memory usage without flush after 10902 tasks: 1030.2890625 MB +worker1-1 | 2024/06/04 01:45:43.1932024/06/04 01:45:43.194 FIG10b: Memory usage without flush after 11036 tasks: 1030.60546875 MB +worker1-1 | [worker]2024/06/04 01:45:44.197 FIG10b: Memory usage without flush after 11156 tasks: 1030.7578125 MB +worker1-1 | 2024/06/04 01:45:45.206 FIG10b: Memory usage without flush after 11278 tasks: 1031.359375 MB +worker1-1 | 2024/06/04 01:45:46.207 FIG10b: Memory usage without flush after 11407 tasks: 1031.94921875 MB +worker1-1 | 2024/06/04 01:45:47.208 FIG10b: Memory usage without flush after 11544 tasks: 1032.20703125 MB +worker1-1 | 2024/06/04 01:45:48.214 FIG10b: Memory usage without flush after 11685 tasks: 1033.19921875 MB +worker1-1 | 2024/06/04 01:45:49.216 2024/06/04 01:45:49.217 FIG10b: Memory usage without flush after 11813 tasks: 1032.8203125 MB +worker1-1 | 2024/06/04 01:45:50.2172024/06/04 01:45:50.217 FIG10b: Memory usage without flush after 11941 tasks: 1032.90625 MB +worker1-1 | 2024/06/04 01:45:51.221 FIG10b: Memory usage without flush after 12077 tasks: 1032.45703125 MB +worker1-1 | 2024/06/04 01:45:52.223 2024/06/04 01:45:52.2232024/06/04 01:45:52.223 FIG10b: Memory usage without flush after 12208 tasks: 1033.28515625 MB +worker1-1 | 2024/06/04 01:45:53.224 FIG10b: Memory usage without flush after 12331 tasks: 1033.6171875 MB +worker1-1 | 2024/06/04 01:45:54.225 FIG10b: Memory usage without flush after 12460 tasks: 1033.1796875 MB +worker1-1 | FIG10b: Memory usage without flush after 12594 tasks: 1033.953125 MB2024/06/04 01:45:55.226 [lobs] /DEV FILE CREATED! +worker1-1 | [worker][worker]2024/06/04 01:45:56.232 FIG10b: Memory usage without flush after 12726 tasks: 1034.734375 MB +worker1-1 | 2024/06/04 01:45:57.233 FIG10b: Memory usage without flush after 12854 tasks: 1034.3515625 MB +worker1-1 | 2024/06/04 01:45:58.2432024/06/04 01:45:58.243 FIG10b: Memory usage without flush after 12980 tasks: 1035.453125 MB +worker1-1 | 2024/06/04 01:45:59.245 FIG10b: Memory usage without flush after 13093 tasks: 1035.3203125 MB +worker1-1 | 2024/06/04 01:46:00.247 FIG10b: Memory usage without flush after 13213 tasks: 1035.94140625 MB +worker1-1 | 2024/06/04 01:46:01.255 FIG10b: Memory usage without flush after 13335 tasks: 1037.515625 MB +worker1-1 | 2024/06/04 01:46:02.2552024/06/04 01:46:02.255 FIG10b: Memory usage without flush after 13462 tasks: 1036.3359375 MB +worker1-1 | 2024/06/04 01:46:03.259 2024/06/04 01:46:03.260 FIG10b: Memory usage without flush after 13595 tasks: 1037.36328125 MB +worker1-1 | Executed!2024/06/04 01:46:04.266 FIG10b: Memory usage without flush after 13732 tasks: 1037.1640625 MB +worker1-1 | took this many seconds: 0.0147 2024/06/04 01:46:05.273 FIG10b: Memory usage without flush after 13857 tasks: 1037.9765625 MB +worker1-1 | 2024/06/04 01:46:06.2782024/06/04 01:46:06.279 FIG10b: Memory usage without flush after 13988 tasks: 1043.3984375 MB +worker1-1 | 2024/06/04 01:46:07.2822024/06/04 01:46:07.282 FIG10b: Memory usage without flush after 14110 tasks: 1044.56640625 MB +worker1-1 | 2024/06/04 01:46:08.283 2024/06/04 01:46:08.284 FIG10b: Memory usage without flush after 14246 tasks: 1046.42578125 MB +worker1-1 | 2024/06/04 01:46:09.285 2024/06/04 01:46:09.286 FIG10b: Memory usage without flush after 14381 tasks: 1048.125 MB +worker1-1 | 2024/06/04 01:46:10.290 FIG10b: Memory usage without flush after 14511 tasks: 1048.42578125 MB +worker1-1 | gonna store now2024/06/04 01:46:11.295 FIG10b: Memory usage without flush after 14649 tasks: 1047.91796875 MB +worker1-1 | 2024/06/04 01:46:12.298 FIG10b: Memory usage without flush after 14783 tasks: 1049.0859375 MB +worker1-1 | 2024/06/04 01:46:13.298 [worker] stored...2024/06/04 01:46:13.299 FIG10b: Memory usage without flush after 14922 tasks: 1048.06640625 MB +worker1-1 | 2024/06/04 01:46:14.308 FIG10b: Memory usage without flush after 15052 tasks: 1045.19140625 MB +worker1-1 | [worker]2024/06/04 01:46:15.313 FIG10b: Memory usage without flush after 15185 tasks: 1045.45703125 MB +worker1-1 | [worker] 2024/06/04 01:46:16.315 FIG10b: Memory usage without flush after 15310 tasks: 1046.00390625 MB +worker1-1 | 2024/06/04 01:46:17.324 FIG10b: Memory usage without flush after 15451 tasks: 1049.62890625 MB +worker1-1 | 2024/06/04 01:46:18.332 FIG10b: Memory usage without flush after 15580 tasks: 1050.609375 MB +worker1-1 | FIG10b: Memory usage without flush after 15707 tasks: 1051.09375 MB +worker1-1 | 2024/06/04 01:46:20.3362024/06/04 01:46:20.336 FIG10b: Memory usage without flush after 15827 tasks: 1051.26171875 MB +worker1-1 | 2024/06/04 01:46:21.344 FIG10b: Memory usage without flush after 15946 tasks: 1051.9765625 MB +worker1-1 | 2024/06/04 01:46:22.3452024/06/04 01:46:22.345 FIG10b: Memory usage without flush after 16070 tasks: 1052.578125 MB +worker1-1 | 2024/06/04 01:46:23.352 FIG10b: Memory usage without flush after 16203 tasks: 1053.4765625 MB +worker1-1 | 2024/06/04 01:46:24.357 2024/06/04 01:46:24.357 FIG10b: Memory usage without flush after 16329 tasks: 1053.328125 MB +worker1-1 | [worker]2024/06/04 01:46:25.358 FIG10b: Memory usage without flush after 16468 tasks: 1053.9921875 MB +worker1-1 | FIG10b: Memory usage without flush after 16599 tasks: 1054.05078125 MB +worker1-1 | in Run() rn2024/06/04 01:46:27.369 FIG10b: Memory usage without flush after 16723 tasks: 1055.4375 MB +worker1-1 | 2024/06/04 01:46:28.372 FIG10b: Memory usage without flush after 16845 tasks: 1054.71484375 MB +worker1-1 | 2024/06/04 01:46:29.373 FIG10b: Memory usage without flush after 16961 tasks: 1053.77734375 MB +worker1-1 | 2024/06/04 01:46:30.3812024/06/04 01:46:30.3812024/06/04 01:46:30.381 FIG10b: Memory usage without flush after 17087 tasks: 1053.9765625 MB +worker1-1 | 2024/06/04 01:46:31.3822024/06/04 01:46:31.382 FIG10b: Memory usage without flush after 17208 tasks: 1054.89453125 MB[worker] +worker1-1 | 2024/06/04 01:46:32.387 FIG10b: Memory usage without flush after 17331 tasks: 1055.52734375 MB +worker1-1 | 2024/06/04 01:46:33.396 FIG10b: Memory usage without flush after 17465 tasks: 1054.6484375 MB +worker1-1 | 2024/06/04 01:46:34.397 FIG10b: Memory usage without flush after 17590 tasks: 1054.453125 MB +worker1-1 | 2024/06/04 01:46:35.4012024/06/04 01:46:35.400 FIG10b: Memory usage without flush after 17718 tasks: 1057.6875 MB +worker1-1 | 2024/06/04 01:46:36.4012024/06/04 01:46:36.401 FIG10b: Memory usage without flush after 17849 tasks: 1057.37109375 MB +worker1-1 | 2024/06/04 01:46:37.404 FIG10b: Memory usage without flush after 17985 tasks: 1056.26171875 MB +worker1-1 | 2024/06/04 01:46:38.409 FIG10b: Memory usage without flush after 18110 tasks: 1056.53125 MB +worker1-1 | 2024/06/04 01:46:39.409 FIG10b: Memory usage without flush after 18244 tasks: 1056.0234375 MB +worker1-1 | FIG10b: Memory usage without flush after 18372 tasks: 1056.2890625 MB +worker1-1 | 2024/06/04 01:46:41.423 FIG10b: Memory usage without flush after 18501 tasks: 1056.2890625 MB +worker1-1 | 2024/06/04 01:46:42.432 FIG10b: Memory usage without flush after 18623 tasks: 1054.0234375 MB +worker1-1 | FIG10b: Memory usage without flush after 18667 tasks: 1053.7734375 MB +worker1-1 | 2024/06/04 01:46:44.463 FIG10b: Memory usage without flush after 18744 tasks: 1054.8125 MB +worker1-1 | 2024/06/04 01:46:45.4672024/06/04 01:46:45.467 FIG10b: Memory usage without flush after 18889 tasks: 1057.625 MB +worker1-1 | 2024/06/04 01:46:46.468 FIG10b: Memory usage without flush after 19002 tasks: 1057.4609375 MB +worker1-1 | 2024/06/04 01:46:47.471 FIG10b: Memory usage without flush after 19119 tasks: 1057.5390625 MB +worker1-1 | 2024/06/04 01:46:48.477 [worker] 2024/06/04 01:46:48.477 FIG10b: Memory usage without flush after 19241 tasks: 1058.07421875 MB +worker1-1 | 2024/06/04 01:46:49.483 2024/06/04 01:46:49.484 FIG10b: Memory usage without flush after 19368 tasks: 1057.34375 MB +worker1-1 | 2024/06/04 01:46:50.486 FIG10b: Memory usage without flush after 19491 tasks: 1058.125 MB +worker1-1 | Executed!stored...2024/06/04 01:46:51.492 FIG10b: Memory usage without flush after 19623 tasks: 1057.625 MB +worker1-1 | 2024/06/04 01:46:52.494 FIG10b: Memory usage without flush after 19754 tasks: 1057.8671875 MB +worker1-1 | 2024/06/04 01:46:53.494 2024/06/04 01:46:53.495[worker] in Run() rnFIG10b: Memory usage without flush after 19884 tasks: 1058.58203125 MB +worker1-1 | 2024/06/04 01:46:54.505 FIG10b: Memory usage without flush after 20013 tasks: 1058.33984375 MB +worker1-1 | 2024/06/04 01:46:55.507 FIG10b: Memory usage without flush after 20141 tasks: 1058.59375 MB +worker1-1 | [worker]2024/06/04 01:46:56.511 FIG10b: Memory usage without flush after 20257 tasks: 1058.42578125 MB +worker1-1 | 2024/06/04 01:46:57.516 FIG10b: Memory usage without flush after 20374 tasks: 1059.72265625 MB +worker1-1 | 2024/06/04 01:46:58.538 FIG10b: Memory usage without flush after 20494 tasks: 1059.53515625 MB +worker1-1 | 2024/06/04 01:46:59.542 FIG10b: Memory usage without flush after 20609 tasks: 1059.1328125 MB +worker1-1 | 2024/06/04 01:47:00.544 FIG10b: Memory usage without flush after 20720 tasks: 1059.484375 MB +worker1-1 | 2024/06/04 01:47:01.5462024/06/04 01:47:01.546 FIG10b: Memory usage without flush after 20836 tasks: 1059.82421875 MB +worker1-1 | stored...2024/06/04 01:47:02.553 FIG10b: Memory usage without flush after 20970 tasks: 1059.3125 MB +worker1-1 | 2024/06/04 01:47:03.555 FIG10b: Memory usage without flush after 21097 tasks: 1059.3359375 MB +worker1-1 | 2024/06/04 01:47:04.557 2024/06/04 01:47:04.558 FIG10b: Memory usage without flush after 21229 tasks: 1059.5625 MB +worker1-1 | 2024/06/04 01:47:05.560 FIG10b: Memory usage without flush after 21355 tasks: 1059.09375 MB +worker1-1 | 2024/06/04 01:47:06.562 FIG10b: Memory usage without flush after 21487 tasks: 1059.08984375 MB +worker1-1 | 2024/06/04 01:47:07.5692024/06/04 01:47:07.569 FIG10b: Memory usage without flush after 21615 tasks: 1060.8046875 MB +worker1-1 | [worker] 2024/06/04 01:47:08.5772024/06/04 01:47:08.577 FIG10b: Memory usage without flush after 21737 tasks: 1061.99609375 MB +worker1-1 | 2024/06/04 01:47:09.579 FIG10b: Memory usage without flush after 21864 tasks: 1062.77734375 MB +worker1-1 | 2024/06/04 01:47:10.580 FIG10b: Memory usage without flush after 21993 tasks: 1062.42578125 MB +worker1-1 | 2024/06/04 01:47:11.581 FIG10b: Memory usage without flush after 22125 tasks: 1062.2421875 MB +worker1-1 | 2024/06/04 01:47:12.5852024/06/04 01:47:12.586 FIG10b: Memory usage without flush after 22254 tasks: 1062.734375 MB +worker1-1 | 2024/06/04 01:47:13.587 2024/06/04 01:47:13.587 FIG10b: Memory usage without flush after 22390 tasks: 1063.77734375 MB +worker1-1 | 2024/06/04 01:47:14.5962024/06/04 01:47:14.597 FIG10b: Memory usage without flush after 22520 tasks: 1062.53125 MB +worker1-1 | 2024/06/04 01:47:15.6032024/06/04 01:47:15.604 [worker] Executed! FIG10b: Memory usage without flush after 22655 tasks: 1063.47265625 MB +worker1-1 | 2024/06/04 01:47:16.607 FIG10b: Memory usage without flush after 22779 tasks: 1064.97265625 MB +worker1-1 | FIG10b: Memory usage without flush after 22911 tasks: 1065.3125 MB +worker1-1 | 2024/06/04 01:47:18.6122024/06/04 01:47:18.612 [worker]FIG10b: Memory usage without flush after 23053 tasks: 1065.88671875 MB took this many seconds: 0.00629 +worker1-1 | 2024/06/04 01:47:19.613 FIG10b: Memory usage without flush after 23177 tasks: 1066.9296875 MB +worker1-1 | 2024/06/04 01:47:20.620 FIG10b: Memory usage without flush after 23298 tasks: 1066.99609375 MB +worker1-1 | 2024/06/04 01:47:21.6212024/06/04 01:47:21.6212024/06/04 01:47:21.622 FIG10b: Memory usage without flush after 23415 tasks: 1066.796875 MB +worker1-1 | 2024/06/04 01:47:22.6232024/06/04 01:47:22.623 FIG10b: Memory usage without flush after 23537 tasks: 1066.8671875 MB +worker1-1 | 2024/06/04 01:47:23.625 FIG10b: Memory usage without flush after 23658 tasks: 1066.984375 MB +worker1-1 | 2024/06/04 01:47:24.636 [worker] stored...2024/06/04 01:47:24.637 FIG10b: Memory usage without flush after 23781 tasks: 1067.73828125 MB +worker1-1 | 2024/06/04 01:47:25.638 FIG10b: Memory usage without flush after 23921 tasks: 1068.578125 MB +worker1-1 | 2024/06/04 01:47:26.643 FIG10b: Memory usage without flush after 24054 tasks: 1068.1796875 MB +worker1-1 | 2024/06/04 01:47:27.648 FIG10b: Memory usage without flush after 24179 tasks: 1068.94921875 MB +worker1-1 | 2024/06/04 01:47:28.6492024/06/04 01:47:28.649 FIG10b: Memory usage without flush after 24315 tasks: 1068.9765625 MB +worker1-1 | 2024/06/04 01:47:29.6512024/06/04 01:47:29.651 2024/06/04 01:47:29.652 FIG10b: Memory usage without flush after 24451 tasks: 1069.81640625 MB +worker1-1 | in Run() rn2024/06/04 01:47:30.659 FIG10b: Memory usage without flush after 24580 tasks: 1070.40234375 MB +worker1-1 | in Run() rn2024/06/04 01:47:31.666 FIG10b: Memory usage without flush after 24715 tasks: 1070.03515625 MB +worker1-1 | 2024/06/04 01:47:32.671 FIG10b: Memory usage without flush after 24843 tasks: 1069.390625 MB +worker1-1 | 2024/06/04 01:47:33.6772024/06/04 01:47:33.677 FIG10b: Memory usage without flush after 24976 tasks: 1069.23828125 MB +worker1-1 | 2024/06/04 01:47:34.678 FIG10b: Memory usage without flush after 25108 tasks: 1069.671875 MB +worker1-1 | 2024/06/04 01:47:35.685 FIG10b: Memory usage without flush after 25243 tasks: 1070.22265625 MB +worker1-1 | 2024/06/04 01:47:36.686 [worker]2024/06/04 01:47:36.687 FIG10b: Memory usage without flush after 25374 tasks: 1069.4296875 MB +worker1-1 | 2024/06/04 01:47:37.694 FIG10b: Memory usage without flush after 25495 tasks: 1069.1328125 MB +worker1-1 | 2024/06/04 01:47:38.700 FIG10b: Memory usage without flush after 25621 tasks: 1069.8671875 MB +worker1-1 | 2024/06/04 01:47:39.702 FIG10b: Memory usage without flush after 25740 tasks: 1069.93359375 MB +worker1-1 | 2024/06/04 01:47:40.704 FIG10b: Memory usage without flush after 25870 tasks: 1069.94140625 MB +worker1-1 | [worker]2024/06/04 01:47:41.709 FIG10b: Memory usage without flush after 26001 tasks: 1070.42578125 MB +worker1-1 | 2024/06/04 01:47:42.711 FIG10b: Memory usage without flush after 26133 tasks: 1071.3828125 MB +worker1-1 | 2024/06/04 01:47:43.715 FIG10b: Memory usage without flush after 26263 tasks: 1071.140625 MB +worker1-1 | 2024/06/04 01:47:44.721 FIG10b: Memory usage without flush after 26395 tasks: 1071.86328125 MB +worker1-1 | gonna store now[worker] took this many seconds: 0.02082024/06/04 01:47:45.725 FIG10b: Memory usage without flush after 26525 tasks: 1071.625 MB +worker1-1 | 2024/06/04 01:47:46.7292024/06/04 01:47:46.729 FIG10b: Memory usage without flush after 26659 tasks: 1076.58203125 MB +worker1-1 | 2024/06/04 01:47:47.732 FIG10b: Memory usage without flush after 26779 tasks: 1077.11328125 MB +worker1-1 | 2024/06/04 01:47:48.7382024/06/04 01:47:48.738 FIG10b: Memory usage without flush after 26902 tasks: 1078.12890625 MB +worker1-1 | 2024/06/04 01:47:49.743 FIG10b: Memory usage without flush after 27028 tasks: 1078.8984375 MB +worker1-1 | FIG10b: Memory usage without flush after 27150 tasks: 1083.44140625 MB +worker1-1 | 2024/06/04 01:47:51.755 FIG10b: Memory usage without flush after 27277 tasks: 1083.46484375 MB +worker1-1 | 2024/06/04 01:47:52.759 FIG10b: Memory usage without flush after 27393 tasks: 1083.05078125 MB +worker1-1 | 2024/06/04 01:47:53.762 FIG10b: Memory usage without flush after 27509 tasks: 1084.11328125 MB +worker1-1 | 2024/06/04 01:47:54.767 FIG10b: Memory usage without flush after 27629 tasks: 1084.171875 MB +worker1-1 | 2024/06/04 01:47:55.7712024/06/04 01:47:55.772 FIG10b: Memory usage without flush after 27751 tasks: 1088.953125 MB +worker1-1 | 2024/06/04 01:47:56.776 2024/06/04 01:47:56.777stored... FIG10b: Memory usage without flush after 27870 tasks: 1088.765625 MB +worker1-1 | 2024/06/04 01:47:57.784 FIG10b: Memory usage without flush after 28011 tasks: 1088.45703125 MB +worker1-1 | 2024/06/04 01:47:58.790 FIG10b: Memory usage without flush after 28144 tasks: 1089.6484375 MB +worker1-1 | 2024/06/04 01:47:59.797 FIG10b: Memory usage without flush after 28214 tasks: 1090.2265625 MB +worker1-1 | 2024/06/04 01:48:00.801 FIG10b: Memory usage without flush after 28308 tasks: 1090.1796875 MB +worker1-1 | stored...[worker] 2024/06/04 01:48:01.805 FIG10b: Memory usage without flush after 28413 tasks: 1090.0859375 MB +worker1-1 | 2024/06/04 01:48:02.816 FIG10b: Memory usage without flush after 28541 tasks: 1087.9765625 MB +worker1-1 | 2024/06/04 01:48:03.824 FIG10b: Memory usage without flush after 28675 tasks: 1087.44921875 MB +worker1-1 | 2024/06/04 01:48:04.8282024/06/04 01:48:04.829 FIG10b: Memory usage without flush after 28805 tasks: 1092.8828125 MB +worker1-1 | 2024/06/04 01:48:05.835 FIG10b: Memory usage without flush after 28934 tasks: 1092.90625 MB +worker1-1 | 2024/06/04 01:48:06.837 FIG10b: Memory usage without flush after 29080 tasks: 1091.5703125 MB +worker1-1 | 2024/06/04 01:48:07.838Executed!2024/06/04 01:48:07.840 FIG10b: Memory usage without flush after 29217 tasks: 1090.71875 MB +worker1-1 | 2024/06/04 01:48:08.8422024/06/04 01:48:08.842 FIG10b: Memory usage without flush after 29350 tasks: 1091.6875 MB +worker1-1 | 2024/06/04 01:48:09.8502024/06/04 01:48:09.851 FIG10b: Memory usage without flush after 29475 tasks: 1092.8125 MB +worker1-1 | 2024/06/04 01:48:10.853 FIG10b: Memory usage without flush after 29601 tasks: 1094.234375 MB +worker1-1 | 2024/06/04 01:48:11.853 2024/06/04 01:48:11.854[worker] FIG10b: Memory usage without flush after 29736 tasks: 1093.03125 MB +worker1-1 | 2024/06/04 01:48:12.8622024/06/04 01:48:12.8622024/06/04 01:48:12.862 FIG10b: Memory usage without flush after 29871 tasks: 1093.48828125 MB +worker1-1 | 2024/06/04 01:48:13.8662024/06/04 01:48:13.866 FIG10b: Memory usage without flush after 30002 tasks: 1094.35546875 MB +worker1-1 | 2024/06/04 01:48:14.870 FIG10b: Memory usage without flush after 30126 tasks: 1094.109375 MB +worker1-1 | 2024/06/04 01:48:15.876 2024/06/04 01:48:15.877 FIG10b: Memory usage without flush after 30250 tasks: 1094.5703125 MB +worker1-1 | stored... 2024/06/04 01:48:16.889 FIG10b: Memory usage without flush after 30376 tasks: 1095.33203125 MB +worker1-1 | 2024/06/04 01:48:17.894 [worker] 2024/06/04 01:48:17.895 FIG10b: Memory usage without flush after 30496 tasks: 1096.16015625 MB +worker1-1 | 2024/06/04 01:48:18.895 FIG10b: Memory usage without flush after 30614 tasks: 1095.3984375 MB +worker1-1 | 2024/06/04 01:48:19.895 FIG10b: Memory usage without flush after 30742 tasks: 1095.86328125 MB +worker1-1 | 2024/06/04 01:48:20.8962024/06/04 01:48:20.897 FIG10b: Memory usage without flush after 30873 tasks: 1096.859375 MB +worker1-1 | 2024/06/04 01:48:21.905 FIG10b: Memory usage without flush after 31006 tasks: 1096.32421875 MB +worker1-1 | 2024/06/04 01:48:22.909 FIG10b: Memory usage without flush after 31131 tasks: 1096.24609375 MB +worker1-1 | 2024/06/04 01:48:23.915 FIG10b: Memory usage without flush after 31273 tasks: 1096.22265625 MB +worker1-1 | 2024/06/04 01:48:24.923 FIG10b: Memory usage without flush after 31403 tasks: 1096.70703125 MB +worker1-1 | 2024/06/04 01:48:25.930 FIG10b: Memory usage without flush after 31538 tasks: 1096.33203125 MB +worker1-1 | 2024/06/04 01:48:26.931 FIG10b: Memory usage without flush after 31666 tasks: 1097.38671875 MB +worker1-1 | 2024/06/04 01:48:27.934gonna store now FIG10b: Memory usage without flush after 31791 tasks: 1097.48046875 MB +worker1-1 | 2024/06/04 01:48:28.934 FIG10b: Memory usage without flush after 31910 tasks: 1097.703125 MB +worker1-1 | 2024/06/04 01:48:29.938 [worker]2024/06/04 01:48:29.939 FIG10b: Memory usage without flush after 32039 tasks: 1097.9375 MB +worker1-1 | [worker][worker][worker] gonna store now 2024/06/04 01:48:30.946 FIG10b: Memory usage without flush after 32169 tasks: 1099.4296875 MB +worker1-1 | 2024/06/04 01:48:31.9462024/06/04 01:48:31.946 FIG10b: Memory usage without flush after 32295 tasks: 1099.26171875 MB +worker1-1 | 2024/06/04 01:48:32.952 FIG10b: Memory usage without flush after 32408 tasks: 1098.62890625 MB +worker1-1 | [worker]2024/06/04 01:48:33.953 FIG10b: Memory usage without flush after 32533 tasks: 1098.9921875 MB +worker1-1 | 2024/06/04 01:48:34.958 FIG10b: Memory usage without flush after 32651 tasks: 1098.76953125 MB +worker1-1 | 2024/06/04 01:48:35.964 FIG10b: Memory usage without flush after 32771 tasks: 1098.85546875 MB +worker1-1 | 2024/06/04 01:48:36.969 FIG10b: Memory usage without flush after 32902 tasks: 1099.4921875 MB +worker1-1 | 2024/06/04 01:48:37.973 FIG10b: Memory usage without flush after 33038 tasks: 1099.6171875 MB +worker1-1 | in Run() rn2024/06/04 01:48:38.973 FIG10b: Memory usage without flush after 33173 tasks: 1098.90625 MB +worker1-1 | 2024/06/04 01:48:39.976 2024/06/04 01:48:39.977 FIG10b: Memory usage without flush after 33308 tasks: 1099.36328125 MB +worker1-1 | 2024/06/04 01:48:40.982 FIG10b: Memory usage without flush after 33438 tasks: 1098.63671875 MB +worker1-1 | 2024/06/04 01:48:41.989 2024/06/04 01:48:41.990 FIG10b: Memory usage without flush after 33572 tasks: 1099.109375 MB +worker1-1 | 2024/06/04 01:48:42.9912024/06/04 01:48:42.991 FIG10b: Memory usage without flush after 33719 tasks: 1099.50390625 MB +worker1-1 | [worker] in Run() rn2024/06/04 01:48:43.9952024/06/04 01:48:43.995 FIG10b: Memory usage without flush after 33844 tasks: 1099.5234375 MB +worker1-1 | stored...2024/06/04 01:48:44.996[worker] FIG10b: Memory usage without flush after 33978 tasks: 1100.45703125 MB +worker1-1 | 2024/06/04 01:48:45.997 FIG10b: Memory usage without flush after 34114 tasks: 1100.421875 MB +worker1-1 | 2024/06/04 01:48:47.0032024/06/04 01:48:47.003 [worker]FIG10b: Memory usage without flush after 34243 tasks: 1100.66796875 MB +worker1-1 | 2024/06/04 01:48:48.006 FIG10b: Memory usage without flush after 34376 tasks: 1101.375 MB +worker1-1 | 2024/06/04 01:48:49.007 2024/06/04 01:48:49.008 FIG10b: Memory usage without flush after 34499 tasks: 1102.64453125 MB +worker1-1 | 2024/06/04 01:48:50.0132024/06/04 01:48:50.013 FIG10b: Memory usage without flush after 34624 tasks: 1102.4296875 MB +worker1-1 | 2024/06/04 01:48:51.013 [worker]2024/06/04 01:48:51.014 FIG10b: Memory usage without flush after 34744 tasks: 1102.4921875 MB +worker1-1 | 2024/06/04 01:48:52.017 FIG10b: Memory usage without flush after 34885 tasks: 1102.9296875 MB +worker1-1 | [worker]2024/06/04 01:48:53.025 FIG10b: Memory usage without flush after 35017 tasks: 1103.89453125 MB +worker1-1 | 2024/06/04 01:48:54.028 FIG10b: Memory usage without flush after 35152 tasks: 1103.86328125 MB +worker1-1 | 2024/06/04 01:48:55.033 FIG10b: Memory usage without flush after 35289 tasks: 1103.33984375 MB +worker1-1 | FIG10b: Memory usage without flush after 35418 tasks: 1104.0859375 MB +worker1-1 | 2024/06/04 01:48:57.0402024/06/04 01:48:57.041 FIG10b: Memory usage without flush after 35550 tasks: 1104.0703125 MB +worker1-1 | 2024/06/04 01:48:58.043 FIG10b: Memory usage without flush after 35679 tasks: 1101.92578125 MB +worker1-1 | [worker]in Run() rn 2024/06/04 01:48:59.047 FIG10b: Memory usage without flush after 35799 tasks: 1102.97265625 MB +worker1-1 | 2024/06/04 01:49:00.046 [worker]2024/06/04 01:49:00.047 FIG10b: Memory usage without flush after 35918 tasks: 1103.26171875 MB +worker1-1 | 2024/06/04 01:49:01.050 FIG10b: Memory usage without flush after 36052 tasks: 1103.71484375 MB +worker1-1 | 2024/06/04 01:49:02.0572024/06/04 01:49:02.058 FIG10b: Memory usage without flush after 36183 tasks: 1104.44921875 MB +worker1-1 | 2024/06/04 01:49:03.060 FIG10b: Memory usage without flush after 36321 tasks: 1110.6015625 MB +worker1-1 | 2024/06/04 01:49:04.067 FIG10b: Memory usage without flush after 36444 tasks: 1110.89453125 MB +worker1-1 | 2024/06/04 01:49:05.0682024/06/04 01:49:05.068 FIG10b: Memory usage without flush after 36571 tasks: 1110.67578125 MB [worker] +worker1-1 | 2024/06/04 01:49:06.0732024/06/04 01:49:06.0732024/06/04 01:49:06.073 FIG10b: Memory usage without flush after 36695 tasks: 1110.94921875 MB +worker1-1 | 2024/06/04 01:49:07.081 2024/06/04 01:49:07.082 FIG10b: Memory usage without flush after 36815 tasks: 1110.99609375 MB +worker1-1 | 2024/06/04 01:49:08.085 2024/06/04 01:49:08.0862024/06/04 01:49:08.086 FIG10b: Memory usage without flush after 36940 tasks: 1109.2578125 MB +worker1-1 | Executed!2024/06/04 01:49:38.090 FIG10b: Memory usage without flush after 40802 tasks: 1114.84375 MB +worker1-1 | stored...2024/06/04 01:50:08.094 FIG10b: Memory usage without flush after 44689 tasks: 1122.60546875 MB +worker1-1 | 2024/06/04 01:50:38.096 FIG10b: Memory usage without flush after 48612 tasks: 1130.8515625 MB +worker1-1 | 2024/06/04 01:51:08.099 FIG10b: Memory usage without flush after 52542 tasks: 1138.8671875 MB +worker1-1 | 2024/06/04 01:51:38.103 FIG10b: Memory usage without flush after 56451 tasks: 1166.4609375 MB +worker1-1 | 2024/06/04 01:52:08.104 FIG10b: Memory usage without flush after 60396 tasks: 1178.38671875 MB +worker1-1 | 2024/06/04 01:52:38.107 FIG10b: Memory usage without flush after 64238 tasks: 1180.4375 MB +worker1-1 | 2024/06/04 01:53:08.107 FIG10b: Memory usage without flush after 68106 tasks: 1186.28125 MB +worker1-1 | 2024/06/04 01:53:38.1112024/06/04 01:53:38.111 FIG10b: Memory usage without flush after 72038 tasks: 1189.77734375 MB +worker1-1 | 2024/06/04 01:54:08.118 [worker]gonna store now2024/06/04 01:54:08.119 FIG10b: Memory usage without flush after 75940 tasks: 1223.87890625 MB +worker1-1 | 2024/06/04 01:54:38.123 FIG10b: Memory usage without flush after 79847 tasks: 1221.234375 MB +worker1-1 | [worker] 2024/06/04 01:55:08.133 FIG10b: Memory usage without flush after 83650 tasks: 1220.59765625 MB +worker1-1 | 2024/06/04 01:55:38.132gonna store now2024/06/04 01:55:38.133 FIG10b: Memory usage without flush after 87533 tasks: 1227.30078125 MB +worker1-1 | 2024/06/04 01:56:08.139 FIG10b: Memory usage without flush after 91295 tasks: 1239.5703125 MB +worker1-1 | 2024/06/04 01:56:38.149 FIG10b: Memory usage without flush after 95183 tasks: 1245.96875 MB +worker1-1 | 2024/06/04 01:57:08.156 FIG10b: Memory usage without flush after 99056 tasks: 1254.7734375 MB +worker1-1 | in Run() rntook this many seconds: 0.02512024/06/04 01:57:38.165 FIG10b: Memory usage without flush after 102702 tasks: 1260.3515625 MB +worker1-1 | 2024/06/04 01:58:08.1692024/06/04 01:58:08.169 FIG10b: Memory usage without flush after 106459 tasks: 1267.1171875 MB +worker1-1 | 2024/06/04 01:58:38.169 2024/06/04 01:58:38.169 FIG10b: Memory usage without flush after 110289 tasks: 1308.81640625 MB +worker1-1 | FIG10b: Memory usage without flush after 114090 tasks: 1344.52734375 MB +worker1-1 | 2024/06/04 01:59:38.1752024/06/04 01:59:38.175 FIG10b: Memory usage without flush after 118040 tasks: 1354.4609375 MB +worker1-1 | 2024/06/04 02:00:08.183 FIG10b: Memory usage without flush after 121986 tasks: 1348.3046875 MB +worker1-1 | 2024/06/04 02:00:38.186 FIG10b: Memory usage without flush after 125886 tasks: 1345.56640625 MB +worker1-1 | 2024/06/04 02:01:08.189 FIG10b: Memory usage without flush after 129729 tasks: 1353.29296875 MB +worker1-1 | [worker] [worker] 2024/06/04 02:01:38.196 FIG10b: Memory usage without flush after 133623 tasks: 1359.19921875 MB +worker1-1 | 2024/06/04 02:02:08.1992024/06/04 02:02:08.1992024/06/04 02:02:08.200 FIG10b: Memory usage without flush after 137570 tasks: 1360.1640625 MB +worker1-1 | in Run() rn2024/06/04 02:02:38.204 FIG10b: Memory usage without flush after 141485 tasks: 1369.62890625 MB +worker1-1 | 2024/06/04 02:03:08.207 2024/06/04 02:03:08.208 FIG10b: Memory usage without flush after 145444 tasks: 1378.6953125 MB +worker1-1 | 2024/06/04 02:03:38.214 FIG10b: Memory usage without flush after 149428 tasks: 1388.85546875 MB +worker1-1 | 2024/06/04 02:04:08.223 FIG10b: Memory usage without flush after 153394 tasks: 1432.5390625 MB +worker1-1 | 2024/06/04 02:04:38.225 [worker]2024/06/04 02:04:38.226 FIG10b: Memory usage without flush after 157310 tasks: 1425.765625 MB +worker1-1 | [worker]2024/06/04 02:05:08.228 FIG10b: Memory usage without flush after 161214 tasks: 1415.32421875 MB +worker1-1 | 2024/06/04 02:05:38.233 [worker][worker] 2024/06/04 02:05:38.233 took this many seconds: 0.0125 FIG10b: Memory usage without flush after 165055 tasks: 1425.00390625 MB +worker1-1 | 2024/06/04 02:06:08.241 FIG10b: Memory usage without flush after 168879 tasks: 1430.30859375 MB +worker1-1 | 2024/06/04 02:06:38.246 FIG10b: Memory usage without flush after 172712 tasks: 1432.46875 MB +worker1-1 | 2024/06/04 02:07:08.251 FIG10b: Memory usage without flush after 176626 tasks: 1441.54296875 MB +worker1-1 | [worker]2024/06/04 02:07:38.259 FIG10b: Memory usage without flush after 180567 tasks: 1454.42578125 MB +worker1-1 | 2024/06/04 02:08:08.260 FIG10b: Memory usage without flush after 184248 tasks: 1461.6875 MB +worker1-1 | 2024/06/04 02:08:38.2612024/06/04 02:08:38.261 FIG10b: Memory usage without flush after 188078 tasks: 1451.2265625 MB +worker1-1 | 2024/06/04 02:09:08.269 FIG10b: Memory usage without flush after 191836 tasks: 1540.4609375 MB +worker1-1 | [worker]2024/06/04 02:09:38.274 FIG10b: Memory usage without flush after 195637 tasks: 1551.8671875 MB +worker1-1 | [worker]2024/06/04 02:10:08.276 FIG10b: Memory usage without flush after 199524 tasks: 1538.37890625 MB +worker1-1 | 2024/06/04 02:10:38.280 FIG10b: Memory usage without flush after 203332 tasks: 1518.2578125 MB +worker1-1 | 2024/06/04 02:11:08.283 FIG10b: Memory usage without flush after 206386 tasks: 1508.953125 MB +worker1-1 | 2024/06/04 02:11:38.282 [worker]2024/06/04 02:11:38.283 FIG10b: Memory usage without flush after 210015 tasks: 1521.93359375 MB +worker1-1 | 2024/06/04 02:12:08.290 FIG10b: Memory usage without flush after 213634 tasks: 1563.53515625 MB +worker1-1 | 2024/06/04 02:12:38.302 FIG10b: Memory usage without flush after 217329 tasks: 1554.7265625 MB +worker1-1 | 2024/06/04 02:13:08.310 FIG10b: Memory usage without flush after 221049 tasks: 1594.203125 MB +worker1-1 | 2024/06/04 02:13:38.3232024/06/04 02:13:38.323 FIG10b: Memory usage without flush after 224462 tasks: 1600.4765625 MB +worker1-1 | stored...[worker]2024/06/04 02:14:08.3272024/06/04 02:14:08.327 FIG10b: Memory usage without flush after 227899 tasks: 1696.109375 MB +worker1-1 | 2024/06/04 02:14:38.328 FIG10b: Memory usage without flush after 231372 tasks: 1715.3515625 MB +worker1-1 | 2024/06/04 02:15:08.334 FIG10b: Memory usage without flush after 234591 tasks: 1722.95703125 MB +worker1-1 | 2024/06/04 02:15:38.336 FIG10b: Memory usage without flush after 237599 tasks: 1720.2890625 MB +worker1-1 | 2024/06/04 02:16:08.3472024/06/04 02:16:08.347 FIG10b: Memory usage without flush after 241002 tasks: 1727.76953125 MB[worker] +worker1-1 | gonna store now2024/06/04 02:16:38.352 FIG10b: Memory usage without flush after 244661 tasks: 1674.375 MB +worker1-1 | 2024/06/04 02:17:08.358 FIG10b: Memory usage without flush after 247944 tasks: 1663.44140625 MB +worker1-1 | 2024/06/04 02:17:38.359 FIG10b: Memory usage without flush after 251293 tasks: 1677.34765625 MB +worker1-1 | 2024/06/04 02:18:08.360 FIG10b: Memory usage without flush after 254903 tasks: 1678.1796875 MB +worker1-1 | 2024/06/04 02:18:38.365 2024/06/04 02:18:38.366 FIG10b: Memory usage without flush after 258599 tasks: 1679.25 MB +worker1-1 | 2024/06/04 02:19:08.373 FIG10b: Memory usage without flush after 262302 tasks: 1754.8046875 MB +worker1-1 | 2024/06/04 02:19:38.376 FIG10b: Memory usage without flush after 266227 tasks: 1681.375 MB +worker1-1 | 2024/06/04 02:20:08.378 FIG10b: Memory usage without flush after 270515 tasks: 1684.84375 MB +worker1-1 | 2024/06/04 02:20:38.381 FIG10b: Memory usage without flush after 275673 tasks: 1696.22265625 MB +worker1-1 | 2024/06/04 02:21:08.3872024/06/04 02:21:08.387 FIG10b: Memory usage without flush after 280531 tasks: 1706.1796875 MB +worker1-1 | 2024/06/04 02:21:38.390 [worker] 2024/06/04 02:21:38.390 FIG10b: Memory usage without flush after 285306 tasks: 1714.24609375 MB +worker1-1 | 2024/06/04 02:22:08.395 FIG10b: Memory usage without flush after 289952 tasks: 1719.63671875 MB +worker1-1 | 2024/06/04 02:22:38.395 FIG10b: Memory usage without flush after 294822 tasks: 1711.984375 MB +worker1-1 | 2024/06/04 02:23:08.396 FIG10b: Memory usage without flush after 299783 tasks: 1719.796875 MB +worker1-1 | 2024/06/04 02:23:38.397 FIG10b: Memory usage without flush after 304827 tasks: 1730.06640625 MB +worker1-1 | 2024/06/04 02:24:08.400 FIG10b: Memory usage without flush after 309570 tasks: 1766.78125 MB +worker1-1 | 2024/06/04 02:24:38.405 FIG10b: Memory usage without flush after 314560 tasks: 1775.64453125 MB +worker1-1 | 2024/06/04 02:25:08.4062024/06/04 02:25:08.405 FIG10b: Memory usage without flush after 319253 tasks: 1786.94140625 MB[worker] +worker1-1 | 2024/06/04 02:25:38.411 FIG10b: Memory usage without flush after 324231 tasks: 1788.42578125 MB +worker1-1 | 2024/06/04 02:26:08.413 FIG10b: Memory usage without flush after 328775 tasks: 1801.94140625 MB +worker1-1 | 2024/06/04 02:26:38.417 FIG10b: Memory usage without flush after 333511 tasks: 1809.4609375 MB +worker1-1 | 2024/06/04 02:27:08.419 FIG10b: Memory usage without flush after 338533 tasks: 1816.6640625 MB +worker1-1 | 2024/06/04 02:27:38.420[worker] in Run() rnFIG10b: Memory usage without flush after 343355 tasks: 1791.765625 MB +worker1-1 | 2024/06/04 02:28:08.422 FIG10b: Memory usage without flush after 348073 tasks: 1795.0859375 MB +worker1-1 | 2024/06/04 02:28:38.427 FIG10b: Memory usage without flush after 352909 tasks: 1802.6015625 MB +worker1-1 | [worker]2024/06/04 02:29:08.430 FIG10b: Memory usage without flush after 357462 tasks: 1849.19140625 MB +worker1-1 | 2024/06/04 02:29:38.432 FIG10b: Memory usage without flush after 362300 tasks: 1847.20703125 MB +worker1-1 | 2024/06/04 02:30:08.4332024/06/04 02:30:08.434 FIG10b: Memory usage without flush after 367257 tasks: 1854.25 MB +worker1-1 | 2024/06/04 02:30:38.438 FIG10b: Memory usage without flush after 371537 tasks: 1859.59375 MB +worker1-1 | 2024/06/04 02:31:08.444 2024/06/04 02:31:08.444 FIG10b: Memory usage without flush after 376059 tasks: 1861.40625 MB +worker1-1 | 2024/06/04 02:31:38.448 FIG10b: Memory usage without flush after 381190 tasks: 1868.171875 MB +worker1-1 | 2024/06/04 02:32:08.4522024/06/04 02:32:08.452 FIG10b: Memory usage without flush after 386298 tasks: 1826.0859375 MB +worker1-1 | [worker] 2024/06/04 02:32:38.453 FIG10b: Memory usage without flush after 391259 tasks: 1823.828125 MB +worker1-1 | 2024/06/04 02:33:08.458 FIG10b: Memory usage without flush after 396038 tasks: 1830.6640625 MB +worker1-1 | 2024/06/04 02:33:38.4602024/06/04 02:33:38.461 [worker]FIG10b: Memory usage without flush after 401067 tasks: 1840.07421875 MB +worker1-1 | 2024/06/04 02:34:08.465 FIG10b: Memory usage without flush after 406028 tasks: 1883.7109375 MB +worker1-1 | 2024/06/04 02:34:38.467 FIG10b: Memory usage without flush after 410968 tasks: 1893.63671875 MB +worker1-1 | 2024/06/04 02:35:08.474 FIG10b: Memory usage without flush after 415867 tasks: 1901.78515625 MB +worker1-1 | 2024/06/04 02:35:38.479 FIG10b: Memory usage without flush after 420855 tasks: 1912.9296875 MB +worker1-1 | 2024/06/04 02:36:08.481 FIG10b: Memory usage without flush after 425451 tasks: 1932.51171875 MB +worker1-1 | 2024/06/04 02:36:38.487 FIG10b: Memory usage without flush after 429499 tasks: 1939.3203125 MB +worker1-1 | 2024/06/04 02:37:08.491 FIG10b: Memory usage without flush after 434438 tasks: 1898.3125 MB +worker1-1 | 2024/06/04 02:37:38.494 FIG10b: Memory usage without flush after 439449 tasks: 1906.39453125 MB +worker1-1 | 2024/06/04 02:38:08.498 FIG10b: Memory usage without flush after 444619 tasks: 1913.25 MB +worker1-1 | 2024/06/04 02:38:38.498 FIG10b: Memory usage without flush after 449563 tasks: 1922.63671875 MB +worker1-1 | FIG10b: Memory usage without flush after 454596 tasks: 1972.8203125 MB +worker1-1 | 2024/06/04 02:39:38.503 FIG10b: Memory usage without flush after 459782 tasks: 1979.796875 MB +worker1-1 | 2024/06/04 02:40:08.505 FIG10b: Memory usage without flush after 464223 tasks: 1974.71484375 MB +worker1-1 | 2024/06/04 02:40:38.5102024/06/04 02:40:38.510 FIG10b: Memory usage without flush after 469209 tasks: 1984.91015625 MB +worker1-1 | 2024/06/04 02:41:08.514 FIG10b: Memory usage without flush after 473900 tasks: 1992.9921875 MB +worker1-1 | 2024/06/04 02:41:38.516 FIG10b: Memory usage without flush after 478442 tasks: 2002.17578125 MB diff --git a/analysis/output_10b_run1_cleaned.csv b/analysis/output_10b_run1_cleaned.csv new file mode 100644 index 0000000..eb7fb5c --- /dev/null +++ b/analysis/output_10b_run1_cleaned.csv @@ -0,0 +1,225 @@ +Timestamp,Memory Usage (MB) +2024-06-04 01:44:08.529000,970.90234375 +2024-06-04 01:44:09.530000,977.0 +2024-06-04 01:44:10.533000,986.859375 +2024-06-04 01:44:11.533000,987.625 +2024-06-04 01:44:12.538000,989.30078125 +2024-06-04 01:44:14.551000,991.5859375 +2024-06-04 01:44:15.558000,991.37109375 +2024-06-04 01:44:18.565000,999.69140625 +2024-06-04 01:44:19.571000,999.6328125 +2024-06-04 01:44:21.582000,1001.9453125 +2024-06-04 01:44:23.603000,1002.921875 +2024-06-04 01:44:26.620000,1003.6015625 +2024-06-04 01:44:27.620000,1005.21875 +2024-06-04 01:44:28.620000,1004.91015625 +2024-06-04 01:44:32.635000,1010.0390625 +2024-06-04 01:44:34.655000,1013.94140625 +2024-06-04 01:44:38.734000,1018.734375 +2024-06-04 01:44:39.971000,1018.75390625 +2024-06-04 01:44:40.893000,1018.98828125 +2024-06-04 01:44:41.889000,1019.6015625 +2024-06-04 01:44:42.891000,1023.62109375 +2024-06-04 01:44:43.896000,1023.48046875 +2024-06-04 01:44:44.901000,1023.890625 +2024-06-04 01:44:45.907000,1024.66015625 +2024-06-04 01:44:46.908000,1025.2265625 +2024-06-04 01:44:47.913000,1024.94140625 +2024-06-04 01:44:49.917000,1026.1484375 +2024-06-04 01:44:50.929000,1025.8984375 +2024-06-04 01:44:53.946000,1022.25 +2024-06-04 01:44:54.955000,1021.984375 +2024-06-04 01:44:55.959000,1021.9375 +2024-06-04 01:44:56.961000,1022.0546875 +2024-06-04 01:44:57.967000,1022.19140625 +2024-06-04 01:45:00.974000,1021.69921875 +2024-06-04 01:45:02.982000,1021.86328125 +2024-06-04 01:45:03.990000,1021.4609375 +2024-06-04 01:45:06.003000,1022.73828125 +2024-06-04 01:45:10.023000,1026.3515625 +2024-06-04 01:45:11.028000,1025.765625 +2024-06-04 01:45:12.038000,1025.53125 +2024-06-04 01:45:14.047000,1026.4296875 +2024-06-04 01:45:15.051000,1026.5234375 +2024-06-04 01:45:17.055000,1026.36328125 +2024-06-04 01:45:18.063000,1026.046875 +2024-06-04 01:45:22.081000,1026.33203125 +2024-06-04 01:45:23.084000,1026.7421875 +2024-06-04 01:45:27.102000,1028.72265625 +2024-06-04 01:45:30.134000,1028.61328125 +2024-06-04 01:45:31.138000,1029.0859375 +2024-06-04 01:45:32.145000,1029.4609375 +2024-06-04 01:45:33.153000,1029.68359375 +2024-06-04 01:45:34.154000,1030.1015625 +2024-06-04 01:45:35.163000,1030.99609375 +2024-06-04 01:45:36.171000,1029.703125 +2024-06-04 01:45:37.176000,1029.75 +2024-06-04 01:45:39.180000,1029.24609375 +2024-06-04 01:45:42.189000,1030.2890625 +2024-06-04 01:45:45.206000,1031.359375 +2024-06-04 01:45:46.207000,1031.94921875 +2024-06-04 01:45:47.208000,1032.20703125 +2024-06-04 01:45:48.214000,1033.19921875 +2024-06-04 01:45:51.221000,1032.45703125 +2024-06-04 01:45:53.224000,1033.6171875 +2024-06-04 01:45:54.225000,1033.1796875 +2024-06-04 01:45:57.233000,1034.3515625 +2024-06-04 01:45:59.245000,1035.3203125 +2024-06-04 01:46:00.247000,1035.94140625 +2024-06-04 01:46:01.255000,1037.515625 +2024-06-04 01:46:10.290000,1048.42578125 +2024-06-04 01:46:12.298000,1049.0859375 +2024-06-04 01:46:14.308000,1045.19140625 +2024-06-04 01:46:17.324000,1049.62890625 +2024-06-04 01:46:18.332000,1050.609375 +2024-06-04 01:46:21.344000,1051.9765625 +2024-06-04 01:46:23.352000,1053.4765625 +2024-06-04 01:46:28.372000,1054.71484375 +2024-06-04 01:46:29.373000,1053.77734375 +2024-06-04 01:46:32.387000,1055.52734375 +2024-06-04 01:46:33.396000,1054.6484375 +2024-06-04 01:46:34.397000,1054.453125 +2024-06-04 01:46:37.404000,1056.26171875 +2024-06-04 01:46:38.409000,1056.53125 +2024-06-04 01:46:39.409000,1056.0234375 +2024-06-04 01:46:41.423000,1056.2890625 +2024-06-04 01:46:42.432000,1054.0234375 +2024-06-04 01:46:44.463000,1054.8125 +2024-06-04 01:46:46.468000,1057.4609375 +2024-06-04 01:46:47.471000,1057.5390625 +2024-06-04 01:46:50.486000,1058.125 +2024-06-04 01:46:52.494000,1057.8671875 +2024-06-04 01:46:54.505000,1058.33984375 +2024-06-04 01:46:55.507000,1058.59375 +2024-06-04 01:46:57.516000,1059.72265625 +2024-06-04 01:46:58.538000,1059.53515625 +2024-06-04 01:46:59.542000,1059.1328125 +2024-06-04 01:47:00.544000,1059.484375 +2024-06-04 01:47:03.555000,1059.3359375 +2024-06-04 01:47:05.560000,1059.09375 +2024-06-04 01:47:06.562000,1059.08984375 +2024-06-04 01:47:09.579000,1062.77734375 +2024-06-04 01:47:10.580000,1062.42578125 +2024-06-04 01:47:11.581000,1062.2421875 +2024-06-04 01:47:16.607000,1064.97265625 +2024-06-04 01:47:19.613000,1066.9296875 +2024-06-04 01:47:20.620000,1066.99609375 +2024-06-04 01:47:23.625000,1066.984375 +2024-06-04 01:47:25.638000,1068.578125 +2024-06-04 01:47:26.643000,1068.1796875 +2024-06-04 01:47:27.648000,1068.94921875 +2024-06-04 01:47:32.671000,1069.390625 +2024-06-04 01:47:34.678000,1069.671875 +2024-06-04 01:47:35.685000,1070.22265625 +2024-06-04 01:47:37.694000,1069.1328125 +2024-06-04 01:47:38.700000,1069.8671875 +2024-06-04 01:47:39.702000,1069.93359375 +2024-06-04 01:47:40.704000,1069.94140625 +2024-06-04 01:47:42.711000,1071.3828125 +2024-06-04 01:47:43.715000,1071.140625 +2024-06-04 01:47:44.721000,1071.86328125 +2024-06-04 01:47:47.732000,1077.11328125 +2024-06-04 01:47:49.743000,1078.8984375 +2024-06-04 01:47:51.755000,1083.46484375 +2024-06-04 01:47:52.759000,1083.05078125 +2024-06-04 01:47:53.762000,1084.11328125 +2024-06-04 01:47:54.767000,1084.171875 +2024-06-04 01:47:57.784000,1088.45703125 +2024-06-04 01:47:58.790000,1089.6484375 +2024-06-04 01:47:59.797000,1090.2265625 +2024-06-04 01:48:00.801000,1090.1796875 +2024-06-04 01:48:02.816000,1087.9765625 +2024-06-04 01:48:03.824000,1087.44921875 +2024-06-04 01:48:05.835000,1092.90625 +2024-06-04 01:48:06.837000,1091.5703125 +2024-06-04 01:48:10.853000,1094.234375 +2024-06-04 01:48:14.870000,1094.109375 +2024-06-04 01:48:18.895000,1095.3984375 +2024-06-04 01:48:19.895000,1095.86328125 +2024-06-04 01:48:21.905000,1096.32421875 +2024-06-04 01:48:22.909000,1096.24609375 +2024-06-04 01:48:23.915000,1096.22265625 +2024-06-04 01:48:24.923000,1096.70703125 +2024-06-04 01:48:25.930000,1096.33203125 +2024-06-04 01:48:26.931000,1097.38671875 +2024-06-04 01:48:28.934000,1097.703125 +2024-06-04 01:48:32.952000,1098.62890625 +2024-06-04 01:48:34.958000,1098.76953125 +2024-06-04 01:48:35.964000,1098.85546875 +2024-06-04 01:48:36.969000,1099.4921875 +2024-06-04 01:48:37.973000,1099.6171875 +2024-06-04 01:48:40.982000,1098.63671875 +2024-06-04 01:48:45.997000,1100.421875 +2024-06-04 01:48:48.006000,1101.375 +2024-06-04 01:48:52.017000,1102.9296875 +2024-06-04 01:48:54.028000,1103.86328125 +2024-06-04 01:48:55.033000,1103.33984375 +2024-06-04 01:48:58.043000,1101.92578125 +2024-06-04 01:49:01.050000,1103.71484375 +2024-06-04 01:49:03.060000,1110.6015625 +2024-06-04 01:49:04.067000,1110.89453125 +2024-06-04 01:50:38.096000,1130.8515625 +2024-06-04 01:51:08.099000,1138.8671875 +2024-06-04 01:51:38.103000,1166.4609375 +2024-06-04 01:52:08.104000,1178.38671875 +2024-06-04 01:52:38.107000,1180.4375 +2024-06-04 01:53:08.107000,1186.28125 +2024-06-04 01:54:38.123000,1221.234375 +2024-06-04 01:56:08.139000,1239.5703125 +2024-06-04 01:56:38.149000,1245.96875 +2024-06-04 01:57:08.156000,1254.7734375 +2024-06-04 02:00:08.183000,1348.3046875 +2024-06-04 02:00:38.186000,1345.56640625 +2024-06-04 02:01:08.189000,1353.29296875 +2024-06-04 02:03:38.214000,1388.85546875 +2024-06-04 02:04:08.223000,1432.5390625 +2024-06-04 02:06:08.241000,1430.30859375 +2024-06-04 02:06:38.246000,1432.46875 +2024-06-04 02:07:08.251000,1441.54296875 +2024-06-04 02:08:08.260000,1461.6875 +2024-06-04 02:09:08.269000,1540.4609375 +2024-06-04 02:10:38.280000,1518.2578125 +2024-06-04 02:11:08.283000,1508.953125 +2024-06-04 02:12:08.290000,1563.53515625 +2024-06-04 02:12:38.302000,1554.7265625 +2024-06-04 02:13:08.310000,1594.203125 +2024-06-04 02:14:38.328000,1715.3515625 +2024-06-04 02:15:08.334000,1722.95703125 +2024-06-04 02:15:38.336000,1720.2890625 +2024-06-04 02:17:08.358000,1663.44140625 +2024-06-04 02:17:38.359000,1677.34765625 +2024-06-04 02:18:08.360000,1678.1796875 +2024-06-04 02:19:08.373000,1754.8046875 +2024-06-04 02:19:38.376000,1681.375 +2024-06-04 02:20:08.378000,1684.84375 +2024-06-04 02:20:38.381000,1696.22265625 +2024-06-04 02:22:08.395000,1719.63671875 +2024-06-04 02:22:38.395000,1711.984375 +2024-06-04 02:23:08.396000,1719.796875 +2024-06-04 02:23:38.397000,1730.06640625 +2024-06-04 02:24:08.400000,1766.78125 +2024-06-04 02:24:38.405000,1775.64453125 +2024-06-04 02:25:38.411000,1788.42578125 +2024-06-04 02:26:08.413000,1801.94140625 +2024-06-04 02:26:38.417000,1809.4609375 +2024-06-04 02:27:08.419000,1816.6640625 +2024-06-04 02:28:08.422000,1795.0859375 +2024-06-04 02:28:38.427000,1802.6015625 +2024-06-04 02:29:38.432000,1847.20703125 +2024-06-04 02:30:38.438000,1859.59375 +2024-06-04 02:31:38.448000,1868.171875 +2024-06-04 02:33:08.458000,1830.6640625 +2024-06-04 02:34:08.465000,1883.7109375 +2024-06-04 02:34:38.467000,1893.63671875 +2024-06-04 02:35:08.474000,1901.78515625 +2024-06-04 02:35:38.479000,1912.9296875 +2024-06-04 02:36:08.481000,1932.51171875 +2024-06-04 02:36:38.487000,1939.3203125 +2024-06-04 02:37:08.491000,1898.3125 +2024-06-04 02:37:38.494000,1906.39453125 +2024-06-04 02:38:08.498000,1913.25 +2024-06-04 02:38:38.498000,1922.63671875 +2024-06-04 02:39:38.503000,1979.796875 +2024-06-04 02:40:08.505000,1974.71484375 +2024-06-04 02:41:08.514000,1992.9921875 +2024-06-04 02:41:38.516000,2002.17578125 diff --git a/analysis/output_10b_run2_300s_flush_2b_memory_limit.txt b/analysis/output_10b_run2_300s_flush_2b_memory_limit.txt new file mode 100644 index 0000000..eec01ff --- /dev/null +++ b/analysis/output_10b_run2_300s_flush_2b_memory_limit.txt @@ -0,0 +1,440 @@ +worker1-1 | 2024/06/04 03:33:54.719 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 03:33:54.734 FIG10b: Starting tasks submission... +worker1-1 | 2024/06/04 03:33:55.734 FIG10b: Memory usage without flush after 122 tasks: 1012.97265625 MB +worker1-1 | 2024/06/04 03:33:56.736 FIG10b: Memory usage without flush after 243 tasks: 1013.26171875 MB +worker1-1 | [worker] stored...[worker]2024/06/04 03:33:57.744 FIG10b: Memory usage without flush after 367 tasks: 1014.125 MB +worker1-1 | 2024/06/04 03:33:58.747 FIG10b: Memory usage without flush after 473 tasks: 1017.6484375 MB +worker1-1 | 2024/06/04 03:33:59.759 FIG10b: Memory usage without flush after 579 tasks: 1021.41796875 MB +worker1-1 | 2024/06/04 03:34:00.767 2024/06/04 03:34:00.767 FIG10b: Memory usage without flush after 691 tasks: 1022.6875 MB +worker1-1 | Executed!2024/06/04 03:34:01.777 FIG10b: Memory usage without flush after 807 tasks: 1022.875 MB +worker1-1 | 2024/06/04 03:34:02.777 FIG10b: Memory usage without flush after 924 tasks: 1022.78125 MB +worker1-1 | 2024/06/04 03:34:03.785 FIG10b: Memory usage without flush after 1037 tasks: 1026.54296875 MB +worker1-1 | 2024/06/04 03:34:04.793 FIG10b: Memory usage without flush after 1137 tasks: 1025.8046875 MB +worker1-1 | [worker]2024/06/04 03:34:05.794 FIG10b: Memory usage without flush after 1237 tasks: 1025.7734375 MB +worker1-1 | 2024/06/04 03:34:06.800 FIG10b: Memory usage without flush after 1338 tasks: 1026.49609375 MB +worker1-1 | 2024/06/04 03:34:07.803[worker] FIG10b: Memory usage without flush after 1441 tasks: 1025.46484375 MB +worker1-1 | [worker] [worker]2024/06/04 03:34:08.809 FIG10b: Memory usage without flush after 1548 tasks: 1025.01171875 MB +worker1-1 | 2024/06/04 03:34:09.812 FIG10b: Memory usage without flush after 1645 tasks: 1025.4765625 MB +worker1-1 | 2024/06/04 03:34:10.816 FIG10b: Memory usage without flush after 1747 tasks: 1025.19140625 MB +worker1-1 | 2024/06/04 03:34:11.822 2024/06/04 03:34:11.823 FIG10b: Memory usage without flush after 1845 tasks: 1026.1015625 MB +worker1-1 | 2024/06/04 03:34:12.826 FIG10b: Memory usage without flush after 1948 tasks: 1025.671875 MB +worker1-1 | 2024/06/04 03:34:13.827 FIG10b: Memory usage without flush after 2043 tasks: 1025.4296875 MB +worker1-1 | 2024/06/04 03:34:14.846 FIG10b: Memory usage without flush after 2140 tasks: 1025.375 MB +worker1-1 | 2024/06/04 03:34:15.847 2024/06/04 03:34:15.847 FIG10b: Memory usage without flush after 2243 tasks: 1025.87890625 MB +worker1-1 | 2024/06/04 03:34:16.855 FIG10b: Memory usage without flush after 2333 tasks: 1026.3203125 MB +worker1-1 | 2024/06/04 03:34:17.865 2024/06/04 03:34:17.866 FIG10b: Memory usage without flush after 2420 tasks: 1025.1640625 MB +worker1-1 | 2024/06/04 03:34:18.872 FIG10b: Memory usage without flush after 2515 tasks: 1030.8359375 MB +worker1-1 | 2024/06/04 03:34:19.881 FIG10b: Memory usage without flush after 2598 tasks: 1035.09765625 MB +worker1-1 | 2024/06/04 03:34:20.8932024/06/04 03:34:20.893 FIG10b: Memory usage without flush after 2693 tasks: 1034.47265625 MB +worker1-1 | 2024/06/04 03:34:21.899 FIG10b: Memory usage without flush after 2791 tasks: 1035.06640625 MB +worker1-1 | 2024/06/04 03:34:22.904 FIG10b: Memory usage without flush after 2911 tasks: 1034.64453125 MB +worker1-1 | 2024/06/04 03:34:23.911 [worker] 2024/06/04 03:34:23.913 FIG10b: Memory usage without flush after 3016 tasks: 1035.40625 MB +worker1-1 | 2024/06/04 03:34:24.915 FIG10b: Memory usage without flush after 3133 tasks: 1034.56640625 MB +worker1-1 | FIG10b: Memory usage without flush after 3237 tasks: 1035.44140625 MB +worker1-1 | 2024/06/04 03:34:26.925 FIG10b: Memory usage without flush after 3335 tasks: 1037.55859375 MB +worker1-1 | 2024/06/04 03:34:27.934 FIG10b: Memory usage without flush after 3453 tasks: 1038.2890625 MB +worker1-1 | 2024/06/04 03:34:28.955 FIG10b: Memory usage without flush after 3562 tasks: 1039.37109375 MB +worker1-1 | FIG10b: Memory usage without flush after 3653 tasks: 1039.29296875 MB +worker1-1 | 2024/06/04 03:34:30.965 FIG10b: Memory usage without flush after 3764 tasks: 1039.109375 MB +worker1-1 | [worker]2024/06/04 03:34:31.978 [worker]2024/06/04 03:34:31.979 FIG10b: Memory usage without flush after 3879 tasks: 1039.41796875 MB +worker1-1 | 2024/06/04 03:34:32.9932024/06/04 03:34:32.993 FIG10b: Memory usage without flush after 3978 tasks: 1039.7734375 MB +worker1-1 | 2024/06/04 03:34:33.9982024/06/04 03:34:33.998 FIG10b: Memory usage without flush after 4071 tasks: 1039.69921875 MB +worker1-1 | 2024/06/04 03:34:35.0012024/06/04 03:34:35.001 FIG10b: Memory usage without flush after 4171 tasks: 1039.90234375 MB +worker1-1 | 2024/06/04 03:34:36.0022024/06/04 03:34:36.003 FIG10b: Memory usage without flush after 4294 tasks: 1039.9453125 MB +worker1-1 | 2024/06/04 03:34:37.010 FIG10b: Memory usage without flush after 4403 tasks: 1046.2890625 MB +worker1-1 | 2024/06/04 03:34:38.0342024/06/04 03:34:38.035 FIG10b: Memory usage without flush after 4505 tasks: 1047.60546875 MB +worker1-1 | took this many seconds: 0.03362024/06/04 03:34:39.036 FIG10b: Memory usage without flush after 4620 tasks: 1047.2734375 MB +worker1-1 | 2024/06/04 03:34:40.0362024/06/04 03:34:40.0362024/06/04 03:34:40.036 FIG10b: Memory usage without flush after 4737 tasks: 1047.328125 MB +worker1-1 | 2024/06/04 03:34:41.042 2024/06/04 03:34:41.042 FIG10b: Memory usage without flush after 4845 tasks: 1048.1328125 MB +worker1-1 | 2024/06/04 03:34:42.052 FIG10b: Memory usage without flush after 4965 tasks: 1049.14453125 MB +worker1-1 | 2024/06/04 03:34:43.052 FIG10b: Memory usage without flush after 5063 tasks: 1050.30078125 MB +worker1-1 | 2024/06/04 03:34:44.061 FIG10b: Memory usage without flush after 5153 tasks: 1050.24609375 MB +worker1-1 | [worker] in Run() rn2024/06/04 03:34:45.063 FIG10b: Memory usage without flush after 5249 tasks: 1052.046875 MB +worker1-1 | 2024/06/04 03:34:46.073 FIG10b: Memory usage without flush after 5330 tasks: 1051.2734375 MB +worker1-1 | 2024/06/04 03:34:47.079 FIG10b: Memory usage without flush after 5443 tasks: 1051.11328125 MB +worker1-1 | 2024/06/04 03:34:48.081 FIG10b: Memory usage without flush after 5553 tasks: 1057.07421875 MB +worker1-1 | 2024/06/04 03:34:49.0892024/06/04 03:34:49.090 FIG10b: Memory usage without flush after 5681 tasks: 1056.59375 MB +worker1-1 | 2024/06/04 03:34:50.0942024/06/04 03:34:50.094 FIG10b: Memory usage without flush after 5804 tasks: 1063.7734375 MB +worker1-1 | 2024/06/04 03:34:51.100 FIG10b: Memory usage without flush after 5918 tasks: 1063.61328125 MB +worker1-1 | [worker]2024/06/04 03:34:52.103 FIG10b: Memory usage without flush after 6036 tasks: 1064.015625 MB +worker1-1 | 2024/06/04 03:34:53.106 FIG10b: Memory usage without flush after 6152 tasks: 1065.19140625 MB +worker1-1 | 2024/06/04 03:34:54.110 FIG10b: Memory usage without flush after 6267 tasks: 1064.7109375 MB +worker1-1 | gonna store now2024/06/04 03:34:55.113 FIG10b: Memory usage without flush after 6383 tasks: 1065.71484375 MB +worker1-1 | 2024/06/04 03:34:56.118 FIG10b: Memory usage without flush after 6510 tasks: 1066.42578125 MB +worker1-1 | [worker] 2024/06/04 03:34:57.1232024/06/04 03:34:57.123 FIG10b: Memory usage without flush after 6629 tasks: 1067.671875 MB +worker1-1 | 2024/06/04 03:34:58.134 FIG10b: Memory usage without flush after 6748 tasks: 1068.85546875 MB +worker1-1 | 2024/06/04 03:34:59.137 FIG10b: Memory usage without flush after 6873 tasks: 1068.48046875 MB +worker1-1 | 2024/06/04 03:35:00.142 FIG10b: Memory usage without flush after 7012 tasks: 1069.43359375 MB +worker1-1 | Executed!2024/06/04 03:35:01.157 FIG10b: Memory usage without flush after 7149 tasks: 1069.0859375 MB +worker1-1 | 2024/06/04 03:35:02.158 FIG10b: Memory usage without flush after 7271 tasks: 1070.07421875 MB +worker1-1 | 2024/06/04 03:35:03.166 FIG10b: Memory usage without flush after 7382 tasks: 1069.4140625 MB +worker1-1 | 2024/06/04 03:35:04.167 FIG10b: Memory usage without flush after 7503 tasks: 1068.91015625 MB +worker1-1 | 2024/06/04 03:35:05.171 FIG10b: Memory usage without flush after 7622 tasks: 1070.0625 MB +worker1-1 | [worker]2024/06/04 03:35:06.174 FIG10b: Memory usage without flush after 7739 tasks: 1070.83984375 MB +worker1-1 | 2024/06/04 03:35:07.180 FIG10b: Memory usage without flush after 7846 tasks: 1070.69921875 MB +worker1-1 | 2024/06/04 03:35:08.184 FIG10b: Memory usage without flush after 7966 tasks: 1071.61328125 MB +worker1-1 | [worker] stored... Executed!2024/06/04 03:35:09.188 FIG10b: Memory usage without flush after 8078 tasks: 1072.328125 MB +worker1-1 | 2024/06/04 03:35:10.192 2024/06/04 03:35:10.1922024/06/04 03:35:10.1932024/06/04 03:35:10.193 FIG10b: Memory usage without flush after 8204 tasks: 1072.8359375 MB +worker1-1 | 2024/06/04 03:35:11.193 FIG10b: Memory usage without flush after 8333 tasks: 1071.91796875 MB +worker1-1 | 2024/06/04 03:35:12.193 2024/06/04 03:35:12.193 FIG10b: Memory usage without flush after 8443 tasks: 1072.87890625 MB +worker1-1 | 2024/06/04 03:35:13.203 FIG10b: Memory usage without flush after 8550 tasks: 1073.85546875 MB +worker1-1 | 2024/06/04 03:35:14.205 2024/06/04 03:35:14.205 FIG10b: Memory usage without flush after 8673 tasks: 1073.84375 MB +worker1-1 | 2024/06/04 03:35:15.211 FIG10b: Memory usage without flush after 8794 tasks: 1075.859375 MB +worker1-1 | 2024/06/04 03:35:16.214 [worker]2024/06/04 03:35:16.215 FIG10b: Memory usage without flush after 8921 tasks: 1075.84375 MB +worker1-1 | 2024/06/04 03:35:17.217 FIG10b: Memory usage without flush after 9042 tasks: 1076.56640625 MB +worker1-1 | 2024/06/04 03:35:18.2262024/06/04 03:35:18.227 FIG10b: Memory usage without flush after 9157 tasks: 1076.85546875 MB +worker1-1 | FIG10b: Memory usage without flush after 9258 tasks: 1077.8671875 MB +worker1-1 | 2024/06/04 03:35:20.239 FIG10b: Memory usage without flush after 9390 tasks: 1069.203125 MB +worker1-1 | 2024/06/04 03:35:21.248 [worker]2024/06/04 03:35:21.249 FIG10b: Memory usage without flush after 9516 tasks: 1069.6484375 MB +worker1-1 | 2024/06/04 03:35:22.254 FIG10b: Memory usage without flush after 9632 tasks: 1069.48828125 MB +worker1-1 | 2024/06/04 03:35:23.254 FIG10b: Memory usage without flush after 9746 tasks: 1069.06640625 MB +worker1-1 | 2024/06/04 03:35:24.260 FIG10b: Memory usage without flush after 9862 tasks: 1069.4765625 MB +worker1-1 | Executed!2024/06/04 03:35:25.265 FIG10b: Memory usage without flush after 9970 tasks: 1069.65625 MB +worker1-1 | 2024/06/04 03:35:26.2652024/06/04 03:35:26.265 FIG10b: Memory usage without flush after 10084 tasks: 1069.08203125 MB +worker1-1 | 2024/06/04 03:35:27.272 FIG10b: Memory usage without flush after 10192 tasks: 1069.46875 MB +worker1-1 | 2024/06/04 03:35:28.2752024/06/04 03:35:28.276[worker] FIG10b: Memory usage without flush after 10301 tasks: 1069.15234375 MB +worker1-1 | 2024/06/04 03:35:29.2762024/06/04 03:35:29.276 FIG10b: Memory usage without flush after 10408 tasks: 1069.77734375 MB +worker1-1 | 2024/06/04 03:35:30.280 FIG10b: Memory usage without flush after 10524 tasks: 1070.6875 MB +worker1-1 | 2024/06/04 03:35:31.285 in Run() rn2024/06/04 03:35:31.285 FIG10b: Memory usage without flush after 10629 tasks: 1070.36328125 MB +worker1-1 | 2024/06/04 03:35:32.288 FIG10b: Memory usage without flush after 10750 tasks: 1070.51171875 MB +worker1-1 | 2024/06/04 03:35:33.2882024/06/04 03:35:33.289 FIG10b: Memory usage without flush after 10874 tasks: 1070.49609375 MB +worker1-1 | Executed!2024/06/04 03:35:34.295 FIG10b: Memory usage without flush after 11015 tasks: 1069.796875 MB +worker1-1 | 2024/06/04 03:35:35.2952024/06/04 03:35:35.2952024/06/04 03:35:35.296 FIG10b: Memory usage without flush after 11148 tasks: 1070.359375 MB +worker1-1 | 2024/06/04 03:35:36.298 FIG10b: Memory usage without flush after 11279 tasks: 1070.12109375 MB +worker1-1 | 2024/06/04 03:35:37.3002024/06/04 03:35:37.300 FIG10b: Memory usage without flush after 11408 tasks: 1070.00390625 MB +worker1-1 | 2024/06/04 03:35:38.301 FIG10b: Memory usage without flush after 11548 tasks: 1070.30078125 MB +worker1-1 | 2024/06/04 03:35:39.304 FIG10b: Memory usage without flush after 11683 tasks: 1070.375 MB +worker1-1 | [worker] Executed!2024/06/04 03:35:40.309 FIG10b: Memory usage without flush after 11822 tasks: 1070.44140625 MB +worker1-1 | 2024/06/04 03:35:41.312 2024/06/04 03:35:41.313 FIG10b: Memory usage without flush after 11953 tasks: 1075.01171875 MB +worker1-1 | 2024/06/04 03:35:42.316 FIG10b: Memory usage without flush after 12080 tasks: 1075.08984375 MB +worker1-1 | 2024/06/04 03:35:43.3172024/06/04 03:35:43.317 FIG10b: Memory usage without flush after 12211 tasks: 1075.15234375 MB +worker1-1 | 2024/06/04 03:35:44.319 FIG10b: Memory usage without flush after 12351 tasks: 1076.2265625 MB +worker1-1 | 2024/06/04 03:35:45.319 FIG10b: Memory usage without flush after 12490 tasks: 1076.9453125 MB +worker1-1 | 2024/06/04 03:35:46.327 FIG10b: Memory usage without flush after 12612 tasks: 1077.54296875 MB +worker1-1 | gonna store now2024/06/04 03:35:47.345 FIG10b: Memory usage without flush after 12717 tasks: 1078.91796875 MB +worker1-1 | Executed!2024/06/04 03:35:48.3502024/06/04 03:35:48.350 FIG10b: Memory usage without flush after 12829 tasks: 1078.58203125 MB +worker1-1 | 2024/06/04 03:35:49.367 FIG10b: Memory usage without flush after 12960 tasks: 1095.51171875 MB +worker1-1 | 2024/06/04 03:35:50.371 FIG10b: Memory usage without flush after 13074 tasks: 1096.14453125 MB +worker1-1 | 2024/06/04 03:35:51.373 2024/06/04 03:35:51.374 FIG10b: Memory usage without flush after 13198 tasks: 1095.47265625 MB[worker] +worker1-1 | [worker]2024/06/04 03:35:52.375 FIG10b: Memory usage without flush after 13318 tasks: 1091.81640625 MB +worker1-1 | 2024/06/04 03:35:53.378 FIG10b: Memory usage without flush after 13452 tasks: 1091.921875 MB +worker1-1 | 2024/06/04 03:35:54.382 FIG10b: Memory usage without flush after 13579 tasks: 1093.2421875 MB +worker1-1 | 2024/06/04 03:35:55.387 FIG10b: Memory usage without flush after 13703 tasks: 1094.23828125 MB +worker1-1 | 2024/06/04 03:35:56.389 FIG10b: Memory usage without flush after 13831 tasks: 1094.46484375 MB +worker1-1 | 2024/06/04 03:35:57.396 FIG10b: Memory usage without flush after 13941 tasks: 1095.83984375 MB +worker1-1 | 2024/06/04 03:35:58.398 FIG10b: Memory usage without flush after 14054 tasks: 1096.02734375 MB +worker1-1 | 2024/06/04 03:35:59.406 FIG10b: Memory usage without flush after 14161 tasks: 1095.27734375 MB +worker1-1 | FIG10b: Memory usage without flush after 14266 tasks: 1095.265625 MB +worker1-1 | 2024/06/04 03:36:01.410 FIG10b: Memory usage without flush after 14382 tasks: 1095.453125 MB +worker1-1 | 2024/06/04 03:36:02.417 FIG10b: Memory usage without flush after 14487 tasks: 1096.1484375 MB +worker1-1 | 2024/06/04 03:36:03.4222024/06/04 03:36:03.423 FIG10b: Memory usage without flush after 14604 tasks: 1096.5625 MB +worker1-1 | 2024/06/04 03:36:04.434 FIG10b: Memory usage without flush after 14700 tasks: 1096.8203125 MB[worker] took this many seconds: 0.0298 +worker1-1 | 2024/06/04 03:36:05.444 FIG10b: Memory usage without flush after 14785 tasks: 1097.08203125 MB +worker1-1 | 2024/06/04 03:36:06.502 FIG10b: Memory usage without flush after 14899 tasks: 1097.53125 MB +worker1-1 | 2024/06/04 03:36:07.511 FIG10b: Memory usage without flush after 15011 tasks: 1097.96875 MB +worker1-1 | 2024/06/04 03:36:08.515 FIG10b: Memory usage without flush after 15101 tasks: 1097.9609375 MB +worker1-1 | 2024/06/04 03:36:09.5262024/06/04 03:36:09.526 FIG10b: Memory usage without flush after 15202 tasks: 1098.64453125 MB +worker1-1 | 2024/06/04 03:36:10.529 FIG10b: Memory usage without flush after 15305 tasks: 1098.609375 MB +worker1-1 | 2024/06/04 03:36:11.532 FIG10b: Memory usage without flush after 15392 tasks: 1098.86328125 MB +worker1-1 | 2024/06/04 03:36:12.546 FIG10b: Memory usage without flush after 15464 tasks: 1098.69140625 MB +worker1-1 | 2024/06/04 03:36:13.551 FIG10b: Memory usage without flush after 15554 tasks: 1099.46875 MB +worker1-1 | 2024/06/04 03:36:14.557 FIG10b: Memory usage without flush after 15644 tasks: 1099.46484375 MB +worker1-1 | 2024/06/04 03:36:15.5642024/06/04 03:36:15.565 FIG10b: Memory usage without flush after 15748 tasks: 1099.40234375 MB +worker1-1 | [worker] stored...2024/06/04 03:36:16.571 FIG10b: Memory usage without flush after 15848 tasks: 1097.1796875 MB +worker1-1 | [worker] stored...2024/06/04 03:36:17.5822024/06/04 03:36:17.582 FIG10b: Memory usage without flush after 15939 tasks: 1098.3671875 MB +worker1-1 | 2024/06/04 03:36:18.5842024/06/04 03:36:18.584 FIG10b: Memory usage without flush after 16026 tasks: 1097.3671875 MB +worker1-1 | 2024/06/04 03:36:19.5962024/06/04 03:36:19.596 2024/06/04 03:36:19.597 FIG10b: Memory usage without flush after 16113 tasks: 1097.875 MB +worker1-1 | 2024/06/04 03:36:20.600 FIG10b: Memory usage without flush after 16219 tasks: 1098.4921875 MB +worker1-1 | 2024/06/04 03:36:21.601 2024/06/04 03:36:21.602 FIG10b: Memory usage without flush after 16330 tasks: 1097.99609375 MB +worker1-1 | 2024/06/04 03:36:22.607[worker] FIG10b: Memory usage without flush after 16451 tasks: 1096.95703125 MB +worker1-1 | 2024/06/04 03:36:23.6132024/06/04 03:36:23.613 FIG10b: Memory usage without flush after 16577 tasks: 1097.08203125 MB +worker1-1 | 2024/06/04 03:36:24.615 FIG10b: Memory usage without flush after 16692 tasks: 1098.44921875 MB +worker1-1 | 2024/06/04 03:36:25.619 FIG10b: Memory usage without flush after 16810 tasks: 1098.3203125 MB +worker1-1 | 2024/06/04 03:36:26.6232024/06/04 03:36:26.623 FIG10b: Memory usage without flush after 16929 tasks: 1098.296875 MB +worker1-1 | [worker] [worker] took this many seconds: 0.00839 2024/06/04 03:36:27.629 FIG10b: Memory usage without flush after 17054 tasks: 1098.3046875 MB +worker1-1 | 2024/06/04 03:36:28.639 2024/06/04 03:36:28.639FIG10b: Memory usage without flush after 17176 tasks: 1098.6953125 MB +worker1-1 | 2024/06/04 03:36:29.644 FIG10b: Memory usage without flush after 17299 tasks: 1098.828125 MB +worker1-1 | 2024/06/04 03:36:30.649 FIG10b: Memory usage without flush after 17423 tasks: 1099.31640625 MB +worker1-1 | 2024/06/04 03:36:31.657 FIG10b: Memory usage without flush after 17544 tasks: 1099.48046875 MB +worker1-1 | in Run() rn2024/06/04 03:36:32.659 FIG10b: Memory usage without flush after 17673 tasks: 1099.81640625 MB +worker1-1 | 2024/06/04 03:36:33.6642024/06/04 03:36:33.665 FIG10b: Memory usage without flush after 17787 tasks: 1099.8125 MB +worker1-1 | 2024/06/04 03:36:34.665 FIG10b: Memory usage without flush after 17902 tasks: 1099.734375 MB +worker1-1 | 2024/06/04 03:36:35.6672024/06/04 03:36:35.667 FIG10b: Memory usage without flush after 18025 tasks: 1100.22265625 MB +worker1-1 | [worker] 2024/06/04 03:36:36.673 FIG10b: Memory usage without flush after 18159 tasks: 1100.91015625 MB +worker1-1 | FIG10b: Memory usage without flush after 18273 tasks: 1101.87109375 MB +worker1-1 | 2024/06/04 03:36:38.681 FIG10b: Memory usage without flush after 18390 tasks: 1101.14453125 MB +worker1-1 | 2024/06/04 03:36:39.684 FIG10b: Memory usage without flush after 18504 tasks: 1101.84765625 MB +worker1-1 | 2024/06/04 03:36:40.6882024/06/04 03:36:40.688 FIG10b: Memory usage without flush after 18616 tasks: 1102.7890625 MB +worker1-1 | 2024/06/04 03:36:41.696 FIG10b: Memory usage without flush after 18741 tasks: 1102.9375 MB +worker1-1 | FIG10b: Memory usage without flush after 18871 tasks: 1103.6171875 MB +worker1-1 | 2024/06/04 03:36:43.712 FIG10b: Memory usage without flush after 18998 tasks: 1104.55859375 MB +worker1-1 | [worker] in Run() rn2024/06/04 03:36:44.712 FIG10b: Memory usage without flush after 19127 tasks: 1105.1796875 MB +worker1-1 | 2024/06/04 03:36:45.713 FIG10b: Memory usage without flush after 19259 tasks: 1105.37890625 MB +worker1-1 | 2024/06/04 03:36:46.721 FIG10b: Memory usage without flush after 19372 tasks: 1105.16015625 MB +worker1-1 | 2024/06/04 03:36:47.735 FIG10b: Memory usage without flush after 19424 tasks: 1105.28125 MB +worker1-1 | 2024/06/04 03:36:48.7402024/06/04 03:36:48.740 FIG10b: Memory usage without flush after 19522 tasks: 1105.9453125 MB +worker1-1 | 2024/06/04 03:36:49.753 FIG10b: Memory usage without flush after 19635 tasks: 1115.32421875 MB +worker1-1 | 2024/06/04 03:36:50.765 FIG10b: Memory usage without flush after 19716 tasks: 1116.33984375 MB +worker1-1 | stored...2024/06/04 03:36:51.773 FIG10b: Memory usage without flush after 19820 tasks: 1116.3515625 MB +worker1-1 | 2024/06/04 03:36:52.782 FIG10b: Memory usage without flush after 19906 tasks: 1116.1484375 MB +worker1-1 | in Run() rn2024/06/04 03:36:53.788 FIG10b: Memory usage without flush after 20029 tasks: 1117.4765625 MB +worker1-1 | 2024/06/04 03:36:54.8012024/06/04 03:36:54.802 FIG10b: Memory usage without flush after 20145 tasks: 1117.5625 MB +worker1-1 | 2024/06/04 03:36:55.8042024/06/04 03:36:55.805 FIG10b: Memory usage without flush after 20265 tasks: 1118.0 MB +worker1-1 | 2024/06/04 03:36:56.804 [worker]2024/06/04 03:36:56.804 FIG10b: Memory usage without flush after 20381 tasks: 1118.46875 MB +worker1-1 | 2024/06/04 03:36:57.807 FIG10b: Memory usage without flush after 20496 tasks: 1118.36328125 MB +worker1-1 | 2024/06/04 03:36:58.810 [worker]2024/06/04 03:36:58.811 FIG10b: Memory usage without flush after 20611 tasks: 1119.22265625 MB +worker1-1 | 2024/06/04 03:36:59.815 FIG10b: Memory usage without flush after 20730 tasks: 1119.8828125 MB +worker1-1 | 2024/06/04 03:37:00.827 FIG10b: Memory usage without flush after 20852 tasks: 1120.515625 MB +worker1-1 | 2024/06/04 03:37:01.830 FIG10b: Memory usage without flush after 20969 tasks: 1120.85546875 MB +worker1-1 | 2024/06/04 03:37:02.836 [worker] 2024/06/04 03:37:02.837 FIG10b: Memory usage without flush after 21093 tasks: 1120.75390625 MB +worker1-1 | 2024/06/04 03:37:03.844 FIG10b: Memory usage without flush after 21214 tasks: 1121.13671875 MB +worker1-1 | 2024/06/04 03:37:04.845 FIG10b: Memory usage without flush after 21338 tasks: 1121.2421875 MB +worker1-1 | 2024/06/04 03:37:05.845 FIG10b: Memory usage without flush after 21456 tasks: 1121.77734375 MB +worker1-1 | 2024/06/04 03:37:06.851 FIG10b: Memory usage without flush after 21582 tasks: 1121.4296875 MB +worker1-1 | 2024/06/04 03:37:07.855 FIG10b: Memory usage without flush after 21698 tasks: 1121.3203125 MB +worker1-1 | 2024/06/04 03:37:08.855 FIG10b: Memory usage without flush after 21819 tasks: 1122.23046875 MB +worker1-1 | 2024/06/04 03:37:09.8592024/06/04 03:37:09.860 FIG10b: Memory usage without flush after 21938 tasks: 1121.94140625 MB +worker1-1 | 2024/06/04 03:37:10.865 FIG10b: Memory usage without flush after 22061 tasks: 1122.046875 MB +worker1-1 | 2024/06/04 03:37:11.867 2024/06/04 03:37:11.867[worker] Executed!FIG10b: Memory usage without flush after 22185 tasks: 1122.3828125 MB +worker1-1 | 2024/06/04 03:37:12.868 FIG10b: Memory usage without flush after 22307 tasks: 1122.72265625 MB +worker1-1 | 2024/06/04 03:37:13.879 FIG10b: Memory usage without flush after 22425 tasks: 1122.13671875 MB +worker1-1 | 2024/06/04 03:37:14.888 FIG10b: Memory usage without flush after 22551 tasks: 1123.8359375 MB +worker1-1 | 2024/06/04 03:37:15.890 FIG10b: Memory usage without flush after 22671 tasks: 1123.9375 MB +worker1-1 | [worker] [worker] 2024/06/04 03:37:16.893 FIG10b: Memory usage without flush after 22789 tasks: 1123.1796875 MB +worker1-1 | 2024/06/04 03:37:17.903 FIG10b: Memory usage without flush after 22897 tasks: 1123.37109375 MB +worker1-1 | 2024/06/04 03:37:18.911 FIG10b: Memory usage without flush after 23011 tasks: 1123.01953125 MB +worker1-1 | FIG10b: Memory usage without flush after 23127 tasks: 1123.34765625 MB +worker1-1 | 2024/06/04 03:37:20.918 FIG10b: Memory usage without flush after 23234 tasks: 1123.0546875 MB +worker1-1 | 2024/06/04 03:37:21.922 FIG10b: Memory usage without flush after 23352 tasks: 1122.96875 MB +worker1-1 | 2024/06/04 03:37:22.927 FIG10b: Memory usage without flush after 23475 tasks: 1124.24609375 MB +worker1-1 | 2024/06/04 03:37:23.937 2024/06/04 03:37:23.938 FIG10b: Memory usage without flush after 23599 tasks: 1124.05859375 MB +worker1-1 | 2024/06/04 03:37:24.941 FIG10b: Memory usage without flush after 23723 tasks: 1124.3671875 MB +worker1-1 | FIG10b: Memory usage without flush after 23850 tasks: 1124.79296875 MB +worker1-1 | 2024/06/04 03:37:26.951 FIG10b: Memory usage without flush after 23975 tasks: 1124.68359375 MB +worker1-1 | 2024/06/04 03:37:27.958 [worker] 2024/06/04 03:37:27.959 FIG10b: Memory usage without flush after 24085 tasks: 1124.91015625 MB +worker1-1 | 2024/06/04 03:37:28.959 FIG10b: Memory usage without flush after 24202 tasks: 1124.66015625 MB +worker1-1 | 2024/06/04 03:37:29.968 FIG10b: Memory usage without flush after 24319 tasks: 1124.80078125 MB +worker1-1 | 2024/06/04 03:37:30.968 FIG10b: Memory usage without flush after 24438 tasks: 1123.5859375 MB +worker1-1 | 2024/06/04 03:37:31.970 FIG10b: Memory usage without flush after 24565 tasks: 1125.1875 MB +worker1-1 | 2024/06/04 03:37:32.978 FIG10b: Memory usage without flush after 24694 tasks: 1125.23828125 MB +worker1-1 | Executed!2024/06/04 03:37:33.990 FIG10b: Memory usage without flush after 24818 tasks: 1124.94140625 MB +worker1-1 | 2024/06/04 03:37:34.9982024/06/04 03:37:34.998 FIG10b: Memory usage without flush after 24936 tasks: 1125.39453125 MB +worker1-1 | 2024/06/04 03:37:36.003 FIG10b: Memory usage without flush after 25061 tasks: 1125.3203125 MB +worker1-1 | 2024/06/04 03:37:37.010 FIG10b: Memory usage without flush after 25176 tasks: 1126.4296875 MB +worker1-1 | 2024/06/04 03:37:38.010 2024/06/04 03:37:38.010 2024/06/04 03:37:38.011 FIG10b: Memory usage without flush after 25291 tasks: 1126.6875 MB +worker1-1 | 2024/06/04 03:37:39.020 FIG10b: Memory usage without flush after 25403 tasks: 1126.13671875 MB +worker1-1 | 2024/06/04 03:37:40.029 FIG10b: Memory usage without flush after 25521 tasks: 1126.3515625 MB +worker1-1 | 2024/06/04 03:37:41.029 FIG10b: Memory usage without flush after 25635 tasks: 1126.3203125 MB +worker1-1 | 2024/06/04 03:37:42.078 FIG10b: Memory usage without flush after 25706 tasks: 1126.84375 MB +worker1-1 | 2024/06/04 03:37:43.086 FIG10b: Memory usage without flush after 25765 tasks: 1127.20703125 MB +worker1-1 | 2024/06/04 03:37:44.089 FIG10b: Memory usage without flush after 25870 tasks: 1127.4453125 MB +worker1-1 | 2024/06/04 03:37:45.093 FIG10b: Memory usage without flush after 25993 tasks: 1127.94921875 MB +worker1-1 | [worker] gonna store now2024/06/04 03:37:46.097 FIG10b: Memory usage without flush after 26123 tasks: 1128.65234375 MB +worker1-1 | 2024/06/04 03:37:47.100 [worker]2024/06/04 03:37:47.100 FIG10b: Memory usage without flush after 26246 tasks: 1130.03125 MB +worker1-1 | 2024/06/04 03:37:48.102 FIG10b: Memory usage without flush after 26364 tasks: 1130.19921875 MB +worker1-1 | 2024/06/04 03:37:49.106 FIG10b: Memory usage without flush after 26484 tasks: 1130.30859375 MB +worker1-1 | 2024/06/04 03:37:50.111 FIG10b: Memory usage without flush after 26617 tasks: 1135.0 MB +worker1-1 | 2024/06/04 03:37:51.118 FIG10b: Memory usage without flush after 26744 tasks: 1140.1171875 MB +worker1-1 | 2024/06/04 03:37:52.123 FIG10b: Memory usage without flush after 26860 tasks: 1140.51953125 MB +worker1-1 | FIG10b: Memory usage without flush after 26984 tasks: 1140.375 MB +worker1-1 | 2024/06/04 03:37:54.135 FIG10b: Memory usage without flush after 27100 tasks: 1140.375 MB +worker1-1 | 2024/06/04 03:37:55.138 FIG10b: Memory usage without flush after 27218 tasks: 1141.25390625 MB +worker1-1 | 2024/06/04 03:37:56.143 FIG10b: Memory usage without flush after 27332 tasks: 1139.09375 MB +worker1-1 | 2024/06/04 03:37:57.149 FIG10b: Memory usage without flush after 27451 tasks: 1140.0078125 MB +worker1-1 | 2024/06/04 03:37:58.152 FIG10b: Memory usage without flush after 27578 tasks: 1144.21875 MB +worker1-1 | 2024/06/04 03:37:59.155 FIG10b: Memory usage without flush after 27698 tasks: 1144.8671875 MB +worker1-1 | 2024/06/04 03:38:00.164 FIG10b: Memory usage without flush after 27830 tasks: 1145.25390625 MB +worker1-1 | 2024/06/04 03:38:01.168 FIG10b: Memory usage without flush after 27952 tasks: 1145.1640625 MB +worker1-1 | [worker] 2024/06/04 03:38:02.177stored... FIG10b: Memory usage without flush after 28075 tasks: 1146.30859375 MB +worker1-1 | 2024/06/04 03:38:03.1812024/06/04 03:38:03.181 FIG10b: Memory usage without flush after 28195 tasks: 1146.046875 MB +worker1-1 | 2024/06/04 03:38:04.191 FIG10b: Memory usage without flush after 28321 tasks: 1146.23828125 MB +worker1-1 | 2024/06/04 03:38:05.198 FIG10b: Memory usage without flush after 28443 tasks: 1146.875 MB +worker1-1 | 2024/06/04 03:38:06.1992024/06/04 03:38:06.199 FIG10b: Memory usage without flush after 28567 tasks: 1147.00390625 MB +worker1-1 | 2024/06/04 03:38:07.202 FIG10b: Memory usage without flush after 28687 tasks: 1147.734375 MB +worker1-1 | stored...2024/06/04 03:38:08.2082024/06/04 03:38:08.208 FIG10b: Memory usage without flush after 28812 tasks: 1147.4140625 MB +worker1-1 | 2024/06/04 03:38:09.212 FIG10b: Memory usage without flush after 28931 tasks: 1147.59765625 MB +worker1-1 | 2024/06/04 03:38:10.216 FIG10b: Memory usage without flush after 29010 tasks: 1147.1953125 MB +worker1-1 | [worker]2024/06/04 03:38:11.225 FIG10b: Memory usage without flush after 29110 tasks: 1146.921875 MB +worker1-1 | FIG10b: Memory usage without flush after 29236 tasks: 1148.05078125 MB +worker1-1 | 2024/06/04 03:38:13.2322024/06/04 03:38:13.232 [worker]FIG10b: Memory usage without flush after 29359 tasks: 1148.52734375 MB +worker1-1 | 2024/06/04 03:38:14.2402024/06/04 03:38:14.240 FIG10b: Memory usage without flush after 29478 tasks: 1151.6875 MB +worker1-1 | 2024/06/04 03:38:15.2432024/06/04 03:38:15.244 [worker]FIG10b: Memory usage without flush after 29600 tasks: 1150.91796875 MB +worker1-1 | 2024/06/04 03:38:16.255 FIG10b: Memory usage without flush after 29716 tasks: 1150.5625 MB +worker1-1 | 2024/06/04 03:38:17.260 FIG10b: Memory usage without flush after 29842 tasks: 1148.6796875 MB +worker1-1 | [worker] stored...2024/06/04 03:38:18.275 FIG10b: Memory usage without flush after 29974 tasks: 1147.09765625 MB +worker1-1 | 2024/06/04 03:38:19.285 FIG10b: Memory usage without flush after 30098 tasks: 1146.625 MB +worker1-1 | 2024/06/04 03:38:20.289 FIG10b: Memory usage without flush after 30192 tasks: 1146.55078125 MB +worker1-1 | 2024/06/04 03:38:21.2932024/06/04 03:38:21.294 FIG10b: Memory usage without flush after 30301 tasks: 1146.21875 MB +worker1-1 | 2024/06/04 03:38:22.301 [worker]2024/06/04 03:38:22.3022024/06/04 03:38:22.303 FIG10b: Memory usage without flush after 30418 tasks: 1146.8828125 MB +worker1-1 | Executed![worker]2024/06/04 03:38:23.307 FIG10b: Memory usage without flush after 30543 tasks: 1146.13671875 MB +worker1-1 | 2024/06/04 03:38:24.309 FIG10b: Memory usage without flush after 30668 tasks: 1147.875 MB +worker1-1 | 2024/06/04 03:38:25.311 FIG10b: Memory usage without flush after 30793 tasks: 1147.71875 MB +worker1-1 | 2024/06/04 03:38:26.317 FIG10b: Memory usage without flush after 30915 tasks: 1147.58984375 MB +worker1-1 | 2024/06/04 03:38:27.323 2024/06/04 03:38:27.323 2024/06/04 03:38:27.324 FIG10b: Memory usage without flush after 31047 tasks: 1147.44140625 MB +worker1-1 | 2024/06/04 03:38:28.3372024/06/04 03:38:28.338 FIG10b: Memory usage without flush after 31175 tasks: 1147.54296875 MB +worker1-1 | 2024/06/04 03:38:29.341 FIG10b: Memory usage without flush after 31313 tasks: 1147.875 MB +worker1-1 | [worker] 2024/06/04 03:38:30.345 FIG10b: Memory usage without flush after 31437 tasks: 1147.96875 MB +worker1-1 | [worker][worker]2024/06/04 03:38:31.352 FIG10b: Memory usage without flush after 31560 tasks: 1148.609375 MB +worker1-1 | 2024/06/04 03:38:32.352 FIG10b: Memory usage without flush after 31685 tasks: 1148.92578125 MB +worker1-1 | 2024/06/04 03:38:33.3572024/06/04 03:38:33.357 FIG10b: Memory usage without flush after 31809 tasks: 1149.67578125 MB +worker1-1 | 2024/06/04 03:38:34.360 FIG10b: Memory usage without flush after 31941 tasks: 1153.609375 MB +worker1-1 | 2024/06/04 03:38:35.364 2024/06/04 03:38:35.364 FIG10b: Memory usage without flush after 32062 tasks: 1153.7421875 MB +worker1-1 | 2024/06/04 03:38:36.369 FIG10b: Memory usage without flush after 32193 tasks: 1153.8046875 MB +worker1-1 | 2024/06/04 03:38:37.3752024/06/04 03:38:37.376 FIG10b: Memory usage without flush after 32320 tasks: 1153.8984375 MB +worker1-1 | 2024/06/04 03:38:38.383 FIG10b: Memory usage without flush after 32445 tasks: 1153.578125 MB +worker1-1 | 2024/06/04 03:38:39.389 FIG10b: Memory usage without flush after 32571 tasks: 1154.625 MB +worker1-1 | 2024/06/04 03:38:40.3912024/06/04 03:38:40.392 FIG10b: Memory usage without flush after 32701 tasks: 1154.89453125 MB +worker1-1 | 2024/06/04 03:38:41.394 FIG10b: Memory usage without flush after 32826 tasks: 1154.28125 MB +worker1-1 | FIG10b: Memory usage without flush after 32949 tasks: 1155.33984375 MB +worker1-1 | 2024/06/04 03:38:43.4042024/06/04 03:38:43.404 2024/06/04 03:38:43.404 FIG10b: Memory usage without flush after 33069 tasks: 1155.69921875 MB +worker1-1 | in Run() rn2024/06/04 03:38:44.410 FIG10b: Memory usage without flush after 33196 tasks: 1156.5 MB +worker1-1 | FIG10b: Memory usage without flush after 33315 tasks: 1156.82421875 MB +worker1-1 | 2024/06/04 03:38:46.416 FIG10b: Memory usage without flush after 33444 tasks: 1156.66015625 MB +worker1-1 | 2024/06/04 03:38:47.418 2024/06/04 03:38:47.419 FIG10b: Memory usage without flush after 33569 tasks: 1157.25 MB +worker1-1 | stored...2024/06/04 03:38:48.422 2024/06/04 03:38:48.423 FIG10b: Memory usage without flush after 33699 tasks: 1157.5703125 MB +worker1-1 | 2024/06/04 03:38:49.429 FIG10b: Memory usage without flush after 33843 tasks: 1166.3828125 MB +worker1-1 | FIG10b: Memory usage without flush after 33977 tasks: 1166.5703125 MB +worker1-1 | 2024/06/04 03:38:51.439 FIG10b: Memory usage without flush after 34102 tasks: 1166.8671875 MB +worker1-1 | 2024/06/04 03:38:52.438 FIG10b: Memory usage without flush after 34224 tasks: 1167.05859375 MB +worker1-1 | 2024/06/04 03:38:53.446 FIG10b: Memory usage without flush after 34349 tasks: 1160.99609375 MB +worker1-1 | gonna store now2024/06/04 03:38:54.447 2024/06/04 03:38:54.448 FIG10b: Memory usage without flush after 34476 tasks: 1160.08984375 MB +worker1-1 | FIG10b: Memory usage without flush after 34607 tasks: 1160.0078125 MB +worker1-1 | 2024/06/04 03:39:25.455 FIG10b: Memory usage without flush after 38383 tasks: 1168.05078125 MB +worker1-1 | 2024/06/04 03:39:55.459 2024/06/04 03:39:55.459[worker] stored...FIG10b: Memory usage without flush after 42110 tasks: 1179.35546875 MB +worker1-1 | 2024/06/04 03:40:25.4622024/06/04 03:40:25.463 2024/06/04 03:40:25.463 FIG10b: Memory usage without flush after 45621 tasks: 1181.609375 MB +worker1-1 | 2024/06/04 03:40:55.469 FIG10b: Memory usage without flush after 49078 tasks: 1239.30859375 MB +worker1-1 | [worker]Executed! 2024/06/04 03:41:25.473 FIG10b: Memory usage without flush after 52745 tasks: 1236.12890625 MB +worker1-1 | 2024/06/04 03:41:55.479 FIG10b: Memory usage without flush after 56159 tasks: 1259.33203125 MB +worker1-1 | 2024/06/04 03:42:25.482 2024/06/04 03:42:25.483 FIG10b: Memory usage without flush after 59753 tasks: 1264.109375 MB +worker1-1 | 2024/06/04 03:42:55.488 FIG10b: Memory usage without flush after 63191 tasks: 1283.40625 MB +worker1-1 | 2024/06/04 03:43:25.496 FIG10b: Memory usage without flush after 66727 tasks: 1284.46875 MB +worker1-1 | [worker] 2024/06/04 03:43:55.501 FIG10b: Memory usage without flush after 70311 tasks: 1311.26953125 MB +worker1-1 | FIG10b: Memory usage without flush after 74003 tasks: 1314.0703125 MB +worker1-1 | [worker]2024/06/04 03:44:55.505 FIG10b: Memory usage without flush after 77762 tasks: 1350.19140625 MB +worker1-1 | FIG10b: Memory usage without flush after 81519 tasks: 1345.234375 MB +worker1-1 | 2024/06/04 03:45:55.516 2024/06/04 03:45:55.517[worker] FIG10b: Memory usage without flush after 85240 tasks: 1362.51171875 MB +worker1-1 | 2024/06/04 03:46:25.521 FIG10b: Memory usage without flush after 88858 tasks: 1366.875 MB +worker1-1 | 2024/06/04 03:46:55.5212024/06/04 03:46:55.521 FIG10b: Memory usage without flush after 92388 tasks: 1377.2109375 MB +worker1-1 | 2024/06/04 03:47:25.524 FIG10b: Memory usage without flush after 95891 tasks: 1382.34375 MB +worker1-1 | 2024/06/04 03:47:55.534 FIG10b: Memory usage without flush after 99075 tasks: 1372.3203125 MB +worker1-1 | 2024/06/04 03:48:25.535[worker] FIG10b: Memory usage without flush after 102285 tasks: 1379.0234375 MBin Run() rn +worker1-1 | 2024/06/04 03:48:55.535 2024/06/04 03:48:55.536[worker] FIG10b: Memory usage without flush after 105536 tasks: 1406.2265625 MB +worker1-1 | 2024/06/04 03:49:25.541 FIG10b: Memory usage without flush after 108916 tasks: 1444.30078125 MB +worker1-1 | 2024/06/04 03:49:55.5422024/06/04 03:49:55.542 FIG10b: Memory usage without flush after 112426 tasks: 1444.98828125 MB +worker1-1 | 2024/06/04 03:50:25.546 FIG10b: Memory usage without flush after 115719 tasks: 1449.98828125 MB +worker1-1 | 2024/06/04 03:50:55.551 FIG10b: Memory usage without flush after 118837 tasks: 1466.87890625 MB +worker1-1 | 2024/06/04 03:51:25.5532024/06/04 03:51:25.553 FIG10b: Memory usage without flush after 122080 tasks: 1470.30859375 MB +worker1-1 | 2024/06/04 03:51:55.557 FIG10b: Memory usage without flush after 125602 tasks: 1483.0234375 MB +worker1-1 | 2024/06/04 03:52:25.559 [worker]2024/06/04 03:52:25.560 FIG10b: Memory usage without flush after 129043 tasks: 1481.91796875 MB +worker1-1 | 2024/06/04 03:52:55.563 FIG10b: Memory usage without flush after 132450 tasks: 1513.37890625 MB +worker1-1 | 2024/06/04 03:53:25.566 FIG10b: Memory usage without flush after 135785 tasks: 1512.1796875 MB +worker1-1 | 2024/06/04 03:53:55.569 2024/06/04 03:53:55.570 FIG10b: Memory usage without flush after 139343 tasks: 1516.23828125 MB +worker1-1 | 2024/06/04 03:54:25.576 FIG10b: Memory usage without flush after 142807 tasks: 1515.0546875 MB +worker1-1 | 2024/06/04 03:54:55.578 FIG10b: Memory usage without flush after 146258 tasks: 1530.1796875 MB +worker1-1 | 2024/06/04 03:55:25.5922024/06/04 03:55:25.593 FIG10b: Memory usage without flush after 149944 tasks: 1538.828125 MB +worker1-1 | 2024/06/04 03:55:55.593 FIG10b: Memory usage without flush after 153732 tasks: 1552.4609375 MB +worker1-1 | 2024/06/04 03:56:25.5972024/06/04 03:56:25.597 [worker] gonna store now FIG10b: Memory usage without flush after 157388 tasks: 1559.82421875 MB +worker1-1 | 2024/06/04 03:56:55.602 FIG10b: Memory usage without flush after 160873 tasks: 1554.2890625 MB +worker1-1 | 2024/06/04 03:57:25.603 FIG10b: Memory usage without flush after 164476 tasks: 1551.7109375 MB +worker1-1 | 2024/06/04 03:57:55.605 FIG10b: Memory usage without flush after 167926 tasks: 1572.1171875 MB +worker1-1 | 2024/06/04 03:58:25.605 FIG10b: Memory usage without flush after 171576 tasks: 1562.6640625 MB +worker1-1 | 2024/06/04 03:58:55.606 FIG10b: Memory usage without flush after 174582 tasks: 1589.87109375 MB +worker1-1 | 2024/06/04 03:59:25.609 FIG10b: Memory usage without flush after 177860 tasks: 1593.6640625 MB +worker1-1 | 2024/06/04 03:59:55.6102024/06/04 03:59:55.610 FIG10b: Memory usage without flush after 181241 tasks: 1601.9296875 MB +worker1-1 | 2024/06/04 04:00:25.6152024/06/04 04:00:25.615 [worker] FIG10b: Memory usage without flush after 184613 tasks: 1588.4140625 MBExecuted! +worker1-1 | in Run() rn2024/06/04 04:00:55.616 FIG10b: Memory usage without flush after 187904 tasks: 1619.25 MB +worker1-1 | 2024/06/04 04:01:25.623 FIG10b: Memory usage without flush after 191567 tasks: 1618.2421875 MB +worker1-1 | 2024/06/04 04:01:55.630 FIG10b: Memory usage without flush after 195241 tasks: 1628.71484375 MB +worker1-1 | 2024/06/04 04:02:25.632 FIG10b: Memory usage without flush after 198610 tasks: 1638.32421875 MB +worker1-1 | 2024/06/04 04:02:55.637 FIG10b: Memory usage without flush after 201951 tasks: 1632.52734375 MB +worker1-1 | 2024/06/04 04:03:25.640 FIG10b: Memory usage without flush after 205401 tasks: 1623.203125 MB +worker1-1 | [worker]gonna store now 2024/06/04 04:03:55.641 FIG10b: Memory usage without flush after 208998 tasks: 1651.734375 MB +worker1-1 | FIG10b: Memory usage without flush after 212537 tasks: 1638.828125 MB +worker1-1 | 2024/06/04 04:04:55.648 FIG10b: Memory usage without flush after 216026 tasks: 1695.09375 MB +worker1-1 | 2024/06/04 04:05:25.6512024/06/04 04:05:25.652 FIG10b: Memory usage without flush after 219372 tasks: 1726.16015625 MB +worker1-1 | 2024/06/04 04:05:55.653 FIG10b: Memory usage without flush after 222864 tasks: 1734.11328125 MB +worker1-1 | [worker]2024/06/04 04:06:25.6542024/06/04 04:06:25.654 FIG10b: Memory usage without flush after 226081 tasks: 1731.8203125 MB +worker1-1 | 2024/06/04 04:06:55.658 FIG10b: Memory usage without flush after 229701 tasks: 1756.484375 MB +worker1-1 | 2024/06/04 04:07:25.660 FIG10b: Memory usage without flush after 233367 tasks: 1700.55078125 MB +worker1-1 | 2024/06/04 04:07:55.664 FIG10b: Memory usage without flush after 236870 tasks: 1724.6171875 MB +worker1-1 | 2024/06/04 04:08:25.668 FIG10b: Memory usage without flush after 240530 tasks: 1729.3984375 MB +worker1-1 | 2024/06/04 04:08:55.668 2024/06/04 04:08:55.669 FIG10b: Memory usage without flush after 244174 tasks: 1721.2421875 MB +worker1-1 | 2024/06/04 04:09:25.679 FIG10b: Memory usage without flush after 247469 tasks: 1728.60546875 MB +worker1-1 | 2024/06/04 04:09:55.678 FIG10b: Memory usage without flush after 250949 tasks: 1728.96875 MB +worker1-1 | FIG10b: Memory usage without flush after 254053 tasks: 1734.640625 MB +worker1-1 | 2024/06/04 04:10:55.684 FIG10b: Memory usage without flush after 257225 tasks: 1768.9609375 MB +worker1-1 | 2024/06/04 04:11:25.689 FIG10b: Memory usage without flush after 260317 tasks: 1773.13671875 MB +worker1-1 | 2024/06/04 04:11:55.691 FIG10b: Memory usage without flush after 263734 tasks: 1784.29296875 MB +worker1-1 | 2024/06/04 04:12:25.6922024/06/04 04:12:25.692 FIG10b: Memory usage without flush after 268056 tasks: 1514.75390625 MB +worker1-1 | 2024/06/04 04:12:55.693 FIG10b: Memory usage without flush after 273259 tasks: 1509.0625 MB +worker1-1 | 2024/06/04 04:13:25.701 FIG10b: Memory usage without flush after 278844 tasks: 1508.140625 MB +worker1-1 | 2024/06/04 04:13:55.702 FIG10b: Memory usage without flush after 284273 tasks: 1508.68359375 MB +worker1-1 | 2024/06/04 04:14:25.705 FIG10b: Memory usage without flush after 289699 tasks: 1504.7578125 MB +worker1-1 | 2024/06/04 04:14:55.709 FIG10b: Memory usage without flush after 295409 tasks: 1491.7421875 MB +worker1-1 | 2024/06/04 04:15:25.7092024/06/04 04:15:25.709 FIG10b: Memory usage without flush after 300543 tasks: 1506.33984375 MB +worker1-1 | 2024/06/04 04:15:55.714 FIG10b: Memory usage without flush after 305443 tasks: 1508.61328125 MB +worker1-1 | [worker]2024/06/04 04:16:25.720 FIG10b: Memory usage without flush after 311212 tasks: 1500.80859375 MB +worker1-1 | 2024/06/04 04:16:55.722 FIG10b: Memory usage without flush after 316949 tasks: 1501.8125 MB +worker1-1 | 2024/06/04 04:17:25.727 FIG10b: Memory usage without flush after 322754 tasks: 1497.07421875 MB +worker1-1 | 2024/06/04 04:17:55.729 FIG10b: Memory usage without flush after 328758 tasks: 1498.6484375 MB +worker1-1 | 2024/06/04 04:18:25.732 FIG10b: Memory usage without flush after 334751 tasks: 1496.65234375 MB +worker1-1 | 2024/06/04 04:18:55.737 FIG10b: Memory usage without flush after 340765 tasks: 1499.8125 MB +worker1-1 | 2024/06/04 04:19:25.737 FIG10b: Memory usage without flush after 346433 tasks: 1515.08203125 MB +worker1-1 | 2024/06/04 04:19:55.744 FIG10b: Memory usage without flush after 350800 tasks: 1512.96875 MB +worker1-1 | 2024/06/04 04:20:25.744 FIG10b: Memory usage without flush after 356010 tasks: 1511.6015625 MB +worker1-1 | 2024/06/04 04:20:55.7462024/06/04 04:20:55.746 FIG10b: Memory usage without flush after 361839 tasks: 1513.046875 MB +worker1-1 | 2024/06/04 04:21:25.748 FIG10b: Memory usage without flush after 367734 tasks: 1513.9921875 MB +worker1-1 | 2024/06/04 04:21:55.753 FIG10b: Memory usage without flush after 373440 tasks: 1521.23046875 MB +worker1-1 | 2024/06/04 04:22:25.754 FIG10b: Memory usage without flush after 379209 tasks: 1522.34375 MB +worker1-1 | 2024/06/04 04:22:55.759 [worker] gonna store now2024/06/04 04:22:55.759 FIG10b: Memory usage without flush after 385098 tasks: 1524.828125 MB +worker1-1 | 2024/06/04 04:23:25.763 [worker] 2024/06/04 04:23:25.763 FIG10b: Memory usage without flush after 391063 tasks: 1524.265625 MB +worker1-1 | 2024/06/04 04:23:55.768 2024/06/04 04:23:55.770[worker] FIG10b: Memory usage without flush after 396503 tasks: 1524.72265625 MB +worker1-1 | 2024/06/04 04:24:25.772 FIG10b: Memory usage without flush after 401902 tasks: 1524.765625 MB +worker1-1 | 2024/06/04 04:24:55.773 FIG10b: Memory usage without flush after 406157 tasks: 1514.16796875 MB +worker1-1 | 2024/06/04 04:25:25.776 FIG10b: Memory usage without flush after 411477 tasks: 1508.60546875 MB +worker1-1 | 2024/06/04 04:25:55.780 FIG10b: Memory usage without flush after 417100 tasks: 1505.30859375 MB +worker1-1 | 2024/06/04 04:26:25.782 FIG10b: Memory usage without flush after 422902 tasks: 1510.6328125 MB +worker1-1 | 2024/06/04 04:26:55.787 2024/06/04 04:26:55.787FIG10b: Memory usage without flush after 428513 tasks: 1482.6640625 MB +worker1-1 | 2024/06/04 04:27:25.789 FIG10b: Memory usage without flush after 434246 tasks: 1496.74609375 MB +worker1-1 | 2024/06/04 04:27:55.793 FIG10b: Memory usage without flush after 440186 tasks: 1498.75 MB +worker1-1 | 2024/06/04 04:28:25.794 FIG10b: Memory usage without flush after 446138 tasks: 1508.09375 MB +worker1-1 | FIG10b: Memory usage without flush after 452049 tasks: 1510.578125 MB +worker1-1 | 2024/06/04 04:29:25.797 FIG10b: Memory usage without flush after 457856 tasks: 1517.16015625 MB +worker1-1 | 2024/06/04 04:29:55.803 FIG10b: Memory usage without flush after 463820 tasks: 1510.99609375 MB +worker1-1 | 2024/06/04 04:30:25.804 FIG10b: Memory usage without flush after 469815 tasks: 1522.984375 MB +worker1-1 | 2024/06/04 04:30:55.8062024/06/04 04:30:55.806 FIG10b: Memory usage without flush after 475805 tasks: 1523.7578125 MB +worker1-1 | 2024/06/04 04:31:25.808 FIG10b: Memory usage without flush after 481768 tasks: 1523.3515625 MB +worker1-1 | 2024/06/04 04:31:55.811 FIG10b: Memory usage without flush after 486783 tasks: 1527.33203125 MB +worker1-1 | 2024/06/04 04:32:25.811 FIG10b: Memory usage without flush after 492761 tasks: 1529.6015625 MB +worker1-1 | 2024/06/04 04:32:55.816 2024/06/04 04:32:55.817 FIG10b: Memory usage without flush after 498701 tasks: 1528.40234375 MB +worker1-1 | 2024/06/04 04:33:25.821 FIG10b: Memory usage without flush after 504730 tasks: 1532.078125 MB +worker1-1 | 2024/06/04 04:33:55.824 FIG10b: Memory usage without flush after 510617 tasks: 1531.3671875 MB +worker1-1 | [worker] 2024/06/04 04:34:25.829 FIG10b: Memory usage without flush after 516592 tasks: 1520.625 MB +worker1-1 | 2024/06/04 04:34:55.838 FIG10b: Memory usage without flush after 522535 tasks: 1514.1953125 MB +worker1-1 | 2024/06/04 04:35:25.840 FIG10b: Memory usage without flush after 528541 tasks: 1522.14453125 MB +worker1-1 | FIG10b: Memory usage without flush after 534348 tasks: 1526.203125 MB +worker1-1 | 2024/06/04 04:39:34.984 FIG10b: Memory usage without flush after 538865 tasks: 1527.59375 MB +worker1-1 | 2024/06/04 04:40:04.9862024/06/04 04:40:04.986 FIG10b: Memory usage without flush after 543675 tasks: 1524.3828125 MB +worker1-1 | 2024/06/04 04:40:34.991 FIG10b: Memory usage without flush after 548015 tasks: 1540.796875 MB +worker1-1 | 2024/06/04 04:41:04.992 FIG10b: Memory usage without flush after 552634 tasks: 1538.796875 MB +worker1-1 | 2024/06/04 04:41:34.993 FIG10b: Memory usage without flush after 558653 tasks: 1545.36328125 MB +worker1-1 | 2024/06/04 04:42:05.000 FIG10b: Memory usage without flush after 564666 tasks: 1542.1640625 MB +worker1-1 | 2024/06/04 04:42:35.004 FIG10b: Memory usage without flush after 570630 tasks: 1532.55078125 MB +worker1-1 | 2024/06/04 04:43:05.009 FIG10b: Memory usage without flush after 576229 tasks: 1533.05859375 MB +worker1-1 | 2024/06/04 04:43:35.016 FIG10b: Memory usage without flush after 581646 tasks: 1535.55859375 MB +worker1-1 | 2024/06/04 04:44:05.020 FIG10b: Memory usage without flush after 587138 tasks: 1535.04296875 MB +worker1-1 | 2024/06/04 04:44:35.024 FIG10b: Memory usage without flush after 593129 tasks: 1536.75 MB +worker1-1 | gonna store now2024/06/04 04:45:05.028 FIG10b: Memory usage without flush after 599081 tasks: 1540.94140625 MB +worker1-1 | 2024/06/04 04:45:35.030 FIG10b: Memory usage without flush after 604470 tasks: 1541.0234375 MB +worker1-1 | 2024/06/04 04:46:05.030 FIG10b: Memory usage without flush after 608515 tasks: 1529.6484375 MB +worker1-1 | FIG10b: Memory usage without flush after 612468 tasks: 1543.72265625 MB +worker1-1 | 2024/06/04 04:47:05.047 FIG10b: Memory usage without flush after 617421 tasks: 1543.23828125 MB +worker1-1 | 2024/06/04 04:47:35.050 FIG10b: Memory usage without flush after 621885 tasks: 1544.4140625 MB +worker1-1 | 2024/06/04 04:48:05.052 FIG10b: Memory usage without flush after 627460 tasks: 1534.34375 MB +worker1-1 | 2024/06/04 04:48:35.0542024/06/04 04:48:35.054 FIG10b: Memory usage without flush after 633249 tasks: 1529.2890625 MB +worker1-1 | 2024/06/04 04:49:05.056 FIG10b: Memory usage without flush after 639038 tasks: 1531.3984375 MB +worker1-1 | 2024/06/04 04:49:35.060 FIG10b: Memory usage without flush after 644336 tasks: 1532.1796875 MB +worker1-1 | 2024/06/04 04:50:05.064 FIG10b: Memory usage without flush after 649827 tasks: 1537.39453125 MB +worker1-1 | 2024/06/04 04:50:35.068 FIG10b: Memory usage without flush after 655347 tasks: 1534.8828125 MB +worker1-1 | 2024/06/04 04:51:05.071 FIG10b: Memory usage without flush after 660475 tasks: 1534.60546875 MB +worker1-1 | 2024/06/04 04:51:35.0732024/06/04 04:51:35.074 FIG10b: Memory usage without flush after 665921 tasks: 1532.44921875 MB diff --git a/analysis/output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv b/analysis/output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv new file mode 100644 index 0000000..613afdf --- /dev/null +++ b/analysis/output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv @@ -0,0 +1,266 @@ +Timestamp,Memory Usage (MB) +2024-06-04 03:33:55.734000,1012.97265625 +2024-06-04 03:33:56.736000,1013.26171875 +2024-06-04 03:33:58.747000,1017.6484375 +2024-06-04 03:33:59.759000,1021.41796875 +2024-06-04 03:34:02.777000,1022.78125 +2024-06-04 03:34:03.785000,1026.54296875 +2024-06-04 03:34:04.793000,1025.8046875 +2024-06-04 03:34:06.800000,1026.49609375 +2024-06-04 03:34:09.812000,1025.4765625 +2024-06-04 03:34:10.816000,1025.19140625 +2024-06-04 03:34:12.826000,1025.671875 +2024-06-04 03:34:13.827000,1025.4296875 +2024-06-04 03:34:14.846000,1025.375 +2024-06-04 03:34:16.855000,1026.3203125 +2024-06-04 03:34:18.872000,1030.8359375 +2024-06-04 03:34:19.881000,1035.09765625 +2024-06-04 03:34:21.899000,1035.06640625 +2024-06-04 03:34:22.904000,1034.64453125 +2024-06-04 03:34:24.915000,1034.56640625 +2024-06-04 03:34:26.925000,1037.55859375 +2024-06-04 03:34:27.934000,1038.2890625 +2024-06-04 03:34:28.955000,1039.37109375 +2024-06-04 03:34:30.965000,1039.109375 +2024-06-04 03:34:37.010000,1046.2890625 +2024-06-04 03:34:42.052000,1049.14453125 +2024-06-04 03:34:43.052000,1050.30078125 +2024-06-04 03:34:44.061000,1050.24609375 +2024-06-04 03:34:46.073000,1051.2734375 +2024-06-04 03:34:47.079000,1051.11328125 +2024-06-04 03:34:48.081000,1057.07421875 +2024-06-04 03:34:51.100000,1063.61328125 +2024-06-04 03:34:53.106000,1065.19140625 +2024-06-04 03:34:54.110000,1064.7109375 +2024-06-04 03:34:56.118000,1066.42578125 +2024-06-04 03:34:58.134000,1068.85546875 +2024-06-04 03:34:59.137000,1068.48046875 +2024-06-04 03:35:00.142000,1069.43359375 +2024-06-04 03:35:02.158000,1070.07421875 +2024-06-04 03:35:03.166000,1069.4140625 +2024-06-04 03:35:04.167000,1068.91015625 +2024-06-04 03:35:05.171000,1070.0625 +2024-06-04 03:35:07.180000,1070.69921875 +2024-06-04 03:35:08.184000,1071.61328125 +2024-06-04 03:35:11.193000,1071.91796875 +2024-06-04 03:35:13.203000,1073.85546875 +2024-06-04 03:35:15.211000,1075.859375 +2024-06-04 03:35:17.217000,1076.56640625 +2024-06-04 03:35:20.239000,1069.203125 +2024-06-04 03:35:22.254000,1069.48828125 +2024-06-04 03:35:23.254000,1069.06640625 +2024-06-04 03:35:24.260000,1069.4765625 +2024-06-04 03:35:27.272000,1069.46875 +2024-06-04 03:35:30.280000,1070.6875 +2024-06-04 03:35:32.288000,1070.51171875 +2024-06-04 03:35:36.298000,1070.12109375 +2024-06-04 03:35:38.301000,1070.30078125 +2024-06-04 03:35:39.304000,1070.375 +2024-06-04 03:35:42.316000,1075.08984375 +2024-06-04 03:35:44.319000,1076.2265625 +2024-06-04 03:35:45.319000,1076.9453125 +2024-06-04 03:35:46.327000,1077.54296875 +2024-06-04 03:35:49.367000,1095.51171875 +2024-06-04 03:35:50.371000,1096.14453125 +2024-06-04 03:35:53.378000,1091.921875 +2024-06-04 03:35:54.382000,1093.2421875 +2024-06-04 03:35:55.387000,1094.23828125 +2024-06-04 03:35:56.389000,1094.46484375 +2024-06-04 03:35:57.396000,1095.83984375 +2024-06-04 03:35:58.398000,1096.02734375 +2024-06-04 03:35:59.406000,1095.27734375 +2024-06-04 03:36:01.410000,1095.453125 +2024-06-04 03:36:02.417000,1096.1484375 +2024-06-04 03:36:04.434000,1096.8203125 +2024-06-04 03:36:05.444000,1097.08203125 +2024-06-04 03:36:06.502000,1097.53125 +2024-06-04 03:36:07.511000,1097.96875 +2024-06-04 03:36:08.515000,1097.9609375 +2024-06-04 03:36:10.529000,1098.609375 +2024-06-04 03:36:11.532000,1098.86328125 +2024-06-04 03:36:12.546000,1098.69140625 +2024-06-04 03:36:13.551000,1099.46875 +2024-06-04 03:36:14.557000,1099.46484375 +2024-06-04 03:36:20.600000,1098.4921875 +2024-06-04 03:36:24.615000,1098.44921875 +2024-06-04 03:36:25.619000,1098.3203125 +2024-06-04 03:36:29.644000,1098.828125 +2024-06-04 03:36:30.649000,1099.31640625 +2024-06-04 03:36:31.657000,1099.48046875 +2024-06-04 03:36:34.665000,1099.734375 +2024-06-04 03:36:38.681000,1101.14453125 +2024-06-04 03:36:39.684000,1101.84765625 +2024-06-04 03:36:41.696000,1102.9375 +2024-06-04 03:36:43.712000,1104.55859375 +2024-06-04 03:36:45.713000,1105.37890625 +2024-06-04 03:36:46.721000,1105.16015625 +2024-06-04 03:36:47.735000,1105.28125 +2024-06-04 03:36:49.753000,1115.32421875 +2024-06-04 03:36:50.765000,1116.33984375 +2024-06-04 03:36:52.782000,1116.1484375 +2024-06-04 03:36:57.807000,1118.36328125 +2024-06-04 03:36:59.815000,1119.8828125 +2024-06-04 03:37:00.827000,1120.515625 +2024-06-04 03:37:01.830000,1120.85546875 +2024-06-04 03:37:03.844000,1121.13671875 +2024-06-04 03:37:04.845000,1121.2421875 +2024-06-04 03:37:05.845000,1121.77734375 +2024-06-04 03:37:06.851000,1121.4296875 +2024-06-04 03:37:07.855000,1121.3203125 +2024-06-04 03:37:08.855000,1122.23046875 +2024-06-04 03:37:10.865000,1122.046875 +2024-06-04 03:37:12.868000,1122.72265625 +2024-06-04 03:37:13.879000,1122.13671875 +2024-06-04 03:37:14.888000,1123.8359375 +2024-06-04 03:37:15.890000,1123.9375 +2024-06-04 03:37:17.903000,1123.37109375 +2024-06-04 03:37:18.911000,1123.01953125 +2024-06-04 03:37:20.918000,1123.0546875 +2024-06-04 03:37:21.922000,1122.96875 +2024-06-04 03:37:22.927000,1124.24609375 +2024-06-04 03:37:24.941000,1124.3671875 +2024-06-04 03:37:26.951000,1124.68359375 +2024-06-04 03:37:28.959000,1124.66015625 +2024-06-04 03:37:29.968000,1124.80078125 +2024-06-04 03:37:30.968000,1123.5859375 +2024-06-04 03:37:31.970000,1125.1875 +2024-06-04 03:37:32.978000,1125.23828125 +2024-06-04 03:37:36.003000,1125.3203125 +2024-06-04 03:37:37.010000,1126.4296875 +2024-06-04 03:37:39.020000,1126.13671875 +2024-06-04 03:37:40.029000,1126.3515625 +2024-06-04 03:37:41.029000,1126.3203125 +2024-06-04 03:37:42.078000,1126.84375 +2024-06-04 03:37:43.086000,1127.20703125 +2024-06-04 03:37:44.089000,1127.4453125 +2024-06-04 03:37:45.093000,1127.94921875 +2024-06-04 03:37:48.102000,1130.19921875 +2024-06-04 03:37:49.106000,1130.30859375 +2024-06-04 03:37:50.111000,1135.0 +2024-06-04 03:37:51.118000,1140.1171875 +2024-06-04 03:37:52.123000,1140.51953125 +2024-06-04 03:37:54.135000,1140.375 +2024-06-04 03:37:55.138000,1141.25390625 +2024-06-04 03:37:56.143000,1139.09375 +2024-06-04 03:37:57.149000,1140.0078125 +2024-06-04 03:37:58.152000,1144.21875 +2024-06-04 03:37:59.155000,1144.8671875 +2024-06-04 03:38:00.164000,1145.25390625 +2024-06-04 03:38:01.168000,1145.1640625 +2024-06-04 03:38:04.191000,1146.23828125 +2024-06-04 03:38:05.198000,1146.875 +2024-06-04 03:38:07.202000,1147.734375 +2024-06-04 03:38:09.212000,1147.59765625 +2024-06-04 03:38:10.216000,1147.1953125 +2024-06-04 03:38:16.255000,1150.5625 +2024-06-04 03:38:17.260000,1148.6796875 +2024-06-04 03:38:19.285000,1146.625 +2024-06-04 03:38:20.289000,1146.55078125 +2024-06-04 03:38:24.309000,1147.875 +2024-06-04 03:38:25.311000,1147.71875 +2024-06-04 03:38:26.317000,1147.58984375 +2024-06-04 03:38:29.341000,1147.875 +2024-06-04 03:38:32.352000,1148.92578125 +2024-06-04 03:38:34.360000,1153.609375 +2024-06-04 03:38:36.369000,1153.8046875 +2024-06-04 03:38:38.383000,1153.578125 +2024-06-04 03:38:39.389000,1154.625 +2024-06-04 03:38:41.394000,1154.28125 +2024-06-04 03:38:46.416000,1156.66015625 +2024-06-04 03:38:49.429000,1166.3828125 +2024-06-04 03:38:51.439000,1166.8671875 +2024-06-04 03:38:52.438000,1167.05859375 +2024-06-04 03:38:53.446000,1160.99609375 +2024-06-04 03:39:25.455000,1168.05078125 +2024-06-04 03:40:55.469000,1239.30859375 +2024-06-04 03:41:55.479000,1259.33203125 +2024-06-04 03:42:55.488000,1283.40625 +2024-06-04 03:43:25.496000,1284.46875 +2024-06-04 03:46:25.521000,1366.875 +2024-06-04 03:47:25.524000,1382.34375 +2024-06-04 03:47:55.534000,1372.3203125 +2024-06-04 03:49:25.541000,1444.30078125 +2024-06-04 03:50:25.546000,1449.98828125 +2024-06-04 03:50:55.551000,1466.87890625 +2024-06-04 03:51:55.557000,1483.0234375 +2024-06-04 03:52:55.563000,1513.37890625 +2024-06-04 03:53:25.566000,1512.1796875 +2024-06-04 03:54:25.576000,1515.0546875 +2024-06-04 03:54:55.578000,1530.1796875 +2024-06-04 03:55:55.593000,1552.4609375 +2024-06-04 03:56:55.602000,1554.2890625 +2024-06-04 03:57:25.603000,1551.7109375 +2024-06-04 03:57:55.605000,1572.1171875 +2024-06-04 03:58:25.605000,1562.6640625 +2024-06-04 03:58:55.606000,1589.87109375 +2024-06-04 03:59:25.609000,1593.6640625 +2024-06-04 04:01:25.623000,1618.2421875 +2024-06-04 04:01:55.630000,1628.71484375 +2024-06-04 04:02:25.632000,1638.32421875 +2024-06-04 04:02:55.637000,1632.52734375 +2024-06-04 04:03:25.640000,1623.203125 +2024-06-04 04:04:55.648000,1695.09375 +2024-06-04 04:05:55.653000,1734.11328125 +2024-06-04 04:06:55.658000,1756.484375 +2024-06-04 04:07:25.660000,1700.55078125 +2024-06-04 04:07:55.664000,1724.6171875 +2024-06-04 04:08:25.668000,1729.3984375 +2024-06-04 04:09:25.679000,1728.60546875 +2024-06-04 04:09:55.678000,1728.96875 +2024-06-04 04:10:55.684000,1768.9609375 +2024-06-04 04:11:25.689000,1773.13671875 +2024-06-04 04:11:55.691000,1784.29296875 +2024-06-04 04:12:55.693000,1509.0625 +2024-06-04 04:13:25.701000,1508.140625 +2024-06-04 04:13:55.702000,1508.68359375 +2024-06-04 04:14:25.705000,1504.7578125 +2024-06-04 04:14:55.709000,1491.7421875 +2024-06-04 04:15:55.714000,1508.61328125 +2024-06-04 04:16:55.722000,1501.8125 +2024-06-04 04:17:25.727000,1497.07421875 +2024-06-04 04:17:55.729000,1498.6484375 +2024-06-04 04:18:25.732000,1496.65234375 +2024-06-04 04:18:55.737000,1499.8125 +2024-06-04 04:19:25.737000,1515.08203125 +2024-06-04 04:19:55.744000,1512.96875 +2024-06-04 04:20:25.744000,1511.6015625 +2024-06-04 04:21:25.748000,1513.9921875 +2024-06-04 04:21:55.753000,1521.23046875 +2024-06-04 04:22:25.754000,1522.34375 +2024-06-04 04:24:25.772000,1524.765625 +2024-06-04 04:24:55.773000,1514.16796875 +2024-06-04 04:25:25.776000,1508.60546875 +2024-06-04 04:25:55.780000,1505.30859375 +2024-06-04 04:26:25.782000,1510.6328125 +2024-06-04 04:27:25.789000,1496.74609375 +2024-06-04 04:27:55.793000,1498.75 +2024-06-04 04:28:25.794000,1508.09375 +2024-06-04 04:29:25.797000,1517.16015625 +2024-06-04 04:29:55.803000,1510.99609375 +2024-06-04 04:30:25.804000,1522.984375 +2024-06-04 04:31:25.808000,1523.3515625 +2024-06-04 04:31:55.811000,1527.33203125 +2024-06-04 04:32:25.811000,1529.6015625 +2024-06-04 04:33:25.821000,1532.078125 +2024-06-04 04:33:55.824000,1531.3671875 +2024-06-04 04:34:55.838000,1514.1953125 +2024-06-04 04:35:25.840000,1522.14453125 +2024-06-04 04:39:34.984000,1527.59375 +2024-06-04 04:40:34.991000,1540.796875 +2024-06-04 04:41:04.992000,1538.796875 +2024-06-04 04:41:34.993000,1545.36328125 +2024-06-04 04:42:35.004000,1532.55078125 +2024-06-04 04:43:05.009000,1533.05859375 +2024-06-04 04:43:35.016000,1535.55859375 +2024-06-04 04:44:05.020000,1535.04296875 +2024-06-04 04:44:35.024000,1536.75 +2024-06-04 04:45:35.030000,1541.0234375 +2024-06-04 04:46:05.030000,1529.6484375 +2024-06-04 04:47:05.047000,1543.23828125 +2024-06-04 04:47:35.050000,1544.4140625 +2024-06-04 04:48:05.052000,1534.34375 +2024-06-04 04:49:05.056000,1531.3984375 +2024-06-04 04:49:35.060000,1532.1796875 +2024-06-04 04:50:05.064000,1537.39453125 +2024-06-04 04:50:35.068000,1534.8828125 +2024-06-04 04:51:05.071000,1534.60546875 diff --git a/analysis/output_10b_run3_40s_flush_2gb_memory_limit.txt b/analysis/output_10b_run3_40s_flush_2gb_memory_limit.txt new file mode 100644 index 0000000..cd8d1a3 --- /dev/null +++ b/analysis/output_10b_run3_40s_flush_2gb_memory_limit.txt @@ -0,0 +1,409 @@ +worker1-1 | 2024/06/04 05:02:07.015 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 05:02:07.040 FIG10b: Starting tasks submission... +worker1-1 | 2024/06/04 05:02:08.045 FIG10b: Memory usage without flush after 99 tasks: 1046.09765625 MB +worker1-1 | 2024/06/04 05:02:09.053 FIG10b: Memory usage without flush after 190 tasks: 1047.3203125 MB +worker1-1 | 2024/06/04 05:02:10.057 FIG10b: Memory usage without flush after 293 tasks: 1038.96484375 MB +worker1-1 | 2024/06/04 05:02:11.060 FIG10b: Memory usage without flush after 409 tasks: 1044.33203125 MB +worker1-1 | 2024/06/04 05:02:12.069 FIG10b: Memory usage without flush after 526 tasks: 1049.59375 MB +worker1-1 | 2024/06/04 05:02:13.069 FIG10b: Memory usage without flush after 639 tasks: 1055.1796875 MB +worker1-1 | 2024/06/04 05:02:14.072 2024/06/04 05:02:14.074[worker] FIG10b: Memory usage without flush after 753 tasks: 1056.06640625 MB +worker1-1 | 2024/06/04 05:02:15.084 FIG10b: Memory usage without flush after 865 tasks: 1055.5078125 MB +worker1-1 | 2024/06/04 05:02:16.088 FIG10b: Memory usage without flush after 990 tasks: 1055.58203125 MB +worker1-1 | 2024/06/04 05:02:17.094[worker] FIG10b: Memory usage without flush after 1077 tasks: 1056.84375 MB +worker1-1 | 2024/06/04 05:02:18.099 FIG10b: Memory usage without flush after 1170 tasks: 1057.578125 MB +worker1-1 | 2024/06/04 05:02:19.109 FIG10b: Memory usage without flush after 1279 tasks: 1057.734375 MB +worker1-1 | 2024/06/04 05:02:20.128 FIG10b: Memory usage without flush after 1372 tasks: 1059.52734375 MB +worker1-1 | 2024/06/04 05:02:21.134 FIG10b: Memory usage without flush after 1459 tasks: 1060.06640625 MB +worker1-1 | [worker] stored... [worker]2024/06/04 05:02:22.146 FIG10b: Memory usage without flush after 1537 tasks: 1060.64453125 MB +worker1-1 | 2024/06/04 05:02:23.153 FIG10b: Memory usage without flush after 1630 tasks: 1061.59375 MB +worker1-1 | 2024/06/04 05:02:24.1552024/06/04 05:02:24.155 FIG10b: Memory usage without flush after 1750 tasks: 1064.69140625 MB +worker1-1 | 2024/06/04 05:02:25.161 FIG10b: Memory usage without flush after 1826 tasks: 1065.05078125 MB +worker1-1 | 2024/06/04 05:02:26.171 FIG10b: Memory usage without flush after 1875 tasks: 1065.30078125 MB +worker1-1 | 2024/06/04 05:02:27.474 FIG10b: Memory usage without flush after 1898 tasks: 1065.41015625 MB +worker1-1 | 2024/06/04 05:02:28.481 FIG10b: Memory usage without flush after 1927 tasks: 1069.33984375 MB +worker1-1 | 2024/06/04 05:02:29.515 FIG10b: Memory usage without flush after 1980 tasks: 1069.31640625 MB +worker1-1 | 2024/06/04 05:02:30.520 FIG10b: Memory usage without flush after 2041 tasks: 1073.29296875 MB +worker1-1 | 2024/06/04 05:02:31.5212024/06/04 05:02:31.521 FIG10b: Memory usage without flush after 2110 tasks: 1073.7578125 MB +worker1-1 | 2024/06/04 05:02:32.522 FIG10b: Memory usage without flush after 2198 tasks: 1074.93359375 MB +worker1-1 | 2024/06/04 05:02:33.5282024/06/04 05:02:33.528 FIG10b: Memory usage without flush after 2305 tasks: 1075.1875 MB +worker1-1 | 2024/06/04 05:02:34.5412024/06/04 05:02:34.541 FIG10b: Memory usage without flush after 2407 tasks: 1075.49609375 MB +worker1-1 | 2024/06/04 05:02:35.543 FIG10b: Memory usage without flush after 2488 tasks: 1074.82421875 MB +worker1-1 | 2024/06/04 05:02:36.5432024/06/04 05:02:36.543 FIG10b: Memory usage without flush after 2588 tasks: 1076.00390625 MB +worker1-1 | 2024/06/04 05:02:37.547 [worker]2024/06/04 05:02:37.548 FIG10b: Memory usage without flush after 2697 tasks: 1076.19140625 MB +worker1-1 | 2024/06/04 05:02:38.554 FIG10b: Memory usage without flush after 2803 tasks: 1076.67578125 MB +worker1-1 | 2024/06/04 05:02:39.556 FIG10b: Memory usage without flush after 2894 tasks: 1076.46875 MB +worker1-1 | 2024/06/04 05:02:40.638 FIG10b: Memory usage without flush after 2995 tasks: 1076.50390625 MB +worker1-1 | 2024/06/04 05:02:41.666 FIG10b: Memory usage without flush after 3080 tasks: 1086.84765625 MB +worker1-1 | 2024/06/04 05:02:42.665 [worker]2024/06/04 05:02:42.666 FIG10b: Memory usage without flush after 3178 tasks: 1085.69921875 MB +worker1-1 | 2024/06/04 05:02:43.666 FIG10b: Memory usage without flush after 3285 tasks: 1086.4140625 MB +worker1-1 | 2024/06/04 05:02:44.674 FIG10b: Memory usage without flush after 3403 tasks: 1088.109375 MB +worker1-1 | FIG10b: Memory usage without flush after 3527 tasks: 1089.0546875 MB +worker1-1 | 2024/06/04 05:02:46.683 2024/06/04 05:02:46.6842024/06/04 05:02:46.685 FIG10b: Memory usage without flush after 3649 tasks: 1089.08984375 MB +worker1-1 | [worker]stored...2024/06/04 05:02:47.688 FIG10b: Memory usage without flush after 3754 tasks: 1089.1015625 MB +worker1-1 | 2024/06/04 05:02:48.696 FIG10b: Memory usage without flush after 3872 tasks: 1088.625 MB +worker1-1 | 2024/06/04 05:02:49.701 [worker] 2024/06/04 05:02:49.703in Run() rn FIG10b: Memory usage without flush after 3978 tasks: 1089.31640625 MB +worker1-1 | 2024/06/04 05:02:50.7062024/06/04 05:02:50.706 FIG10b: Memory usage without flush after 4084 tasks: 1091.57421875 MB +worker1-1 | 2024/06/04 05:02:51.7132024/06/04 05:02:51.714 2024/06/04 05:02:51.714 FIG10b: Memory usage without flush after 4204 tasks: 1091.6328125 MB +worker1-1 | 2024/06/04 05:02:52.716 FIG10b: Memory usage without flush after 4326 tasks: 1091.8828125 MB +worker1-1 | 2024/06/04 05:02:53.7242024/06/04 05:02:53.725 FIG10b: Memory usage without flush after 4431 tasks: 1093.078125 MB +worker1-1 | 2024/06/04 05:02:54.736 FIG10b: Memory usage without flush after 4542 tasks: 1093.75 MB +worker1-1 | 2024/06/04 05:02:55.7402024/06/04 05:02:55.740 FIG10b: Memory usage without flush after 4644 tasks: 1093.98046875 MB +worker1-1 | [worker][worker]2024/06/04 05:02:56.748 FIG10b: Memory usage without flush after 4748 tasks: 1093.734375 MB +worker1-1 | 2024/06/04 05:02:57.758 2024/06/04 05:02:57.759 FIG10b: Memory usage without flush after 4864 tasks: 1093.1484375 MB +worker1-1 | 2024/06/04 05:02:58.760 FIG10b: Memory usage without flush after 4953 tasks: 1093.91015625 MB +worker1-1 | 2024/06/04 05:02:59.763 FIG10b: Memory usage without flush after 5056 tasks: 1093.84765625 MB +worker1-1 | [worker] 2024/06/04 05:03:00.771 FIG10b: Memory usage without flush after 5156 tasks: 1094.3125 MB +worker1-1 | 2024/06/04 05:03:01.777 FIG10b: Memory usage without flush after 5251 tasks: 1094.30859375 MB +worker1-1 | FIG10b: Memory usage without flush after 5359 tasks: 1094.29296875 MB +worker1-1 | 2024/06/04 05:03:03.784 FIG10b: Memory usage without flush after 5474 tasks: 1094.41015625 MB +worker1-1 | FIG10b: Memory usage without flush after 5562 tasks: 1094.703125 MB +worker1-1 | FIG10b: Memory usage without flush after 5656 tasks: 1094.7265625 MB +worker1-1 | 2024/06/04 05:03:06.818 FIG10b: Memory usage without flush after 5745 tasks: 1094.76171875 MB +worker1-1 | 2024/06/04 05:03:07.8252024/06/04 05:03:07.825 FIG10b: Memory usage without flush after 5837 tasks: 1095.03515625 MB +worker1-1 | 2024/06/04 05:03:08.8392024/06/04 05:03:08.840 FIG10b: Memory usage without flush after 5936 tasks: 1095.2578125 MB +worker1-1 | 2024/06/04 05:03:09.845 FIG10b: Memory usage without flush after 6051 tasks: 1094.91796875 MB +worker1-1 | 2024/06/04 05:03:10.847[worker]2024/06/04 05:03:10.847 2024/06/04 05:03:10.848 FIG10b: Memory usage without flush after 6170 tasks: 1095.015625 MB +worker1-1 | 2024/06/04 05:03:11.848 FIG10b: Memory usage without flush after 6276 tasks: 1095.92578125 MB +worker1-1 | 2024/06/04 05:03:12.860 FIG10b: Memory usage without flush after 6383 tasks: 1095.65234375 MB +worker1-1 | FIG10b: Memory usage without flush after 6506 tasks: 1096.24609375 MB +worker1-1 | 2024/06/04 05:03:14.872 FIG10b: Memory usage without flush after 6633 tasks: 1097.30078125 MB +worker1-1 | 2024/06/04 05:03:16.0052024/06/04 05:03:16.004 FIG10b: Memory usage without flush after 6730 tasks: 1097.03515625 MB[worker] +worker1-1 | 2024/06/04 05:03:17.012 FIG10b: Memory usage without flush after 6837 tasks: 1096.73046875 MB +worker1-1 | 2024/06/04 05:03:18.0152024/06/04 05:03:18.015 FIG10b: Memory usage without flush after 6946 tasks: 1097.6484375 MB +worker1-1 | 2024/06/04 05:03:19.022 FIG10b: Memory usage without flush after 7071 tasks: 1097.453125 MB +worker1-1 | 2024/06/04 05:03:20.023 FIG10b: Memory usage without flush after 7183 tasks: 1097.36328125 MB +worker1-1 | 2024/06/04 05:03:21.0282024/06/04 05:03:21.028 FIG10b: Memory usage without flush after 7279 tasks: 1098.08203125 MB +worker1-1 | [worker]2024/06/04 05:03:22.032 FIG10b: Memory usage without flush after 7411 tasks: 1117.26171875 MB +worker1-1 | 2024/06/04 05:03:23.0332024/06/04 05:03:23.033 FIG10b: Memory usage without flush after 7514 tasks: 1117.23046875 MB +worker1-1 | 2024/06/04 05:03:24.034 FIG10b: Memory usage without flush after 7632 tasks: 1117.375 MB +worker1-1 | 2024/06/04 05:03:25.037 2024/06/04 05:03:25.038 FIG10b: Memory usage without flush after 7750 tasks: 1117.7421875 MB +worker1-1 | 2024/06/04 05:03:26.043[worker]2024/06/04 05:03:26.044 FIG10b: Memory usage without flush after 7862 tasks: 1118.1484375 MB +worker1-1 | 2024/06/04 05:03:27.047 2024/06/04 05:03:27.047 FIG10b: Memory usage without flush after 7975 tasks: 1118.29296875 MB +worker1-1 | 2024/06/04 05:03:28.0492024/06/04 05:03:28.049 FIG10b: Memory usage without flush after 8089 tasks: 1119.4453125 MB +worker1-1 | 2024/06/04 05:03:29.054 [worker]2024/06/04 05:03:29.055 FIG10b: Memory usage without flush after 8201 tasks: 1120.78515625 MB +worker1-1 | 2024/06/04 05:03:30.060 FIG10b: Memory usage without flush after 8325 tasks: 1121.453125 MB +worker1-1 | 2024/06/04 05:03:31.064 2024/06/04 05:03:31.065 FIG10b: Memory usage without flush after 8444 tasks: 1122.74609375 MB +worker1-1 | 2024/06/04 05:03:32.067 FIG10b: Memory usage without flush after 8566 tasks: 1122.484375 MB +worker1-1 | 2024/06/04 05:03:33.071 FIG10b: Memory usage without flush after 8689 tasks: 1122.0546875 MB +worker1-1 | 2024/06/04 05:03:34.074 FIG10b: Memory usage without flush after 8801 tasks: 1122.00390625 MB +worker1-1 | 2024/06/04 05:03:35.076 FIG10b: Memory usage without flush after 8906 tasks: 1121.9609375 MB +worker1-1 | 2024/06/04 05:03:36.080 FIG10b: Memory usage without flush after 9026 tasks: 1121.7421875 MB +worker1-1 | 2024/06/04 05:03:37.149 FIG10b: Memory usage without flush after 9151 tasks: 1121.2734375 MB +worker1-1 | 2024/06/04 05:03:38.155 FIG10b: Memory usage without flush after 9280 tasks: 1121.265625 MB +worker1-1 | 2024/06/04 05:03:39.157 FIG10b: Memory usage without flush after 9412 tasks: 1122.70703125 MB +worker1-1 | 2024/06/04 05:03:40.157 FIG10b: Memory usage without flush after 9539 tasks: 1123.43359375 MB +worker1-1 | 2024/06/04 05:03:41.160 FIG10b: Memory usage without flush after 9663 tasks: 1123.70703125 MB +worker1-1 | 2024/06/04 05:03:42.163 [worker]2024/06/04 05:03:42.1642024/06/04 05:03:42.164 FIG10b: Memory usage without flush after 9792 tasks: 1124.19140625 MB +worker1-1 | 2024/06/04 05:03:43.1652024/06/04 05:03:43.165 FIG10b: Memory usage without flush after 9907 tasks: 1124.6171875 MB +worker1-1 | 2024/06/04 05:03:44.166 FIG10b: Memory usage without flush after 10010 tasks: 1124.14453125 MB +worker1-1 | 2024/06/04 05:03:45.178 FIG10b: Memory usage without flush after 10107 tasks: 1124.15234375 MB +worker1-1 | 2024/06/04 05:03:46.259 FIG10b: Memory usage without flush after 10209 tasks: 1123.47265625 MB +worker1-1 | 2024/06/04 05:03:47.2602024/06/04 05:03:47.261 FIG10b: Memory usage without flush after 10313 tasks: 1124.1796875 MB +worker1-1 | 2024/06/04 05:03:48.2692024/06/04 05:03:48.269 FIG10b: Memory usage without flush after 10422 tasks: 1124.65234375 MB +worker1-1 | 2024/06/04 05:03:49.2742024/06/04 05:03:49.275 FIG10b: Memory usage without flush after 10532 tasks: 1124.453125 MB +worker1-1 | 2024/06/04 05:03:50.276 FIG10b: Memory usage without flush after 10644 tasks: 1124.7265625 MB +worker1-1 | [worker]2024/06/04 05:03:51.2762024/06/04 05:03:51.277 FIG10b: Memory usage without flush after 10752 tasks: 1124.53125 MB +worker1-1 | Executed!2024/06/04 05:03:52.283 FIG10b: Memory usage without flush after 10852 tasks: 1124.94140625 MB +worker1-1 | [worker]2024/06/04 05:03:53.285 FIG10b: Memory usage without flush after 10968 tasks: 1125.73046875 MB +worker1-1 | 2024/06/04 05:03:54.2862024/06/04 05:03:54.286 FIG10b: Memory usage without flush after 11067 tasks: 1125.9921875 MB +worker1-1 | 2024/06/04 05:03:55.288 FIG10b: Memory usage without flush after 11169 tasks: 1125.7421875 MB +worker1-1 | 2024/06/04 05:03:56.298 FIG10b: Memory usage without flush after 11253 tasks: 1125.046875 MB +worker1-1 | 2024/06/04 05:03:57.299 FIG10b: Memory usage without flush after 11304 tasks: 1125.49609375 MB +worker1-1 | 2024/06/04 05:03:58.322 [worker]2024/06/04 05:03:58.328 FIG10b: Memory usage without flush after 11363 tasks: 1125.99609375 MB +worker1-1 | FIG10b: Memory usage without flush after 11426 tasks: 1124.56640625 MB +worker1-1 | [worker]2024/06/04 05:04:00.342 FIG10b: Memory usage without flush after 11546 tasks: 1124.44921875 MB +worker1-1 | 2024/06/04 05:04:01.344 FIG10b: Memory usage without flush after 11652 tasks: 1126.3515625 MB +worker1-1 | 2024/06/04 05:04:02.353 FIG10b: Memory usage without flush after 11767 tasks: 1135.16796875 MB +worker1-1 | 2024/06/04 05:04:03.373 FIG10b: Memory usage without flush after 11871 tasks: 1135.34765625 MB +worker1-1 | 2024/06/04 05:04:04.377 FIG10b: Memory usage without flush after 11965 tasks: 1135.10546875 MB +worker1-1 | 2024/06/04 05:04:05.3772024/06/04 05:04:05.377 FIG10b: Memory usage without flush after 12078 tasks: 1135.76171875 MB +worker1-1 | 2024/06/04 05:04:06.3822024/06/04 05:04:06.383 FIG10b: Memory usage without flush after 12183 tasks: 1135.49609375 MB +worker1-1 | [worker] 2024/06/04 05:04:07.390 FIG10b: Memory usage without flush after 12289 tasks: 1135.65234375 MB +worker1-1 | 2024/06/04 05:04:08.396 FIG10b: Memory usage without flush after 12388 tasks: 1135.859375 MB +worker1-1 | 2024/06/04 05:04:09.403 FIG10b: Memory usage without flush after 12471 tasks: 1135.97265625 MB2024/06/04 05:04:09.403 +worker1-1 | 2024/06/04 05:04:10.414 FIG10b: Memory usage without flush after 12574 tasks: 1135.953125 MB +worker1-1 | 2024/06/04 05:04:11.427 FIG10b: Memory usage without flush after 12665 tasks: 1135.75 MB +worker1-1 | 2024/06/04 05:04:12.431 FIG10b: Memory usage without flush after 12770 tasks: 1135.921875 MB +worker1-1 | 2024/06/04 05:04:13.4342024/06/04 05:04:13.434 FIG10b: Memory usage without flush after 12866 tasks: 1136.171875 MB +worker1-1 | 2024/06/04 05:04:14.446 FIG10b: Memory usage without flush after 12952 tasks: 1136.7109375 MB +worker1-1 | stored...[worker]2024/06/04 05:04:15.4562024/06/04 05:04:15.456 FIG10b: Memory usage without flush after 13074 tasks: 1137.08203125 MB +worker1-1 | 2024/06/04 05:04:16.462 FIG10b: Memory usage without flush after 13192 tasks: 1137.453125 MB +worker1-1 | 2024/06/04 05:04:17.468 FIG10b: Memory usage without flush after 13316 tasks: 1138.04296875 MB +worker1-1 | FIG10b: Memory usage without flush after 13439 tasks: 1138.6640625 MB +worker1-1 | 2024/06/04 05:04:19.478 FIG10b: Memory usage without flush after 13561 tasks: 1139.48046875 MB +worker1-1 | 2024/06/04 05:04:20.482 FIG10b: Memory usage without flush after 13680 tasks: 1140.09375 MB +worker1-1 | 2024/06/04 05:04:21.486 FIG10b: Memory usage without flush after 13800 tasks: 1140.7265625 MB +worker1-1 | 2024/06/04 05:04:22.494 FIG10b: Memory usage without flush after 13921 tasks: 1141.3671875 MB +worker1-1 | 2024/06/04 05:04:23.495 FIG10b: Memory usage without flush after 14046 tasks: 1141.96484375 MB +worker1-1 | 2024/06/04 05:04:24.5012024/06/04 05:04:24.502 [worker] 2024/06/04 05:04:24.502 FIG10b: Memory usage without flush after 14166 tasks: 1141.828125 MB +worker1-1 | 2024/06/04 05:04:25.509 FIG10b: Memory usage without flush after 14279 tasks: 1142.01171875 MB +worker1-1 | 2024/06/04 05:04:26.512 2024/06/04 05:04:26.5132024/06/04 05:04:26.513 FIG10b: Memory usage without flush after 14396 tasks: 1141.9140625 MB +worker1-1 | 2024/06/04 05:04:27.515 2024/06/04 05:04:27.515[worker] 2024/06/04 05:04:27.516 FIG10b: Memory usage without flush after 14515 tasks: 1142.26953125 MB +worker1-1 | 2024/06/04 05:04:28.522 FIG10b: Memory usage without flush after 14639 tasks: 1142.375 MB +worker1-1 | took this many seconds: 0.032024/06/04 05:04:29.524 FIG10b: Memory usage without flush after 14744 tasks: 1142.41015625 MB +worker1-1 | 2024/06/04 05:04:30.531 2024/06/04 05:04:30.531 FIG10b: Memory usage without flush after 14851 tasks: 1141.79296875 MB +worker1-1 | 2024/06/04 05:04:31.535 FIG10b: Memory usage without flush after 14954 tasks: 1142.7734375 MB +worker1-1 | 2024/06/04 05:04:32.537 FIG10b: Memory usage without flush after 15067 tasks: 1142.21875 MB +worker1-1 | [worker] 2024/06/04 05:04:33.539 FIG10b: Memory usage without flush after 15183 tasks: 1142.64453125 MB +worker1-1 | 2024/06/04 05:04:34.543 FIG10b: Memory usage without flush after 15294 tasks: 1142.89453125 MB +worker1-1 | 2024/06/04 05:04:35.5472024/06/04 05:04:35.547 FIG10b: Memory usage without flush after 15416 tasks: 1142.06640625 MB +worker1-1 | 2024/06/04 05:04:36.548 FIG10b: Memory usage without flush after 15534 tasks: 1142.44921875 MB +worker1-1 | [worker] 2024/06/04 05:04:37.554 FIG10b: Memory usage without flush after 15646 tasks: 1142.3671875 MB +worker1-1 | 2024/06/04 05:04:38.574 FIG10b: Memory usage without flush after 15755 tasks: 1141.66015625 MB +worker1-1 | 2024/06/04 05:04:39.580 [worker] 2024/06/04 05:04:39.581 FIG10b: Memory usage without flush after 15883 tasks: 1141.34375 MB +worker1-1 | 2024/06/04 05:04:40.5812024/06/04 05:04:40.581 FIG10b: Memory usage without flush after 16006 tasks: 1141.25 MB +worker1-1 | 2024/06/04 05:04:41.589 FIG10b: Memory usage without flush after 16131 tasks: 1149.65234375 MB +worker1-1 | 2024/06/04 05:04:42.597 FIG10b: Memory usage without flush after 16251 tasks: 1161.16796875 MB +worker1-1 | 2024/06/04 05:04:43.598 FIG10b: Memory usage without flush after 16373 tasks: 1156.609375 MB +worker1-1 | 2024/06/04 05:04:44.606 FIG10b: Memory usage without flush after 16496 tasks: 1154.74609375 MB +worker1-1 | 2024/06/04 05:04:45.608in Run() rn FIG10b: Memory usage without flush after 16628 tasks: 1155.5703125 MB +worker1-1 | 2024/06/04 05:04:46.621 FIG10b: Memory usage without flush after 16747 tasks: 1155.28125 MB +worker1-1 | 2024/06/04 05:04:47.6242024/06/04 05:04:47.625 FIG10b: Memory usage without flush after 16862 tasks: 1154.99609375 MB +worker1-1 | 2024/06/04 05:04:48.628 FIG10b: Memory usage without flush after 16973 tasks: 1155.8046875 MB +worker1-1 | [worker]took this many seconds: 0.0156[worker]2024/06/04 05:04:49.635 FIG10b: Memory usage without flush after 17101 tasks: 1155.41796875 MB +worker1-1 | 2024/06/04 05:04:50.636 FIG10b: Memory usage without flush after 17219 tasks: 1155.15234375 MB +worker1-1 | 2024/06/04 05:04:51.639 FIG10b: Memory usage without flush after 17346 tasks: 1155.5078125 MB +worker1-1 | stored...2024/06/04 05:04:52.648 FIG10b: Memory usage without flush after 17458 tasks: 1155.9765625 MB +worker1-1 | 2024/06/04 05:04:53.6532024/06/04 05:04:53.6532024/06/04 05:04:53.653 FIG10b: Memory usage without flush after 17579 tasks: 1157.46484375 MB +worker1-1 | 2024/06/04 05:04:54.656 FIG10b: Memory usage without flush after 17688 tasks: 1157.9921875 MB +worker1-1 | 2024/06/04 05:04:55.656 FIG10b: Memory usage without flush after 17801 tasks: 1158.453125 MB +worker1-1 | 2024/06/04 05:04:56.6582024/06/04 05:04:56.658 FIG10b: Memory usage without flush after 17924 tasks: 1155.96484375 MB +worker1-1 | 2024/06/04 05:04:57.659 FIG10b: Memory usage without flush after 18047 tasks: 1155.86328125 MB +worker1-1 | 2024/06/04 05:04:58.667 FIG10b: Memory usage without flush after 18160 tasks: 1156.83984375 MB +worker1-1 | 2024/06/04 05:04:59.672 2024/06/04 05:04:59.672[worker] FIG10b: Memory usage without flush after 18286 tasks: 1156.71484375 MB +worker1-1 | 2024/06/04 05:05:00.678 FIG10b: Memory usage without flush after 18409 tasks: 1155.984375 MB +worker1-1 | 2024/06/04 05:05:01.680 FIG10b: Memory usage without flush after 18534 tasks: 1156.67578125 MB +worker1-1 | 2024/06/04 05:05:02.6912024/06/04 05:05:02.690 FIG10b: Memory usage without flush after 18660 tasks: 1156.65234375 MB +worker1-1 | 2024/06/04 05:05:03.6962024/06/04 05:05:03.6962024/06/04 05:05:03.696 FIG10b: Memory usage without flush after 18792 tasks: 1156.6484375 MB +worker1-1 | 2024/06/04 05:05:04.6992024/06/04 05:05:04.699 FIG10b: Memory usage without flush after 18909 tasks: 1156.46875 MB +worker1-1 | 2024/06/04 05:05:05.705 FIG10b: Memory usage without flush after 19029 tasks: 1156.47265625 MB +worker1-1 | 2024/06/04 05:05:06.709 FIG10b: Memory usage without flush after 19155 tasks: 1157.63671875 MB +worker1-1 | 2024/06/04 05:05:07.718 FIG10b: Memory usage without flush after 19277 tasks: 1158.078125 MB +worker1-1 | 2024/06/04 05:05:08.724 FIG10b: Memory usage without flush after 19402 tasks: 1157.0 MB +worker1-1 | [worker]2024/06/04 05:05:09.727 FIG10b: Memory usage without flush after 19524 tasks: 1157.7265625 MB +worker1-1 | FIG10b: Memory usage without flush after 19631 tasks: 1157.71875 MB +worker1-1 | 2024/06/04 05:05:11.737 FIG10b: Memory usage without flush after 19751 tasks: 1158.40234375 MB +worker1-1 | 2024/06/04 05:05:12.745 FIG10b: Memory usage without flush after 19849 tasks: 1158.05078125 MB +worker1-1 | 2024/06/04 05:05:13.746 FIG10b: Memory usage without flush after 19952 tasks: 1157.85546875 MB +worker1-1 | 2024/06/04 05:05:14.750 FIG10b: Memory usage without flush after 20055 tasks: 1158.12109375 MB +worker1-1 | 2024/06/04 05:05:15.7542024/06/04 05:05:15.755 FIG10b: Memory usage without flush after 20168 tasks: 1158.3203125 MB +worker1-1 | 2024/06/04 05:05:16.761 FIG10b: Memory usage without flush after 20294 tasks: 1158.09765625 MB +worker1-1 | 2024/06/04 05:05:17.768 FIG10b: Memory usage without flush after 20417 tasks: 1158.53515625 MB +worker1-1 | 2024/06/04 05:05:18.772 FIG10b: Memory usage without flush after 20544 tasks: 1159.5078125 MB +worker1-1 | 2024/06/04 05:05:19.778[worker] took this many seconds: 0.03232024/06/04 05:05:19.779 FIG10b: Memory usage without flush after 20663 tasks: 1159.77734375 MB +worker1-1 | 2024/06/04 05:05:20.787 FIG10b: Memory usage without flush after 20786 tasks: 1159.3125 MB +worker1-1 | 2024/06/04 05:05:21.794 FIG10b: Memory usage without flush after 20935 tasks: 1175.1171875 MB +worker1-1 | 2024/06/04 05:05:22.798 FIG10b: Memory usage without flush after 21040 tasks: 1175.8046875 MB +worker1-1 | 2024/06/04 05:05:23.799 FIG10b: Memory usage without flush after 21169 tasks: 1176.72265625 MB2024/06/04 05:05:23.800 [lobs] WERE GONNA CREATE A /DEV FILE WITH THIS NAME: /dev/shm/5211556889717931035 +worker1-1 | 2024/06/04 05:05:24.8032024/06/04 05:05:24.803 FIG10b: Memory usage without flush after 21294 tasks: 1176.23828125 MB +worker1-1 | 2024/06/04 05:05:25.8052024/06/04 05:05:25.805 FIG10b: Memory usage without flush after 21424 tasks: 1176.46484375 MB +worker1-1 | 2024/06/04 05:05:26.813 FIG10b: Memory usage without flush after 21550 tasks: 1177.3515625 MB +worker1-1 | 2024/06/04 05:05:27.822 FIG10b: Memory usage without flush after 21675 tasks: 1176.828125 MB +worker1-1 | took this many seconds: 0.0182024/06/04 05:05:28.825 FIG10b: Memory usage without flush after 21788 tasks: 1177.84375 MB +worker1-1 | 2024/06/04 05:05:29.8292024/06/04 05:05:29.829 FIG10b: Memory usage without flush after 21922 tasks: 1176.84765625 MB +worker1-1 | 2024/06/04 05:05:30.8312024/06/04 05:05:30.832 FIG10b: Memory usage without flush after 22047 tasks: 1177.5546875 MB +worker1-1 | FIG10b: Memory usage without flush after 22167 tasks: 1178.4609375 MB +worker1-1 | FIG10b: Memory usage without flush after 22286 tasks: 1179.0546875 MB +worker1-1 | 2024/06/04 05:05:33.8442024/06/04 05:05:33.844 FIG10b: Memory usage without flush after 22407 tasks: 1180.11328125 MB +worker1-1 | 2024/06/04 05:05:34.909 FIG10b: Memory usage without flush after 22510 tasks: 1179.80078125 MB +worker1-1 | 2024/06/04 05:05:35.916 FIG10b: Memory usage without flush after 22634 tasks: 1179.89453125 MB +worker1-1 | 2024/06/04 05:05:36.925 FIG10b: Memory usage without flush after 22761 tasks: 1179.71484375 MB +worker1-1 | 2024/06/04 05:05:37.930 FIG10b: Memory usage without flush after 22880 tasks: 1184.296875 MB +worker1-1 | [worker]2024/06/04 05:05:38.931 gonna store nowFIG10b: Memory usage without flush after 22981 tasks: 1184.23828125 MB +worker1-1 | 2024/06/04 05:05:39.9372024/06/04 05:05:39.937 FIG10b: Memory usage without flush after 23091 tasks: 1183.90234375 MB +worker1-1 | 2024/06/04 05:05:40.940 FIG10b: Memory usage without flush after 23213 tasks: 1184.53515625 MB +worker1-1 | 2024/06/04 05:05:41.953 FIG10b: Memory usage without flush after 23340 tasks: 1184.80859375 MB +worker1-1 | 2024/06/04 05:05:42.958 FIG10b: Memory usage without flush after 23449 tasks: 1184.7421875 MB +worker1-1 | 2024/06/04 05:05:43.9612024/06/04 05:05:43.961 FIG10b: Memory usage without flush after 23565 tasks: 1185.1875 MB +worker1-1 | 2024/06/04 05:05:44.960 [worker]2024/06/04 05:05:44.961 FIG10b: Memory usage without flush after 23679 tasks: 1185.80859375 MB +worker1-1 | 2024/06/04 05:05:45.966 FIG10b: Memory usage without flush after 23788 tasks: 1185.24609375 MB +worker1-1 | 2024/06/04 05:05:46.9702024/06/04 05:05:46.971 [worker] took this many seconds: 0.00822FIG10b: Memory usage without flush after 23898 tasks: 1185.58203125 MB +worker1-1 | FIG10b: Memory usage without flush after 23998 tasks: 1185.28125 MB +worker1-1 | 2024/06/04 05:05:48.973 2024/06/04 05:05:48.973 FIG10b: Memory usage without flush after 24109 tasks: 1185.68359375 MB +worker1-1 | 2024/06/04 05:05:49.9782024/06/04 05:05:49.978 FIG10b: Memory usage without flush after 24218 tasks: 1186.0859375 MB +worker1-1 | 2024/06/04 05:05:50.982 FIG10b: Memory usage without flush after 24335 tasks: 1186.41796875 MB +worker1-1 | 2024/06/04 05:05:51.989stored...2024/06/04 05:05:51.989 FIG10b: Memory usage without flush after 24459 tasks: 1186.80859375 MB +worker1-1 | 2024/06/04 05:05:52.993 FIG10b: Memory usage without flush after 24567 tasks: 1187.203125 MB +worker1-1 | 2024/06/04 05:05:53.996 FIG10b: Memory usage without flush after 24692 tasks: 1187.26953125 MB +worker1-1 | [worker] 2024/06/04 05:05:55.002 FIG10b: Memory usage without flush after 24804 tasks: 1187.609375 MB +worker1-1 | 2024/06/04 05:05:56.005 FIG10b: Memory usage without flush after 24912 tasks: 1187.33984375 MB +worker1-1 | 2024/06/04 05:05:57.009 FIG10b: Memory usage without flush after 25042 tasks: 1187.41015625 MB +worker1-1 | 2024/06/04 05:05:58.0162024/06/04 05:05:58.017 FIG10b: Memory usage without flush after 25134 tasks: 1188.08203125 MB +worker1-1 | FIG10b: Memory usage without flush after 25256 tasks: 1187.7578125 MB +worker1-1 | 2024/06/04 05:06:00.023 FIG10b: Memory usage without flush after 25373 tasks: 1187.66015625 MB +worker1-1 | 2024/06/04 05:06:01.034 FIG10b: Memory usage without flush after 25480 tasks: 1188.265625 MB +worker1-1 | 2024/06/04 05:06:02.053 FIG10b: Memory usage without flush after 25595 tasks: 1195.21484375 MB +worker1-1 | 2024/06/04 05:06:03.057 2024/06/04 05:06:03.058 FIG10b: Memory usage without flush after 25707 tasks: 1195.9140625 MB +worker1-1 | Executed![worker]2024/06/04 05:06:04.0622024/06/04 05:06:04.062 FIG10b: Memory usage without flush after 25821 tasks: 1196.6640625 MB +worker1-1 | 2024/06/04 05:06:05.068 FIG10b: Memory usage without flush after 25942 tasks: 1196.41796875 MB +worker1-1 | 2024/06/04 05:06:06.078 FIG10b: Memory usage without flush after 26060 tasks: 1196.69921875 MB +worker1-1 | 2024/06/04 05:06:07.090 FIG10b: Memory usage without flush after 26179 tasks: 1195.98046875 MB +worker1-1 | 2024/06/04 05:06:08.095 FIG10b: Memory usage without flush after 26306 tasks: 1195.9140625 MB +worker1-1 | 2024/06/04 05:06:09.103 FIG10b: Memory usage without flush after 26426 tasks: 1194.25 MB +worker1-1 | 2024/06/04 05:06:10.109 FIG10b: Memory usage without flush after 26532 tasks: 1194.55078125 MB +worker1-1 | 2024/06/04 05:06:11.111 FIG10b: Memory usage without flush after 26653 tasks: 1195.765625 MB +worker1-1 | 2024/06/04 05:06:12.137 [worker]2024/06/04 05:06:12.138 FIG10b: Memory usage without flush after 26779 tasks: 1195.46484375 MB +worker1-1 | [worker][worker] 2024/06/04 05:06:13.138 FIG10b: Memory usage without flush after 26904 tasks: 1196.9296875 MB +worker1-1 | [worker]2024/06/04 05:06:14.139[worker] FIG10b: Memory usage without flush after 27032 tasks: 1197.1171875 MB +worker1-1 | 2024/06/04 05:06:15.146 FIG10b: Memory usage without flush after 27151 tasks: 1196.859375 MB +worker1-1 | 2024/06/04 05:06:16.155 FIG10b: Memory usage without flush after 27280 tasks: 1197.3515625 MB +worker1-1 | 2024/06/04 05:06:17.158 FIG10b: Memory usage without flush after 27409 tasks: 1198.2734375 MB +worker1-1 | 2024/06/04 05:06:18.163 FIG10b: Memory usage without flush after 27537 tasks: 1198.72265625 MB +worker1-1 | 2024/06/04 05:06:19.1722024/06/04 05:06:19.172 FIG10b: Memory usage without flush after 27662 tasks: 1199.4375 MB +worker1-1 | 2024/06/04 05:06:20.177 FIG10b: Memory usage without flush after 27781 tasks: 1206.4609375 MB +worker1-1 | 2024/06/04 05:06:21.1822024/06/04 05:06:21.182 FIG10b: Memory usage without flush after 27910 tasks: 1206.453125 MB +worker1-1 | 2024/06/04 05:06:22.185 FIG10b: Memory usage without flush after 28036 tasks: 1206.6171875 MB +worker1-1 | [worker]stored...in Run() rn2024/06/04 05:06:23.189 FIG10b: Memory usage without flush after 28154 tasks: 1207.31640625 MB +worker1-1 | [worker]2024/06/04 05:06:24.192[worker] 2024/06/04 05:06:24.192 FIG10b: Memory usage without flush after 28269 tasks: 1207.59765625 MB +worker1-1 | 2024/06/04 05:06:25.196 FIG10b: Memory usage without flush after 28387 tasks: 1207.38671875 MB +worker1-1 | 2024/06/04 05:06:26.201 FIG10b: Memory usage without flush after 28509 tasks: 1207.40234375 MB +worker1-1 | 2024/06/04 05:06:27.204 [worker] 2024/06/04 05:06:27.205 FIG10b: Memory usage without flush after 28633 tasks: 1208.078125 MB +worker1-1 | 2024/06/04 05:06:28.2062024/06/04 05:06:28.206 FIG10b: Memory usage without flush after 28764 tasks: 1209.0 MB +worker1-1 | Executed!2024/06/04 05:06:29.211 FIG10b: Memory usage without flush after 28885 tasks: 1209.265625 MB +worker1-1 | 2024/06/04 05:06:30.218 FIG10b: Memory usage without flush after 29017 tasks: 1209.421875 MB +worker1-1 | 2024/06/04 05:06:31.223 FIG10b: Memory usage without flush after 29135 tasks: 1209.6796875 MB +worker1-1 | 2024/06/04 05:06:32.234 2024/06/04 05:06:32.234 FIG10b: Memory usage without flush after 29252 tasks: 1209.71875 MB +worker1-1 | 2024/06/04 05:06:33.235 FIG10b: Memory usage without flush after 29374 tasks: 1210.04296875 MB +worker1-1 | 2024/06/04 05:06:34.246 FIG10b: Memory usage without flush after 29496 tasks: 1209.84375 MB +worker1-1 | 2024/06/04 05:06:35.251 FIG10b: Memory usage without flush after 29624 tasks: 1210.43359375 MB +worker1-1 | [worker]2024/06/04 05:06:36.252 FIG10b: Memory usage without flush after 29745 tasks: 1209.67578125 MB +worker1-1 | 2024/06/04 05:06:37.270 [worker] 2024/06/04 05:06:37.271 FIG10b: Memory usage without flush after 29839 tasks: 1209.05859375 MB +worker1-1 | 2024/06/04 05:06:38.275 FIG10b: Memory usage without flush after 29945 tasks: 1209.65625 MB +worker1-1 | 2024/06/04 05:06:39.2792024/06/04 05:06:39.279 FIG10b: Memory usage without flush after 30072 tasks: 1209.984375 MB +worker1-1 | 2024/06/04 05:06:40.282 FIG10b: Memory usage without flush after 30204 tasks: 1208.9375 MB +worker1-1 | 2024/06/04 05:06:41.2822024/06/04 05:06:41.282 FIG10b: Memory usage without flush after 30336 tasks: 1209.0546875 MB +worker1-1 | 2024/06/04 05:06:42.286 FIG10b: Memory usage without flush after 30460 tasks: 1227.84765625 MB +worker1-1 | 2024/06/04 05:06:43.2852024/06/04 05:06:43.286 [worker]FIG10b: Memory usage without flush after 30584 tasks: 1228.1796875 MB +worker1-1 | 2024/06/04 05:06:44.297 FIG10b: Memory usage without flush after 30708 tasks: 1229.43359375 MB +worker1-1 | gonna store now2024/06/04 05:06:45.305 FIG10b: Memory usage without flush after 30838 tasks: 1229.54296875 MB +worker1-1 | 2024/06/04 05:06:46.311 FIG10b: Memory usage without flush after 30970 tasks: 1229.41796875 MB +worker1-1 | [worker][worker]2024/06/04 05:06:47.313 FIG10b: Memory usage without flush after 31094 tasks: 1229.97265625 MB +worker1-1 | FIG10b: Memory usage without flush after 31229 tasks: 1229.83984375 MB +worker1-1 | 2024/06/04 05:06:49.321 2024/06/04 05:06:49.322 FIG10b: Memory usage without flush after 31354 tasks: 1230.15625 MB +worker1-1 | [worker] 2024/06/04 05:06:50.326 FIG10b: Memory usage without flush after 31485 tasks: 1230.7109375 MB +worker1-1 | 2024/06/04 05:06:51.331 FIG10b: Memory usage without flush after 31609 tasks: 1230.125 MB +worker1-1 | 2024/06/04 05:06:52.337 FIG10b: Memory usage without flush after 31719 tasks: 1229.9921875 MB +worker1-1 | 2024/06/04 05:06:53.339 [worker]2024/06/04 05:06:53.339 FIG10b: Memory usage without flush after 31835 tasks: 1231.0703125 MB +worker1-1 | Executed!2024/06/04 05:06:54.346 FIG10b: Memory usage without flush after 31940 tasks: 1231.6640625 MB +worker1-1 | 2024/06/04 05:06:55.351 [worker] [worker] in Run() rn2024/06/04 05:06:55.352 2024/06/04 05:06:55.352 FIG10b: Memory usage without flush after 32057 tasks: 1231.9921875 MB +worker1-1 | 2024/06/04 05:06:56.357 FIG10b: Memory usage without flush after 32183 tasks: 1231.83984375 MB +worker1-1 | 2024/06/04 05:06:57.359 FIG10b: Memory usage without flush after 32309 tasks: 1231.92578125 MB +worker1-1 | 2024/06/04 05:06:58.363 FIG10b: Memory usage without flush after 32443 tasks: 1232.46484375 MB +worker1-1 | 2024/06/04 05:06:59.366 FIG10b: Memory usage without flush after 32568 tasks: 1231.86328125 MB +worker1-1 | 2024/06/04 05:07:00.373 [worker]2024/06/04 05:07:00.373 FIG10b: Memory usage without flush after 32687 tasks: 1232.2734375 MB +worker1-1 | 2024/06/04 05:07:01.3952024/06/04 05:07:01.395 FIG10b: Memory usage without flush after 32815 tasks: 1231.91796875 MB +worker1-1 | FIG10b: Memory usage without flush after 32915 tasks: 1232.0390625 MB +worker1-1 | 2024/06/04 05:07:03.407 FIG10b: Memory usage without flush after 33013 tasks: 1232.234375 MB +worker1-1 | 2024/06/04 05:07:04.408 FIG10b: Memory usage without flush after 33137 tasks: 1232.72265625 MB +worker1-1 | 2024/06/04 05:07:05.408 FIG10b: Memory usage without flush after 33259 tasks: 1232.98046875 MB +worker1-1 | 2024/06/04 05:07:06.4152024/06/04 05:07:06.415 FIG10b: Memory usage without flush after 33383 tasks: 1232.5546875 MB +worker1-1 | 2024/06/04 05:07:07.420 FIG10b: Memory usage without flush after 33509 tasks: 1233.26171875 MB +worker1-1 | [worker][worker]2024/06/04 05:07:37.421 FIG10b: Memory usage without flush after 36983 tasks: 1247.23828125 MB +worker1-1 | 2024/06/04 05:08:07.422 FIG10b: Memory usage without flush after 40479 tasks: 1292.04296875 MB +worker1-1 | 2024/06/04 05:08:37.433 FIG10b: Memory usage without flush after 43649 tasks: 1284.76171875 MB +worker1-1 | 2024/06/04 05:09:07.435 FIG10b: Memory usage without flush after 47102 tasks: 1307.30859375 MB +worker1-1 | 2024/06/04 05:09:37.437 FIG10b: Memory usage without flush after 50458 tasks: 1310.375 MB +worker1-1 | 2024/06/04 05:10:07.443 FIG10b: Memory usage without flush after 54043 tasks: 1344.3984375 MB +worker1-1 | 2024/06/04 05:10:37.444 FIG10b: Memory usage without flush after 57421 tasks: 1365.17578125 MB +worker1-1 | 2024/06/04 05:11:07.450 FIG10b: Memory usage without flush after 60590 tasks: 1376.80078125 MB +worker1-1 | 2024/06/04 05:11:37.457 FIG10b: Memory usage without flush after 63784 tasks: 1370.8515625 MB +worker1-1 | 2024/06/04 05:12:07.463 FIG10b: Memory usage without flush after 66952 tasks: 1390.1875 MB +worker1-1 | 2024/06/04 05:12:37.467 FIG10b: Memory usage without flush after 70098 tasks: 1391.84765625 MB +worker1-1 | 2024/06/04 05:13:07.468 2024/06/04 05:13:07.469 FIG10b: Memory usage without flush after 73440 tasks: 1402.96484375 MB +worker1-1 | 2024/06/04 05:13:37.4822024/06/04 05:13:37.483 FIG10b: Memory usage without flush after 76749 tasks: 1402.9140625 MB +worker1-1 | 2024/06/04 05:14:07.493[worker] FIG10b: Memory usage without flush after 79857 tasks: 1417.44140625 MB +worker1-1 | 2024/06/04 05:14:37.497 FIG10b: Memory usage without flush after 83054 tasks: 1417.73828125 MB +worker1-1 | 2024/06/04 05:15:07.498 [worker] in Run() rn2024/06/04 05:15:07.499 FIG10b: Memory usage without flush after 86111 tasks: 1424.1328125 MB +worker1-1 | 2024/06/04 05:15:37.499 FIG10b: Memory usage without flush after 89183 tasks: 1435.15625 MB +worker1-1 | 2024/06/04 05:16:07.509 FIG10b: Memory usage without flush after 92171 tasks: 1450.89453125 MB +worker1-1 | 2024/06/04 05:16:37.5142024/06/04 05:16:37.514 FIG10b: Memory usage without flush after 95399 tasks: 1445.234375 MB +worker1-1 | 2024/06/04 05:17:07.516 FIG10b: Memory usage without flush after 98729 tasks: 1459.6484375 MB +worker1-1 | [worker] 2024/06/04 05:17:37.518 FIG10b: Memory usage without flush after 101889 tasks: 1468.859375 MB +worker1-1 | 2024/06/04 05:18:07.520 FIG10b: Memory usage without flush after 105104 tasks: 1477.3515625 MB +worker1-1 | 2024/06/04 05:18:37.523 FIG10b: Memory usage without flush after 108452 tasks: 1489.90625 MB +worker1-1 | 2024/06/04 05:19:07.524 FIG10b: Memory usage without flush after 111871 tasks: 1498.66796875 MB +worker1-1 | 2024/06/04 05:19:37.5292024/06/04 05:19:37.530 FIG10b: Memory usage without flush after 115452 tasks: 1559.05859375 MB +worker1-1 | 2024/06/04 05:20:07.533 2024/06/04 05:20:07.533 FIG10b: Memory usage without flush after 118325 tasks: 1581.3359375 MB +worker1-1 | 2024/06/04 05:20:37.541 FIG10b: Memory usage without flush after 121643 tasks: 1576.234375 MB +worker1-1 | 2024/06/04 05:21:07.544 2024/06/04 05:21:07.544 FIG10b: Memory usage without flush after 125200 tasks: 1609.65234375 MB +worker1-1 | [worker]2024/06/04 05:21:37.563 FIG10b: Memory usage without flush after 128689 tasks: 1587.80859375 MB +worker1-1 | 2024/06/04 05:22:07.570 FIG10b: Memory usage without flush after 132013 tasks: 1588.0546875 MB +worker1-1 | 2024/06/04 05:22:37.579 FIG10b: Memory usage without flush after 135527 tasks: 1569.06640625 MB +worker1-1 | 2024/06/04 05:23:07.580 FIG10b: Memory usage without flush after 139395 tasks: 1677.1171875 MB +worker1-1 | 2024/06/04 05:23:37.5842024/06/04 05:23:37.585 FIG10b: Memory usage without flush after 142900 tasks: 1680.1640625 MB +worker1-1 | 2024/06/04 05:24:07.586 FIG10b: Memory usage without flush after 146401 tasks: 1626.828125 MB +worker1-1 | 2024/06/04 05:24:37.589 FIG10b: Memory usage without flush after 149920 tasks: 1621.16015625 MB +worker1-1 | 2024/06/04 05:25:07.593 [worker] 2024/06/04 05:25:07.593 FIG10b: Memory usage without flush after 153002 tasks: 1613.37890625 MB +worker1-1 | [worker]2024/06/04 05:25:37.594 FIG10b: Memory usage without flush after 156473 tasks: 1619.60546875 MB +worker1-1 | [worker][worker][worker]2024/06/04 05:26:07.599 FIG10b: Memory usage without flush after 159999 tasks: 1633.79296875 MB +worker1-1 | 2024/06/04 05:26:37.606[worker]2024/06/04 05:26:37.606 Executed! FIG10b: Memory usage without flush after 163639 tasks: 1622.60546875 MB +worker1-1 | 2024/06/04 05:27:07.613 FIG10b: Memory usage without flush after 167187 tasks: 1626.93359375 MB +worker1-1 | 2024/06/04 05:27:37.6202024/06/04 05:27:37.620 FIG10b: Memory usage without flush after 170695 tasks: 1628.02734375 MB +worker1-1 | 2024/06/04 05:28:07.6252024/06/04 05:28:07.625 FIG10b: Memory usage without flush after 174250 tasks: 1638.90625 MB +worker1-1 | 2024/06/04 05:28:37.633 in Run() rn2024/06/04 05:28:37.633 FIG10b: Memory usage without flush after 177854 tasks: 1645.7734375 MB +worker1-1 | 2024/06/04 05:29:07.639 [worker]2024/06/04 05:29:07.6402024/06/04 05:29:07.640 FIG10b: Memory usage without flush after 181212 tasks: 1632.65234375 MB +worker1-1 | [worker]2024/06/04 05:29:37.648 FIG10b: Memory usage without flush after 184665 tasks: 1645.6171875 MB +worker1-1 | 2024/06/04 05:30:07.650 FIG10b: Memory usage without flush after 188221 tasks: 1648.38671875 MB +worker1-1 | 2024/06/04 05:30:37.6562024/06/04 05:30:37.656 FIG10b: Memory usage without flush after 191943 tasks: 1637.078125 MB +worker1-1 | 2024/06/04 05:31:07.661 [worker]2024/06/04 05:31:07.661 2024/06/04 05:31:07.661 FIG10b: Memory usage without flush after 195503 tasks: 1674.83203125 MB +worker1-1 | 2024/06/04 05:31:37.6622024/06/04 05:31:37.662 FIG10b: Memory usage without flush after 198972 tasks: 1683.1796875 MB +worker1-1 | 2024/06/04 05:32:07.663 FIG10b: Memory usage without flush after 202546 tasks: 1681.87890625 MB +worker1-1 | 2024/06/04 05:32:37.666 FIG10b: Memory usage without flush after 205837 tasks: 1683.41796875 MB +worker1-1 | 2024/06/04 05:33:07.669 FIG10b: Memory usage without flush after 208914 tasks: 1697.05859375 MB +worker1-1 | 2024/06/04 05:33:37.678 FIG10b: Memory usage without flush after 211956 tasks: 1697.8671875 MB +worker1-1 | 2024/06/04 05:34:07.685 FIG10b: Memory usage without flush after 215103 tasks: 1711.38671875 MB +worker1-1 | 2024/06/04 05:34:37.6932024/06/04 05:34:37.693 FIG10b: Memory usage without flush after 218373 tasks: 1704.4921875 MB +worker1-1 | 2024/06/04 05:35:07.695 2024/06/04 05:35:07.6962024/06/04 05:35:07.696 FIG10b: Memory usage without flush after 221631 tasks: 1735.703125 MB +worker1-1 | 2024/06/04 05:35:37.697 FIG10b: Memory usage without flush after 225768 tasks: 1933.51171875 MB +worker1-1 | 2024/06/04 05:36:07.713 FIG10b: Memory usage without flush after 229021 tasks: 1952.546875 MB +worker1-1 | 2024/06/04 05:36:37.721 FIG10b: Memory usage without flush after 232420 tasks: 1962.08203125 MB +worker1-1 | 2024/06/04 05:37:07.722 FIG10b: Memory usage without flush after 235733 tasks: 1857.82421875 MB +worker1-1 | 2024/06/04 05:37:37.726 FIG10b: Memory usage without flush after 239177 tasks: 1834.546875 MB +worker1-1 | 2024/06/04 05:38:07.728 FIG10b: Memory usage without flush after 242716 tasks: 1841.734375 MB +worker1-1 | in Run() rn2024/06/04 05:38:37.731 FIG10b: Memory usage without flush after 246342 tasks: 1846.953125 MB +worker1-1 | 2024/06/04 05:39:07.733 [worker] 2024/06/04 05:39:07.733 FIG10b: Memory usage without flush after 249650 tasks: 1819.546875 MB +worker1-1 | 2024/06/04 05:39:37.755 FIG10b: Memory usage without flush after 252670 tasks: 1806.94140625 MB +worker1-1 | 2024/06/04 05:40:07.759 FIG10b: Memory usage without flush after 255800 tasks: 1800.9921875 MB +worker1-1 | 2024/06/04 05:40:37.765 2024/06/04 05:40:37.765 FIG10b: Memory usage without flush after 259206 tasks: 1800.98046875 MB +worker1-1 | 2024/06/04 05:41:07.768 FIG10b: Memory usage without flush after 262385 tasks: 1814.44921875 MB +worker1-1 | 2024/06/04 05:41:37.787 2024/06/04 05:41:37.787 FIG10b: Memory usage without flush after 265365 tasks: 1825.640625 MB +worker1-1 | 2024/06/04 05:42:07.7862024/06/04 05:42:07.787 FIG10b: Memory usage without flush after 268546 tasks: 1849.9140625 MB +worker1-1 | 2024/06/04 05:42:37.797 FIG10b: Memory usage without flush after 271943 tasks: 1718.9375 MB +worker1-1 | 2024/06/04 05:43:07.808 FIG10b: Memory usage without flush after 276123 tasks: 1734.16796875 MB +worker1-1 | 2024/06/04 05:43:37.815 FIG10b: Memory usage without flush after 279980 tasks: 1741.33984375 MB +worker1-1 | 2024/06/04 05:44:07.857 FIG10b: Memory usage without flush after 284536 tasks: 1747.734375 MB +worker1-1 | 2024/06/04 05:44:37.8582024/06/04 05:44:37.859 FIG10b: Memory usage without flush after 289278 tasks: 1754.3046875 MB +worker1-1 | 2024/06/04 05:45:07.896 2024/06/04 05:45:07.896 FIG10b: Memory usage without flush after 293687 tasks: 1757.74609375 MB +worker1-1 | 2024/06/04 05:45:37.901 FIG10b: Memory usage without flush after 297528 tasks: 1764.03125 MB +worker1-1 | 2024/06/04 05:46:07.904 FIG10b: Memory usage without flush after 301888 tasks: 1772.7265625 MB +worker1-1 | 2024/06/04 05:46:37.907 FIG10b: Memory usage without flush after 306560 tasks: 1785.47265625 MB +worker1-1 | 2024/06/04 05:47:07.912 FIG10b: Memory usage without flush after 310804 tasks: 1786.0703125 MB +worker1-1 | 2024/06/04 05:47:37.914 FIG10b: Memory usage without flush after 314913 tasks: 1790.37890625 MB +worker1-1 | 2024/06/04 05:48:07.917 FIG10b: Memory usage without flush after 319430 tasks: 1790.66796875 MB +worker1-1 | 2024/06/04 05:48:37.926 FIG10b: Memory usage without flush after 323794 tasks: 1799.15234375 MB +worker1-1 | 2024/06/04 05:49:07.926 FIG10b: Memory usage without flush after 328756 tasks: 1816.2578125 MB +worker1-1 | 2024/06/04 05:49:37.929 FIG10b: Memory usage without flush after 333277 tasks: 1824.21875 MB +worker1-1 | 2024/06/04 05:50:07.934 FIG10b: Memory usage without flush after 337899 tasks: 1832.9375 MB +worker1-1 | 2024/06/04 05:50:37.937 FIG10b: Memory usage without flush after 342472 tasks: 1840.79296875 MB +worker1-1 | [worker] 2024/06/04 05:51:07.941 FIG10b: Memory usage without flush after 346984 tasks: 1849.5546875 MB +worker1-1 | 2024/06/04 05:51:37.942 FIG10b: Memory usage without flush after 351769 tasks: 1852.1875 MB +worker1-1 | FIG10b: Memory usage without flush after 356876 tasks: 1860.515625 MB +worker1-1 | 2024/06/04 05:52:37.996 2024/06/04 05:52:37.997 FIG10b: Memory usage without flush after 361299 tasks: 1863.34765625 MB +worker1-1 | 2024/06/04 05:53:08.005 FIG10b: Memory usage without flush after 365767 tasks: 1875.44140625 MB +worker1-1 | 2024/06/04 05:53:38.005 FIG10b: Memory usage without flush after 370574 tasks: 1877.84765625 MB +worker1-1 | 2024/06/04 05:54:08.009 FIG10b: Memory usage without flush after 375416 tasks: 1888.6796875 MB +worker1-1 | [worker]2024/06/04 05:54:38.010 FIG10b: Memory usage without flush after 379714 tasks: 1895.55078125 MB +worker1-1 | 2024/06/04 05:55:08.011 FIG10b: Memory usage without flush after 384529 tasks: 1865.66015625 MB +worker1-1 | 2024/06/04 05:55:38.011 FIG10b: Memory usage without flush after 389515 tasks: 1887.671875 MB +worker1-1 | 2024/06/04 05:56:08.016 FIG10b: Memory usage without flush after 394431 tasks: 1888.40234375 MB +worker1-1 | 2024/06/04 05:56:38.015 FIG10b: Memory usage without flush after 399252 tasks: 1893.87890625 MB +worker1-1 | 2024/06/04 05:57:08.020 FIG10b: Memory usage without flush after 404301 tasks: 1903.44140625 MB +worker1-1 | 2024/06/04 05:57:38.021 FIG10b: Memory usage without flush after 409302 tasks: 1913.1875 MB +worker1-1 | 2024/06/04 05:58:08.0222024/06/04 05:58:08.022 FIG10b: Memory usage without flush after 414394 tasks: 1935.921875 MB[worker] +worker1-1 | 2024/06/04 05:58:38.028 FIG10b: Memory usage without flush after 419527 tasks: 1930.99609375 MB +worker1-1 | 2024/06/04 05:59:08.0322024/06/04 05:59:08.032 FIG10b: Memory usage without flush after 424439 tasks: 1943.5625 MB +worker1-1 | 2024/06/04 05:59:38.039 FIG10b: Memory usage without flush after 429165 tasks: 1948.5234375 MB +worker1-1 | 2024/06/04 06:00:08.0392024/06/04 06:00:08.040 FIG10b: Memory usage without flush after 434211 tasks: 1955.40625 MB +worker1-1 | 2024/06/04 06:00:38.043 FIG10b: Memory usage without flush after 438970 tasks: 1962.734375 MB +worker1-1 | 2024/06/04 06:01:08.046 FIG10b: Memory usage without flush after 443753 tasks: 1968.8828125 MB +worker1-1 | 2024/06/04 06:01:38.049 FIG10b: Memory usage without flush after 448729 tasks: 2005.7421875 MB diff --git a/analysis/output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv b/analysis/output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv new file mode 100644 index 0000000..fb900b3 --- /dev/null +++ b/analysis/output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv @@ -0,0 +1,237 @@ +Timestamp,Memory Usage (MB) +2024-06-04 05:02:08.045000,1046.09765625 +2024-06-04 05:02:09.053000,1047.3203125 +2024-06-04 05:02:10.057000,1038.96484375 +2024-06-04 05:02:11.060000,1044.33203125 +2024-06-04 05:02:12.069000,1049.59375 +2024-06-04 05:02:13.069000,1055.1796875 +2024-06-04 05:02:15.084000,1055.5078125 +2024-06-04 05:02:16.088000,1055.58203125 +2024-06-04 05:02:18.099000,1057.578125 +2024-06-04 05:02:19.109000,1057.734375 +2024-06-04 05:02:20.128000,1059.52734375 +2024-06-04 05:02:21.134000,1060.06640625 +2024-06-04 05:02:23.153000,1061.59375 +2024-06-04 05:02:25.161000,1065.05078125 +2024-06-04 05:02:26.171000,1065.30078125 +2024-06-04 05:02:27.474000,1065.41015625 +2024-06-04 05:02:28.481000,1069.33984375 +2024-06-04 05:02:29.515000,1069.31640625 +2024-06-04 05:02:30.520000,1073.29296875 +2024-06-04 05:02:32.522000,1074.93359375 +2024-06-04 05:02:35.543000,1074.82421875 +2024-06-04 05:02:38.554000,1076.67578125 +2024-06-04 05:02:39.556000,1076.46875 +2024-06-04 05:02:40.638000,1076.50390625 +2024-06-04 05:02:41.666000,1086.84765625 +2024-06-04 05:02:43.666000,1086.4140625 +2024-06-04 05:02:44.674000,1088.109375 +2024-06-04 05:02:48.696000,1088.625 +2024-06-04 05:02:52.716000,1091.8828125 +2024-06-04 05:02:54.736000,1093.75 +2024-06-04 05:02:58.760000,1093.91015625 +2024-06-04 05:02:59.763000,1093.84765625 +2024-06-04 05:03:01.777000,1094.30859375 +2024-06-04 05:03:03.784000,1094.41015625 +2024-06-04 05:03:06.818000,1094.76171875 +2024-06-04 05:03:09.845000,1094.91796875 +2024-06-04 05:03:11.848000,1095.92578125 +2024-06-04 05:03:12.860000,1095.65234375 +2024-06-04 05:03:14.872000,1097.30078125 +2024-06-04 05:03:17.012000,1096.73046875 +2024-06-04 05:03:19.022000,1097.453125 +2024-06-04 05:03:20.023000,1097.36328125 +2024-06-04 05:03:24.034000,1117.375 +2024-06-04 05:03:30.060000,1121.453125 +2024-06-04 05:03:32.067000,1122.484375 +2024-06-04 05:03:33.071000,1122.0546875 +2024-06-04 05:03:34.074000,1122.00390625 +2024-06-04 05:03:35.076000,1121.9609375 +2024-06-04 05:03:36.080000,1121.7421875 +2024-06-04 05:03:37.149000,1121.2734375 +2024-06-04 05:03:38.155000,1121.265625 +2024-06-04 05:03:39.157000,1122.70703125 +2024-06-04 05:03:40.157000,1123.43359375 +2024-06-04 05:03:41.160000,1123.70703125 +2024-06-04 05:03:44.166000,1124.14453125 +2024-06-04 05:03:45.178000,1124.15234375 +2024-06-04 05:03:46.259000,1123.47265625 +2024-06-04 05:03:50.276000,1124.7265625 +2024-06-04 05:03:55.288000,1125.7421875 +2024-06-04 05:03:56.298000,1125.046875 +2024-06-04 05:03:57.299000,1125.49609375 +2024-06-04 05:04:01.344000,1126.3515625 +2024-06-04 05:04:02.353000,1135.16796875 +2024-06-04 05:04:03.373000,1135.34765625 +2024-06-04 05:04:04.377000,1135.10546875 +2024-06-04 05:04:08.396000,1135.859375 +2024-06-04 05:04:09.403000,1135.97265625 +2024-06-04 05:04:10.414000,1135.953125 +2024-06-04 05:04:11.427000,1135.75 +2024-06-04 05:04:12.431000,1135.921875 +2024-06-04 05:04:14.446000,1136.7109375 +2024-06-04 05:04:16.462000,1137.453125 +2024-06-04 05:04:17.468000,1138.04296875 +2024-06-04 05:04:19.478000,1139.48046875 +2024-06-04 05:04:20.482000,1140.09375 +2024-06-04 05:04:21.486000,1140.7265625 +2024-06-04 05:04:22.494000,1141.3671875 +2024-06-04 05:04:23.495000,1141.96484375 +2024-06-04 05:04:25.509000,1142.01171875 +2024-06-04 05:04:28.522000,1142.375 +2024-06-04 05:04:31.535000,1142.7734375 +2024-06-04 05:04:32.537000,1142.21875 +2024-06-04 05:04:34.543000,1142.89453125 +2024-06-04 05:04:36.548000,1142.44921875 +2024-06-04 05:04:38.574000,1141.66015625 +2024-06-04 05:04:41.589000,1149.65234375 +2024-06-04 05:04:42.597000,1161.16796875 +2024-06-04 05:04:43.598000,1156.609375 +2024-06-04 05:04:44.606000,1154.74609375 +2024-06-04 05:04:46.621000,1155.28125 +2024-06-04 05:04:48.628000,1155.8046875 +2024-06-04 05:04:50.636000,1155.15234375 +2024-06-04 05:04:51.639000,1155.5078125 +2024-06-04 05:04:54.656000,1157.9921875 +2024-06-04 05:04:55.656000,1158.453125 +2024-06-04 05:04:57.659000,1155.86328125 +2024-06-04 05:04:58.667000,1156.83984375 +2024-06-04 05:05:00.678000,1155.984375 +2024-06-04 05:05:01.680000,1156.67578125 +2024-06-04 05:05:05.705000,1156.47265625 +2024-06-04 05:05:06.709000,1157.63671875 +2024-06-04 05:05:07.718000,1158.078125 +2024-06-04 05:05:08.724000,1157.0 +2024-06-04 05:05:11.737000,1158.40234375 +2024-06-04 05:05:12.745000,1158.05078125 +2024-06-04 05:05:13.746000,1157.85546875 +2024-06-04 05:05:14.750000,1158.12109375 +2024-06-04 05:05:16.761000,1158.09765625 +2024-06-04 05:05:17.768000,1158.53515625 +2024-06-04 05:05:18.772000,1159.5078125 +2024-06-04 05:05:20.787000,1159.3125 +2024-06-04 05:05:21.794000,1175.1171875 +2024-06-04 05:05:22.798000,1175.8046875 +2024-06-04 05:05:23.799000,1176.72265625 +2024-06-04 05:05:26.813000,1177.3515625 +2024-06-04 05:05:27.822000,1176.828125 +2024-06-04 05:05:34.909000,1179.80078125 +2024-06-04 05:05:35.916000,1179.89453125 +2024-06-04 05:05:36.925000,1179.71484375 +2024-06-04 05:05:37.930000,1184.296875 +2024-06-04 05:05:40.940000,1184.53515625 +2024-06-04 05:05:41.953000,1184.80859375 +2024-06-04 05:05:42.958000,1184.7421875 +2024-06-04 05:05:45.966000,1185.24609375 +2024-06-04 05:05:50.982000,1186.41796875 +2024-06-04 05:05:52.993000,1187.203125 +2024-06-04 05:05:53.996000,1187.26953125 +2024-06-04 05:05:56.005000,1187.33984375 +2024-06-04 05:05:57.009000,1187.41015625 +2024-06-04 05:06:00.023000,1187.66015625 +2024-06-04 05:06:01.034000,1188.265625 +2024-06-04 05:06:02.053000,1195.21484375 +2024-06-04 05:06:05.068000,1196.41796875 +2024-06-04 05:06:06.078000,1196.69921875 +2024-06-04 05:06:07.090000,1195.98046875 +2024-06-04 05:06:08.095000,1195.9140625 +2024-06-04 05:06:09.103000,1194.25 +2024-06-04 05:06:10.109000,1194.55078125 +2024-06-04 05:06:11.111000,1195.765625 +2024-06-04 05:06:15.146000,1196.859375 +2024-06-04 05:06:16.155000,1197.3515625 +2024-06-04 05:06:17.158000,1198.2734375 +2024-06-04 05:06:18.163000,1198.72265625 +2024-06-04 05:06:20.177000,1206.4609375 +2024-06-04 05:06:22.185000,1206.6171875 +2024-06-04 05:06:25.196000,1207.38671875 +2024-06-04 05:06:26.201000,1207.40234375 +2024-06-04 05:06:30.218000,1209.421875 +2024-06-04 05:06:31.223000,1209.6796875 +2024-06-04 05:06:33.235000,1210.04296875 +2024-06-04 05:06:34.246000,1209.84375 +2024-06-04 05:06:35.251000,1210.43359375 +2024-06-04 05:06:38.275000,1209.65625 +2024-06-04 05:06:40.282000,1208.9375 +2024-06-04 05:06:42.286000,1227.84765625 +2024-06-04 05:06:44.297000,1229.43359375 +2024-06-04 05:06:46.311000,1229.41796875 +2024-06-04 05:06:51.331000,1230.125 +2024-06-04 05:06:52.337000,1229.9921875 +2024-06-04 05:06:56.357000,1231.83984375 +2024-06-04 05:06:57.359000,1231.92578125 +2024-06-04 05:06:58.363000,1232.46484375 +2024-06-04 05:06:59.366000,1231.86328125 +2024-06-04 05:07:03.407000,1232.234375 +2024-06-04 05:07:04.408000,1232.72265625 +2024-06-04 05:07:05.408000,1232.98046875 +2024-06-04 05:07:07.420000,1233.26171875 +2024-06-04 05:08:07.422000,1292.04296875 +2024-06-04 05:08:37.433000,1284.76171875 +2024-06-04 05:09:07.435000,1307.30859375 +2024-06-04 05:09:37.437000,1310.375 +2024-06-04 05:10:07.443000,1344.3984375 +2024-06-04 05:10:37.444000,1365.17578125 +2024-06-04 05:11:07.450000,1376.80078125 +2024-06-04 05:11:37.457000,1370.8515625 +2024-06-04 05:12:07.463000,1390.1875 +2024-06-04 05:12:37.467000,1391.84765625 +2024-06-04 05:14:37.497000,1417.73828125 +2024-06-04 05:15:37.499000,1435.15625 +2024-06-04 05:16:07.509000,1450.89453125 +2024-06-04 05:17:07.516000,1459.6484375 +2024-06-04 05:18:07.520000,1477.3515625 +2024-06-04 05:18:37.523000,1489.90625 +2024-06-04 05:19:07.524000,1498.66796875 +2024-06-04 05:20:37.541000,1576.234375 +2024-06-04 05:22:07.570000,1588.0546875 +2024-06-04 05:22:37.579000,1569.06640625 +2024-06-04 05:23:07.580000,1677.1171875 +2024-06-04 05:24:07.586000,1626.828125 +2024-06-04 05:24:37.589000,1621.16015625 +2024-06-04 05:27:07.613000,1626.93359375 +2024-06-04 05:30:07.650000,1648.38671875 +2024-06-04 05:32:07.663000,1681.87890625 +2024-06-04 05:32:37.666000,1683.41796875 +2024-06-04 05:33:07.669000,1697.05859375 +2024-06-04 05:33:37.678000,1697.8671875 +2024-06-04 05:34:07.685000,1711.38671875 +2024-06-04 05:35:37.697000,1933.51171875 +2024-06-04 05:36:07.713000,1952.546875 +2024-06-04 05:36:37.721000,1962.08203125 +2024-06-04 05:37:07.722000,1857.82421875 +2024-06-04 05:37:37.726000,1834.546875 +2024-06-04 05:38:07.728000,1841.734375 +2024-06-04 05:39:37.755000,1806.94140625 +2024-06-04 05:40:07.759000,1800.9921875 +2024-06-04 05:41:07.768000,1814.44921875 +2024-06-04 05:42:37.797000,1718.9375 +2024-06-04 05:43:07.808000,1734.16796875 +2024-06-04 05:43:37.815000,1741.33984375 +2024-06-04 05:44:07.857000,1747.734375 +2024-06-04 05:45:37.901000,1764.03125 +2024-06-04 05:46:07.904000,1772.7265625 +2024-06-04 05:46:37.907000,1785.47265625 +2024-06-04 05:47:07.912000,1786.0703125 +2024-06-04 05:47:37.914000,1790.37890625 +2024-06-04 05:48:07.917000,1790.66796875 +2024-06-04 05:48:37.926000,1799.15234375 +2024-06-04 05:49:07.926000,1816.2578125 +2024-06-04 05:49:37.929000,1824.21875 +2024-06-04 05:50:07.934000,1832.9375 +2024-06-04 05:50:37.937000,1840.79296875 +2024-06-04 05:51:37.942000,1852.1875 +2024-06-04 05:53:08.005000,1875.44140625 +2024-06-04 05:53:38.005000,1877.84765625 +2024-06-04 05:54:08.009000,1888.6796875 +2024-06-04 05:55:08.011000,1865.66015625 +2024-06-04 05:55:38.011000,1887.671875 +2024-06-04 05:56:08.016000,1888.40234375 +2024-06-04 05:56:38.015000,1893.87890625 +2024-06-04 05:57:08.020000,1903.44140625 +2024-06-04 05:57:38.021000,1913.1875 +2024-06-04 05:58:38.028000,1930.99609375 +2024-06-04 05:59:38.039000,1948.5234375 +2024-06-04 06:00:38.043000,1962.734375 +2024-06-04 06:01:08.046000,1968.8828125 +2024-06-04 06:01:38.049000,2005.7421875 diff --git a/analysis/output_10b_run4_1s_flush_no_manual_garbage_yet.txt b/analysis/output_10b_run4_1s_flush_no_manual_garbage_yet.txt new file mode 100644 index 0000000..68996ca --- /dev/null +++ b/analysis/output_10b_run4_1s_flush_no_manual_garbage_yet.txt @@ -0,0 +1,361 @@ +worker1-1 | 2024/06/04 17:11:16.070 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 17:11:16.080 FIG10b: Starting tasks submission... +worker1-1 | 2024/06/04 17:11:17.088 FIG10b: Memory usage without flush after 126 tasks: 771.67578125 MB +worker1-1 | 2024/06/04 17:11:18.095 FIG10b: Memory usage without flush after 249 tasks: 772.46484375 MB +worker1-1 | 2024/06/04 17:11:19.103 FIG10b: Memory usage without flush after 369 tasks: 777.73046875 MB +worker1-1 | 2024/06/04 17:11:20.1102024/06/04 17:11:20.110 FIG10b: Memory usage without flush after 501 tasks: 778.734375 MB +worker1-1 | 2024/06/04 17:11:21.113 FIG10b: Memory usage without flush after 628 tasks: 780.4921875 MB +worker1-1 | 2024/06/04 17:11:22.118 2024/06/04 17:11:22.119 FIG10b: Memory usage without flush after 761 tasks: 781.50390625 MB +worker1-1 | 2024/06/04 17:11:23.125 2024/06/04 17:11:23.126 FIG10b: Memory usage without flush after 894 tasks: 783.9140625 MB +worker1-1 | 2024/06/04 17:11:24.133 FIG10b: Memory usage without flush after 1013 tasks: 784.97265625 MB +worker1-1 | 2024/06/04 17:11:25.1402024/06/04 17:11:25.140 FIG10b: Memory usage without flush after 1145 tasks: 785.98828125 MB +worker1-1 | 2024/06/04 17:11:26.149 2024/06/04 17:11:26.149 FIG10b: Memory usage without flush after 1264 tasks: 786.765625 MB +worker1-1 | 2024/06/04 17:11:27.150 FIG10b: Memory usage without flush after 1397 tasks: 791.703125 MB2024/06/04 17:11:27.151 [lobs] WERE GONNA CREATE A /DEV FILE WITH THIS NAME: /dev/shm/5157570045114249705 +worker1-1 | 2024/06/04 17:11:28.164 FIG10b: Memory usage without flush after 1522 tasks: 792.44921875 MB +worker1-1 | 2024/06/04 17:11:29.165 FIG10b: Memory usage without flush after 1640 tasks: 792.2421875 MB +worker1-1 | 2024/06/04 17:11:30.167[worker] FIG10b: Memory usage without flush after 1766 tasks: 798.19921875 MB +worker1-1 | 2024/06/04 17:11:31.176 FIG10b: Memory usage without flush after 1892 tasks: 798.71484375 MB +worker1-1 | 2024/06/04 17:11:32.178 took this many seconds: 0.00527FIG10b: Memory usage without flush after 2022 tasks: 799.47265625 MB +worker1-1 | 2024/06/04 17:11:33.1862024/06/04 17:11:33.186took this many seconds: 0.0114in Run() rn FIG10b: Memory usage without flush after 2151 tasks: 799.859375 MB +worker1-1 | 2024/06/04 17:11:34.188 FIG10b: Memory usage without flush after 2283 tasks: 799.8671875 MB +worker1-1 | 2024/06/04 17:11:35.192 FIG10b: Memory usage without flush after 2415 tasks: 804.99609375 MB +worker1-1 | [worker] Executed!2024/06/04 17:11:36.199 FIG10b: Memory usage without flush after 2542 tasks: 804.78515625 MB +worker1-1 | 2024/06/04 17:11:37.200 FIG10b: Memory usage without flush after 2669 tasks: 805.52734375 MB +worker1-1 | 2024/06/04 17:11:38.205 FIG10b: Memory usage without flush after 2803 tasks: 806.2421875 MB +worker1-1 | 2024/06/04 17:11:39.211 2024/06/04 17:11:39.211 [worker]2024/06/04 17:11:39.2122024/06/04 17:11:39.212 FIG10b: Memory usage without flush after 2929 tasks: 811.859375 MB +worker1-1 | 2024/06/04 17:11:40.2132024/06/04 17:11:40.213 FIG10b: Memory usage without flush after 3056 tasks: 813.1015625 MB +worker1-1 | 2024/06/04 17:11:41.218 FIG10b: Memory usage without flush after 3185 tasks: 818.4453125 MB +worker1-1 | 2024/06/04 17:11:42.225 FIG10b: Memory usage without flush after 3321 tasks: 819.41796875 MB +worker1-1 | 2024/06/04 17:11:43.230 FIG10b: Memory usage without flush after 3449 tasks: 820.62890625 MB +worker1-1 | 2024/06/04 17:11:44.234 FIG10b: Memory usage without flush after 3581 tasks: 820.8046875 MB +worker1-1 | 2024/06/04 17:11:45.235 FIG10b: Memory usage without flush after 3710 tasks: 821.234375 MB +worker1-1 | [worker][worker] 2024/06/04 17:11:46.238 FIG10b: Memory usage without flush after 3830 tasks: 821.2890625 MB +worker1-1 | 2024/06/04 17:11:47.243 FIG10b: Memory usage without flush after 3953 tasks: 821.52734375 MB +worker1-1 | 2024/06/04 17:11:48.249[worker] 2024/06/04 17:11:48.250 FIG10b: Memory usage without flush after 4081 tasks: 822.03125 MB +worker1-1 | 2024/06/04 17:11:49.2552024/06/04 17:11:49.256 FIG10b: Memory usage without flush after 4204 tasks: 822.7578125 MB +worker1-1 | 2024/06/04 17:11:50.262 FIG10b: Memory usage without flush after 4329 tasks: 823.5234375 MB +worker1-1 | 2024/06/04 17:11:51.266 FIG10b: Memory usage without flush after 4446 tasks: 823.375 MB +worker1-1 | 2024/06/04 17:11:52.278 2024/06/04 17:11:52.278 FIG10b: Memory usage without flush after 4574 tasks: 823.78125 MB +worker1-1 | 2024/06/04 17:11:53.287 FIG10b: Memory usage without flush after 4708 tasks: 824.54296875 MB +worker1-1 | 2024/06/04 17:11:54.287 FIG10b: Memory usage without flush after 4834 tasks: 824.58984375 MB +worker1-1 | 2024/06/04 17:11:55.293 FIG10b: Memory usage without flush after 4964 tasks: 824.53515625 MB +worker1-1 | 2024/06/04 17:11:56.300 FIG10b: Memory usage without flush after 5104 tasks: 833.94140625 MB +worker1-1 | 2024/06/04 17:11:57.309 FIG10b: Memory usage without flush after 5233 tasks: 833.9296875 MB +worker1-1 | 2024/06/04 17:11:58.309 FIG10b: Memory usage without flush after 5366 tasks: 837.75 MB +worker1-1 | 2024/06/04 17:11:59.312 FIG10b: Memory usage without flush after 5482 tasks: 833.55859375 MB +worker1-1 | 2024/06/04 17:12:00.3132024/06/04 17:12:00.314 FIG10b: Memory usage without flush after 5612 tasks: 838.21875 MB +worker1-1 | 2024/06/04 17:12:01.319 FIG10b: Memory usage without flush after 5740 tasks: 838.6328125 MB +worker1-1 | 2024/06/04 17:12:02.3242024/06/04 17:12:02.325 FIG10b: Memory usage without flush after 5872 tasks: 839.59375 MB +worker1-1 | 2024/06/04 17:12:03.325 FIG10b: Memory usage without flush after 6004 tasks: 840.375 MB +worker1-1 | 2024/06/04 17:12:04.331 2024/06/04 17:12:04.332 FIG10b: Memory usage without flush after 6134 tasks: 840.078125 MB +worker1-1 | 2024/06/04 17:12:05.335 FIG10b: Memory usage without flush after 6256 tasks: 841.01953125 MB +worker1-1 | 2024/06/04 17:12:06.3352024/06/04 17:12:06.335 FIG10b: Memory usage without flush after 6386 tasks: 841.3125 MB +worker1-1 | 2024/06/04 17:12:07.338 2024/06/04 17:12:07.338 FIG10b: Memory usage without flush after 6513 tasks: 841.9609375 MB +worker1-1 | 2024/06/04 17:12:08.3442024/06/04 17:12:08.344 FIG10b: Memory usage without flush after 6642 tasks: 841.26171875 MB +worker1-1 | 2024/06/04 17:12:09.350 2024/06/04 17:12:09.351 FIG10b: Memory usage without flush after 6774 tasks: 841.546875 MB +worker1-1 | 2024/06/04 17:12:10.356 FIG10b: Memory usage without flush after 6904 tasks: 842.140625 MB +worker1-1 | 2024/06/04 17:12:11.361 FIG10b: Memory usage without flush after 7030 tasks: 842.35546875 MB +worker1-1 | 2024/06/04 17:12:12.366 FIG10b: Memory usage without flush after 7164 tasks: 842.37890625 MB +worker1-1 | 2024/06/04 17:12:13.3722024/06/04 17:12:13.373 FIG10b: Memory usage without flush after 7302 tasks: 842.95703125 MB +worker1-1 | [worker] 2024/06/04 17:12:14.374 FIG10b: Memory usage without flush after 7427 tasks: 843.984375 MB +worker1-1 | 2024/06/04 17:12:15.383 FIG10b: Memory usage without flush after 7558 tasks: 843.76953125 MB +worker1-1 | 2024/06/04 17:12:16.390 FIG10b: Memory usage without flush after 7683 tasks: 845.1796875 MB +worker1-1 | 2024/06/04 17:12:17.394 FIG10b: Memory usage without flush after 7812 tasks: 844.82421875 MB +worker1-1 | 2024/06/04 17:12:18.398 FIG10b: Memory usage without flush after 7949 tasks: 845.65625 MB +worker1-1 | 2024/06/04 17:12:19.410 FIG10b: Memory usage without flush after 8088 tasks: 845.83203125 MB +worker1-1 | 2024/06/04 17:12:20.4232024/06/04 17:12:20.423 FIG10b: Memory usage without flush after 8218 tasks: 845.875 MB +worker1-1 | 2024/06/04 17:12:21.4262024/06/04 17:12:21.426 FIG10b: Memory usage without flush after 8348 tasks: 848.12109375 MB +worker1-1 | 2024/06/04 17:12:22.429 FIG10b: Memory usage without flush after 8474 tasks: 848.828125 MB +worker1-1 | took this many seconds: 0.007982024/06/04 17:12:23.4332024/06/04 17:12:23.433 FIG10b: Memory usage without flush after 8600 tasks: 849.41015625 MB +worker1-1 | 2024/06/04 17:12:24.434 FIG10b: Memory usage without flush after 8725 tasks: 850.30078125 MB +worker1-1 | 2024/06/04 17:12:25.438 FIG10b: Memory usage without flush after 8841 tasks: 851.3046875 MB +worker1-1 | 2024/06/04 17:12:26.443 FIG10b: Memory usage without flush after 8959 tasks: 851.3515625 MB +worker1-1 | 2024/06/04 17:12:27.445 FIG10b: Memory usage without flush after 9095 tasks: 852.7578125 MB +worker1-1 | 2024/06/04 17:12:28.457 FIG10b: Memory usage without flush after 9226 tasks: 853.6328125 MB +worker1-1 | 2024/06/04 17:12:29.458 FIG10b: Memory usage without flush after 9353 tasks: 854.24609375 MB +worker1-1 | FIG10b: Memory usage without flush after 9487 tasks: 854.7890625 MB +worker1-1 | 2024/06/04 17:12:31.473 FIG10b: Memory usage without flush after 9616 tasks: 855.4375 MB +worker1-1 | 2024/06/04 17:12:32.480 FIG10b: Memory usage without flush after 9756 tasks: 855.609375 MB +worker1-1 | 2024/06/04 17:12:33.491 FIG10b: Memory usage without flush after 9894 tasks: 855.484375 MB +worker1-1 | FIG10b: Memory usage without flush after 10030 tasks: 856.63671875 MB +worker1-1 | 2024/06/04 17:12:35.495 FIG10b: Memory usage without flush after 10169 tasks: 857.609375 MB +worker1-1 | 2024/06/04 17:12:36.496 FIG10b: Memory usage without flush after 10298 tasks: 858.7734375 MB +worker1-1 | 2024/06/04 17:12:37.499 FIG10b: Memory usage without flush after 10432 tasks: 859.19140625 MB +worker1-1 | 2024/06/04 17:12:38.504 FIG10b: Memory usage without flush after 10563 tasks: 859.2421875 MB +worker1-1 | [worker] 2024/06/04 17:12:39.508 FIG10b: Memory usage without flush after 10700 tasks: 860.328125 MB +worker1-1 | [worker]2024/06/04 17:12:40.515 FIG10b: Memory usage without flush after 10829 tasks: 860.34765625 MB +worker1-1 | 2024/06/04 17:12:41.517 FIG10b: Memory usage without flush after 10960 tasks: 867.90234375 MB +worker1-1 | 2024/06/04 17:12:42.522 2024/06/04 17:12:42.522FIG10b: Memory usage without flush after 11093 tasks: 868.18359375 MB +worker1-1 | 2024/06/04 17:12:43.5222024/06/04 17:12:43.523 FIG10b: Memory usage without flush after 11225 tasks: 868.25 MB[worker] +worker1-1 | 2024/06/04 17:12:44.530 2024/06/04 17:12:44.531 FIG10b: Memory usage without flush after 11360 tasks: 868.62890625 MB +worker1-1 | 2024/06/04 17:12:45.536 FIG10b: Memory usage without flush after 11490 tasks: 869.44921875 MB +worker1-1 | FIG10b: Memory usage without flush after 11618 tasks: 871.24609375 MB +worker1-1 | gonna store now2024/06/04 17:12:47.548 2024/06/04 17:12:47.548 FIG10b: Memory usage without flush after 11756 tasks: 871.8515625 MB +worker1-1 | 2024/06/04 17:12:48.554Executed!2024/06/04 17:12:48.555 FIG10b: Memory usage without flush after 11897 tasks: 876.53125 MB +worker1-1 | 2024/06/04 17:12:49.556 FIG10b: Memory usage without flush after 12034 tasks: 878.81640625 MB +worker1-1 | [worker] gonna store now2024/06/04 17:12:50.557 FIG10b: Memory usage without flush after 12161 tasks: 878.62109375 MB +worker1-1 | 2024/06/04 17:12:51.562 [worker]2024/06/04 17:12:51.563 FIG10b: Memory usage without flush after 12297 tasks: 879.7421875 MB +worker1-1 | 2024/06/04 17:12:52.568 FIG10b: Memory usage without flush after 12428 tasks: 880.1171875 MB +worker1-1 | 2024/06/04 17:12:53.576 FIG10b: Memory usage without flush after 12565 tasks: 880.9609375 MB +worker1-1 | 2024/06/04 17:12:54.588 [worker]2024/06/04 17:12:54.588 FIG10b: Memory usage without flush after 12703 tasks: 880.77734375 MB +worker1-1 | 2024/06/04 17:12:55.5912024/06/04 17:12:55.591 FIG10b: Memory usage without flush after 12835 tasks: 880.2109375 MB +worker1-1 | 2024/06/04 17:12:56.598 2024/06/04 17:12:56.598 FIG10b: Memory usage without flush after 12971 tasks: 882.234375 MB +worker1-1 | 2024/06/04 17:12:57.600 2024/06/04 17:12:57.600 FIG10b: Memory usage without flush after 13118 tasks: 881.69921875 MB +worker1-1 | 2024/06/04 17:12:58.605 FIG10b: Memory usage without flush after 13250 tasks: 882.234375 MB +worker1-1 | 2024/06/04 17:12:59.605 FIG10b: Memory usage without flush after 13387 tasks: 882.9921875 MB +worker1-1 | 2024/06/04 17:13:00.605 2024/06/04 17:13:00.606 FIG10b: Memory usage without flush after 13517 tasks: 883.83984375 MB +worker1-1 | 2024/06/04 17:13:01.608 [worker] in Run() rn2024/06/04 17:13:01.608 FIG10b: Memory usage without flush after 13647 tasks: 884.6875 MB +worker1-1 | 2024/06/04 17:13:02.612 2024/06/04 17:13:02.612 FIG10b: Memory usage without flush after 13784 tasks: 886.5 MB +worker1-1 | 2024/06/04 17:13:03.6182024/06/04 17:13:03.618 FIG10b: Memory usage without flush after 13909 tasks: 887.3125 MB +worker1-1 | 2024/06/04 17:13:04.6192024/06/04 17:13:04.620 FIG10b: Memory usage without flush after 14044 tasks: 887.86328125 MB +worker1-1 | 2024/06/04 17:13:05.623 FIG10b: Memory usage without flush after 14174 tasks: 888.8828125 MB +worker1-1 | 2024/06/04 17:13:06.627 FIG10b: Memory usage without flush after 14303 tasks: 889.046875 MB +worker1-1 | 2024/06/04 17:13:07.631 FIG10b: Memory usage without flush after 14427 tasks: 886.82421875 MB +worker1-1 | 2024/06/04 17:13:08.634 FIG10b: Memory usage without flush after 14576 tasks: 890.421875 MB +worker1-1 | 2024/06/04 17:13:09.635 FIG10b: Memory usage without flush after 14715 tasks: 891.55859375 MB +worker1-1 | 2024/06/04 17:13:10.639 FIG10b: Memory usage without flush after 14862 tasks: 892.66796875 MB +worker1-1 | 2024/06/04 17:13:11.639 FIG10b: Memory usage without flush after 14992 tasks: 893.765625 MB +worker1-1 | 2024/06/04 17:13:12.640 FIG10b: Memory usage without flush after 15124 tasks: 898.5859375 MB +worker1-1 | 2024/06/04 17:13:13.643 FIG10b: Memory usage without flush after 15256 tasks: 898.77734375 MB +worker1-1 | 2024/06/04 17:13:14.649 FIG10b: Memory usage without flush after 15394 tasks: 901.1171875 MB +worker1-1 | 2024/06/04 17:13:15.652 FIG10b: Memory usage without flush after 15512 tasks: 902.05078125 MB +worker1-1 | 2024/06/04 17:13:16.655 FIG10b: Memory usage without flush after 15648 tasks: 903.03125 MB +worker1-1 | 2024/06/04 17:13:17.659 FIG10b: Memory usage without flush after 15791 tasks: 903.9140625 MB +worker1-1 | 2024/06/04 17:13:18.662 FIG10b: Memory usage without flush after 15927 tasks: 904.69140625 MB +worker1-1 | 2024/06/04 17:13:19.666 FIG10b: Memory usage without flush after 16055 tasks: 904.06640625 MB +worker1-1 | 2024/06/04 17:13:20.667 FIG10b: Memory usage without flush after 16176 tasks: 905.9140625 MB +worker1-1 | 2024/06/04 17:13:21.668 FIG10b: Memory usage without flush after 16312 tasks: 906.046875 MB +worker1-1 | 2024/06/04 17:13:22.672 FIG10b: Memory usage without flush after 16445 tasks: 906.39453125 MB +worker1-1 | 2024/06/04 17:13:23.673 FIG10b: Memory usage without flush after 16585 tasks: 908.36328125 MB +worker1-1 | 2024/06/04 17:13:24.674 FIG10b: Memory usage without flush after 16717 tasks: 907.32421875 MB +worker1-1 | 2024/06/04 17:13:25.677 FIG10b: Memory usage without flush after 16850 tasks: 908.99609375 MB +worker1-1 | 2024/06/04 17:13:26.680 FIG10b: Memory usage without flush after 16983 tasks: 909.5703125 MB +worker1-1 | 2024/06/04 17:13:27.684 FIG10b: Memory usage without flush after 17125 tasks: 911.25 MB +worker1-1 | 2024/06/04 17:13:28.685 FIG10b: Memory usage without flush after 17254 tasks: 912.15234375 MB +worker1-1 | 2024/06/04 17:13:29.689 FIG10b: Memory usage without flush after 17387 tasks: 912.078125 MB +worker1-1 | 2024/06/04 17:13:30.691 FIG10b: Memory usage without flush after 17530 tasks: 916.59765625 MB +worker1-1 | 2024/06/04 17:13:31.693 FIG10b: Memory usage without flush after 17671 tasks: 913.24609375 MB +worker1-1 | 2024/06/04 17:13:32.695 FIG10b: Memory usage without flush after 17809 tasks: 914.359375 MB +worker1-1 | 2024/06/04 17:13:33.695 FIG10b: Memory usage without flush after 17953 tasks: 915.4296875 MB +worker1-1 | 2024/06/04 17:13:34.696 FIG10b: Memory usage without flush after 18093 tasks: 916.84765625 MB +worker1-1 | 2024/06/04 17:13:35.697 FIG10b: Memory usage without flush after 18229 tasks: 916.2265625 MB +worker1-1 | 2024/06/04 17:13:36.697 FIG10b: Memory usage without flush after 18362 tasks: 917.09375 MB +worker1-1 | 2024/06/04 17:13:37.701 FIG10b: Memory usage without flush after 18488 tasks: 917.9921875 MB +worker1-1 | 2024/06/04 17:13:38.701 FIG10b: Memory usage without flush after 18629 tasks: 918.15625 MB +worker1-1 | 2024/06/04 17:13:39.705 FIG10b: Memory usage without flush after 18775 tasks: 919.9140625 MB +worker1-1 | 2024/06/04 17:13:40.711 FIG10b: Memory usage without flush after 18917 tasks: 920.96484375 MB +worker1-1 | 2024/06/04 17:13:41.710 FIG10b: Memory usage without flush after 19046 tasks: 921.4609375 MB +worker1-1 | 2024/06/04 17:13:42.713 FIG10b: Memory usage without flush after 19186 tasks: 925.78515625 MB +worker1-1 | 2024/06/04 17:13:43.717 FIG10b: Memory usage without flush after 19324 tasks: 927.91015625 MB +worker1-1 | 2024/06/04 17:13:44.717 FIG10b: Memory usage without flush after 19458 tasks: 925.828125 MB +worker1-1 | 2024/06/04 17:13:45.721 FIG10b: Memory usage without flush after 19583 tasks: 926.21484375 MB +worker1-1 | 2024/06/04 17:13:46.739 FIG10b: Memory usage without flush after 19721 tasks: 933.5078125 MB +worker1-1 | 2024/06/04 17:13:47.741 FIG10b: Memory usage without flush after 19870 tasks: 934.7421875 MB +worker1-1 | 2024/06/04 17:13:48.743 FIG10b: Memory usage without flush after 20006 tasks: 935.6875 MB +worker1-1 | 2024/06/04 17:13:49.7452024/06/04 17:13:49.746 FIG10b: Memory usage without flush after 20152 tasks: 937.8984375 MB +worker1-1 | 2024/06/04 17:13:50.748 FIG10b: Memory usage without flush after 20291 tasks: 937.63671875 MB +worker1-1 | 2024/06/04 17:13:51.753 FIG10b: Memory usage without flush after 20420 tasks: 938.9921875 MB +worker1-1 | 2024/06/04 17:13:52.756 FIG10b: Memory usage without flush after 20558 tasks: 939.62890625 MB +worker1-1 | 2024/06/04 17:13:53.757 FIG10b: Memory usage without flush after 20695 tasks: 939.671875 MB +worker1-1 | 2024/06/04 17:13:54.786 2024/06/04 17:13:54.787 FIG10b: Memory usage without flush after 20825 tasks: 938.67578125 MB +worker1-1 | 2024/06/04 17:13:55.797 FIG10b: Memory usage without flush after 20958 tasks: 938.77734375 MB +worker1-1 | 2024/06/04 17:13:56.799 FIG10b: Memory usage without flush after 21109 tasks: 939.7734375 MB +worker1-1 | 2024/06/04 17:13:57.808 [worker] 2024/06/04 17:13:57.808 FIG10b: Memory usage without flush after 21232 tasks: 940.71875 MB +worker1-1 | 2024/06/04 17:13:58.811 FIG10b: Memory usage without flush after 21366 tasks: 940.22265625 MB +worker1-1 | 2024/06/04 17:13:59.813 FIG10b: Memory usage without flush after 21504 tasks: 941.5234375 MB +worker1-1 | 2024/06/04 17:14:00.835 FIG10b: Memory usage without flush after 21649 tasks: 941.66796875 MB +worker1-1 | 2024/06/04 17:14:01.838 FIG10b: Memory usage without flush after 21793 tasks: 942.91015625 MB +worker1-1 | 2024/06/04 17:14:02.840 FIG10b: Memory usage without flush after 21935 tasks: 941.91796875 MB +worker1-1 | 2024/06/04 17:14:03.842 FIG10b: Memory usage without flush after 22080 tasks: 951.1796875 MB +worker1-1 | [worker] 2024/06/04 17:14:04.848 FIG10b: Memory usage without flush after 22220 tasks: 952.33984375 MB +worker1-1 | 2024/06/04 17:14:05.855 FIG10b: Memory usage without flush after 22355 tasks: 950.19140625 MB +worker1-1 | [worker]2024/06/04 17:14:06.870 FIG10b: Memory usage without flush after 22496 tasks: 951.20703125 MB +worker1-1 | 2024/06/04 17:14:07.874 FIG10b: Memory usage without flush after 22631 tasks: 952.87890625 MB +worker1-1 | 2024/06/04 17:14:08.877 FIG10b: Memory usage without flush after 22778 tasks: 960.58203125 MB +worker1-1 | 2024/06/04 17:14:09.8902024/06/04 17:14:09.890 FIG10b: Memory usage without flush after 22916 tasks: 960.83203125 MB +worker1-1 | stored...2024/06/04 17:14:10.890 FIG10b: Memory usage without flush after 23064 tasks: 959.015625 MB +worker1-1 | 2024/06/04 17:14:11.895 FIG10b: Memory usage without flush after 23200 tasks: 960.03125 MB +worker1-1 | 2024/06/04 17:14:12.898 FIG10b: Memory usage without flush after 23341 tasks: 959.296875 MB +worker1-1 | 2024/06/04 17:14:13.900 FIG10b: Memory usage without flush after 23467 tasks: 959.8125 MB +worker1-1 | 2024/06/04 17:14:14.904 FIG10b: Memory usage without flush after 23623 tasks: 961.9375 MB +worker1-1 | 2024/06/04 17:14:15.944 FIG10b: Memory usage without flush after 23780 tasks: 962.25 MB +worker1-1 | [worker]2024/06/04 17:14:16.950 FIG10b: Memory usage without flush after 23890 tasks: 964.62109375 MB +worker1-1 | stored...2024/06/04 17:14:17.9542024/06/04 17:14:17.9542024/06/04 17:14:17.954 FIG10b: Memory usage without flush after 24017 tasks: 960.94140625 MB +worker1-1 | 2024/06/04 17:14:18.961 FIG10b: Memory usage without flush after 24174 tasks: 965.21875 MB +worker1-1 | 2024/06/04 17:14:19.9652024/06/04 17:14:19.966 FIG10b: Memory usage without flush after 24288 tasks: 973.1640625 MB +worker1-1 | 2024/06/04 17:14:20.968 FIG10b: Memory usage without flush after 24411 tasks: 972.45703125 MB +worker1-1 | 2024/06/04 17:14:21.981 FIG10b: Memory usage without flush after 24550 tasks: 972.38671875 MB +worker1-1 | 2024/06/04 17:14:22.984 FIG10b: Memory usage without flush after 24688 tasks: 973.90625 MB +worker1-1 | 2024/06/04 17:14:23.9872024/06/04 17:14:23.987 FIG10b: Memory usage without flush after 24821 tasks: 980.11328125 MB +worker1-1 | 2024/06/04 17:14:24.989 FIG10b: Memory usage without flush after 24953 tasks: 984.48828125 MB +worker1-1 | 2024/06/04 17:14:25.994 FIG10b: Memory usage without flush after 25094 tasks: 985.94921875 MB +worker1-1 | 2024/06/04 17:14:26.994 FIG10b: Memory usage without flush after 25245 tasks: 985.22265625 MB +worker1-1 | 2024/06/04 17:14:27.997 2024/06/04 17:14:27.997 FIG10b: Memory usage without flush after 25379 tasks: 986.171875 MB +worker1-1 | 2024/06/04 17:14:29.000 2024/06/04 17:14:29.000[worker] FIG10b: Memory usage without flush after 25512 tasks: 988.28515625 MB +worker1-1 | 2024/06/04 17:14:30.004 FIG10b: Memory usage without flush after 25657 tasks: 987.83984375 MB +worker1-1 | 2024/06/04 17:14:31.006 FIG10b: Memory usage without flush after 25792 tasks: 989.4453125 MB +worker1-1 | 2024/06/04 17:14:32.0112024/06/04 17:14:32.012 FIG10b: Memory usage without flush after 25904 tasks: 989.56640625 MB +worker1-1 | 2024/06/04 17:14:33.016 FIG10b: Memory usage without flush after 26035 tasks: 991.87109375 MB +worker1-1 | 2024/06/04 17:14:34.018 FIG10b: Memory usage without flush after 26175 tasks: 987.86328125 MB +worker1-1 | [worker][worker]2024/06/04 17:14:35.024 FIG10b: Memory usage without flush after 26321 tasks: 988.51171875 MB +worker1-1 | 2024/06/04 17:14:36.030 [worker]2024/06/04 17:14:36.030 FIG10b: Memory usage without flush after 26463 tasks: 989.91796875 MB +worker1-1 | [worker]2024/06/04 17:14:37.039 FIG10b: Memory usage without flush after 26598 tasks: 989.5234375 MB +worker1-1 | 2024/06/04 17:14:38.0472024/06/04 17:14:38.047[worker] 2024/06/04 17:14:38.048 FIG10b: Memory usage without flush after 26735 tasks: 990.3125 MB +worker1-1 | 2024/06/04 17:14:39.051 FIG10b: Memory usage without flush after 26885 tasks: 990.1796875 MB +worker1-1 | 2024/06/04 17:14:40.059 FIG10b: Memory usage without flush after 27007 tasks: 992.046875 MB +worker1-1 | FIG10b: Memory usage without flush after 27135 tasks: 991.65234375 MB +worker1-1 | 2024/06/04 17:14:42.063 2024/06/04 17:14:42.0632024/06/04 17:14:42.063 FIG10b: Memory usage without flush after 27258 tasks: 992.27734375 MB +worker1-1 | 2024/06/04 17:14:43.066 FIG10b: Memory usage without flush after 27388 tasks: 993.0703125 MB +worker1-1 | 2024/06/04 17:14:44.0702024/06/04 17:14:44.071 FIG10b: Memory usage without flush after 27525 tasks: 992.69921875 MB +worker1-1 | 2024/06/04 17:14:45.072 FIG10b: Memory usage without flush after 27664 tasks: 992.83984375 MB +worker1-1 | 2024/06/04 17:14:46.079 2024/06/04 17:14:46.080 FIG10b: Memory usage without flush after 27793 tasks: 993.9609375 MB +worker1-1 | 2024/06/04 17:14:47.084 2024/06/04 17:14:47.085 FIG10b: Memory usage without flush after 27925 tasks: 996.33203125 MB +worker1-1 | 2024/06/04 17:14:48.097 FIG10b: Memory usage without flush after 28056 tasks: 996.55078125 MB +worker1-1 | 2024/06/04 17:14:49.107 FIG10b: Memory usage without flush after 28188 tasks: 997.78125 MB +worker1-1 | 2024/06/04 17:14:50.107 2024/06/04 17:14:50.107 FIG10b: Memory usage without flush after 28327 tasks: 998.1953125 MB +worker1-1 | 2024/06/04 17:14:51.113 FIG10b: Memory usage without flush after 28465 tasks: 999.609375 MB +worker1-1 | 2024/06/04 17:14:52.114 2024/06/04 17:14:52.114FIG10b: Memory usage without flush after 28601 tasks: 1001.1484375 MB +worker1-1 | 2024/06/04 17:14:53.121 FIG10b: Memory usage without flush after 28735 tasks: 1001.94140625 MB +worker1-1 | 2024/06/04 17:14:54.1212024/06/04 17:14:54.121 FIG10b: Memory usage without flush after 28882 tasks: 1001.57421875 MB +worker1-1 | 2024/06/04 17:14:55.1282024/06/04 17:14:55.129 FIG10b: Memory usage without flush after 29017 tasks: 998.4765625 MB +worker1-1 | 2024/06/04 17:14:56.132 FIG10b: Memory usage without flush after 29170 tasks: 999.78125 MB +worker1-1 | 2024/06/04 17:14:57.134 FIG10b: Memory usage without flush after 29320 tasks: 1003.5390625 MB +worker1-1 | 2024/06/04 17:14:58.139 FIG10b: Memory usage without flush after 29475 tasks: 1003.18359375 MB +worker1-1 | 2024/06/04 17:14:59.1422024/06/04 17:14:59.143 FIG10b: Memory usage without flush after 29611 tasks: 1001.6875 MB +worker1-1 | 2024/06/04 17:15:00.1482024/06/04 17:15:00.149 FIG10b: Memory usage without flush after 29740 tasks: 1002.1796875 MB +worker1-1 | 2024/06/04 17:15:01.152 FIG10b: Memory usage without flush after 29871 tasks: 1002.89453125 MB +worker1-1 | 2024/06/04 17:15:02.159 FIG10b: Memory usage without flush after 30003 tasks: 1002.61328125 MB +worker1-1 | 2024/06/04 17:15:03.1652024/06/04 17:15:03.165 FIG10b: Memory usage without flush after 30126 tasks: 1003.5625 MB +worker1-1 | 2024/06/04 17:15:04.173 FIG10b: Memory usage without flush after 30264 tasks: 1007.06640625 MB +worker1-1 | 2024/06/04 17:15:05.1752024/06/04 17:15:05.175 FIG10b: Memory usage without flush after 30402 tasks: 1001.265625 MB +worker1-1 | 2024/06/04 17:15:06.178 FIG10b: Memory usage without flush after 30538 tasks: 1000.1328125 MB +worker1-1 | 2024/06/04 17:15:07.182 FIG10b: Memory usage without flush after 30687 tasks: 1002.62890625 MB +worker1-1 | 2024/06/04 17:15:08.185 FIG10b: Memory usage without flush after 30824 tasks: 1003.28515625 MB +worker1-1 | 2024/06/04 17:15:09.185 FIG10b: Memory usage without flush after 30951 tasks: 1005.31640625 MB +worker1-1 | 2024/06/04 17:15:10.191 FIG10b: Memory usage without flush after 31091 tasks: 1003.46484375 MB +worker1-1 | 2024/06/04 17:15:11.204 FIG10b: Memory usage without flush after 31237 tasks: 1012.5546875 MB +worker1-1 | 2024/06/04 17:15:12.209 FIG10b: Memory usage without flush after 31380 tasks: 1014.01171875 MB2024/06/04 17:15:12.209 +worker1-1 | 2024/06/04 17:15:13.211 FIG10b: Memory usage without flush after 31522 tasks: 1014.55078125 MB +worker1-1 | 2024/06/04 17:15:14.220 [worker]2024/06/04 17:15:14.220 FIG10b: Memory usage without flush after 31666 tasks: 1017.1484375 MB +worker1-1 | 2024/06/04 17:15:15.221 FIG10b: Memory usage without flush after 31812 tasks: 1015.99609375 MB +worker1-1 | [worker] 2024/06/04 17:15:16.2242024/06/04 17:15:16.225 FIG10b: Memory usage without flush after 31947 tasks: 1018.33984375 MB +worker1-1 | 2024/06/04 17:15:17.234 FIG10b: Memory usage without flush after 32078 tasks: 1020.75 MB +worker1-1 | 2024/06/04 17:15:18.2352024/06/04 17:15:18.235 FIG10b: Memory usage without flush after 32218 tasks: 1022.55859375 MB +worker1-1 | 2024/06/04 17:15:19.236 FIG10b: Memory usage without flush after 32362 tasks: 1023.15625 MB +worker1-1 | 2024/06/04 17:15:20.239 FIG10b: Memory usage without flush after 32504 tasks: 1023.6484375 MB +worker1-1 | 2024/06/04 17:15:21.2422024/06/04 17:15:21.243 [worker] FIG10b: Memory usage without flush after 32650 tasks: 1024.34765625 MBExecuted! +worker1-1 | 2024/06/04 17:15:22.244 FIG10b: Memory usage without flush after 32795 tasks: 1026.296875 MB +worker1-1 | 2024/06/04 17:15:23.250 FIG10b: Memory usage without flush after 32937 tasks: 1028.3828125 MB +worker1-1 | 2024/06/04 17:15:24.256 FIG10b: Memory usage without flush after 33068 tasks: 1030.3125 MB +worker1-1 | [worker] stored...2024/06/04 17:15:25.266 FIG10b: Memory usage without flush after 33210 tasks: 1029.61328125 MB +worker1-1 | 2024/06/04 17:15:26.274 FIG10b: Memory usage without flush after 33338 tasks: 1028.609375 MB +worker1-1 | 2024/06/04 17:15:27.2822024/06/04 17:15:27.282 FIG10b: Memory usage without flush after 33480 tasks: 1027.95703125 MB +worker1-1 | 2024/06/04 17:15:28.292 FIG10b: Memory usage without flush after 33621 tasks: 1028.49609375 MB +worker1-1 | 2024/06/04 17:15:29.3032024/06/04 17:15:29.303 FIG10b: Memory usage without flush after 33753 tasks: 1027.9921875 MB +worker1-1 | [worker]2024/06/04 17:15:30.304 FIG10b: Memory usage without flush after 33911 tasks: 1030.65625 MB +worker1-1 | 2024/06/04 17:15:31.306 FIG10b: Memory usage without flush after 34036 tasks: 1031.43359375 MB +worker1-1 | FIG10b: Memory usage without flush after 34185 tasks: 1032.24609375 MB +worker1-1 | 2024/06/04 17:15:33.324 FIG10b: Memory usage without flush after 34322 tasks: 1032.0 MB +worker1-1 | 2024/06/04 17:15:34.326 FIG10b: Memory usage without flush after 34462 tasks: 1032.36328125 MB +worker1-1 | 2024/06/04 17:15:35.326 2024/06/04 17:15:35.327 FIG10b: Memory usage without flush after 34594 tasks: 1033.53125 MB +worker1-1 | 2024/06/04 17:15:36.327 FIG10b: Memory usage without flush after 34736 tasks: 1032.14453125 MB +worker1-1 | [worker]in Run() rn 2024/06/04 17:15:37.331 FIG10b: Memory usage without flush after 34871 tasks: 1031.0546875 MB +worker1-1 | 2024/06/04 17:15:38.339 FIG10b: Memory usage without flush after 35021 tasks: 1033.36328125 MB +worker1-1 | 2024/06/04 17:15:39.3482024/06/04 17:15:39.349 FIG10b: Memory usage without flush after 35166 tasks: 1032.73828125 MB +worker1-1 | 2024/06/04 17:15:40.352 FIG10b: Memory usage without flush after 35261 tasks: 1029.91796875 MB +worker1-1 | 2024/06/04 17:15:41.352 FIG10b: Memory usage without flush after 35391 tasks: 1027.015625 MB +worker1-1 | gonna store now2024/06/04 17:15:42.364 FIG10b: Memory usage without flush after 35543 tasks: 1025.765625 MB +worker1-1 | 2024/06/04 17:15:43.367 FIG10b: Memory usage without flush after 35694 tasks: 1032.625 MB +worker1-1 | 2024/06/04 17:15:44.376 FIG10b: Memory usage without flush after 35830 tasks: 1038.8828125 MB +worker1-1 | 2024/06/04 17:15:45.498[worker] stored...FIG10b: Memory usage without flush after 35970 tasks: 1040.40234375 MB +worker1-1 | 2024/06/04 17:15:46.502 FIG10b: Memory usage without flush after 36112 tasks: 1041.7421875 MB +worker1-1 | 2024/06/04 17:15:47.5042024/06/04 17:15:47.5052024/06/04 17:15:47.505 FIG10b: Memory usage without flush after 36268 tasks: 1035.16796875 MB +worker1-1 | 2024/06/04 17:15:48.5092024/06/04 17:15:48.509 2024/06/04 17:15:48.510 FIG10b: Memory usage without flush after 36393 tasks: 1038.96484375 MB +worker1-1 | 2024/06/04 17:15:49.512 [worker]2024/06/04 17:15:49.513 FIG10b: Memory usage without flush after 36532 tasks: 1039.12890625 MB +worker1-1 | 2024/06/04 17:15:50.519 FIG10b: Memory usage without flush after 36667 tasks: 1039.62109375 MB +worker1-1 | 2024/06/04 17:15:51.522 FIG10b: Memory usage without flush after 36814 tasks: 1039.44921875 MB +worker1-1 | 2024/06/04 17:15:52.524 FIG10b: Memory usage without flush after 36942 tasks: 1041.8984375 MB +worker1-1 | FIG10b: Memory usage without flush after 37074 tasks: 1042.37109375 MB +worker1-1 | 2024/06/04 17:15:54.541 FIG10b: Memory usage without flush after 37204 tasks: 1039.86328125 MB +worker1-1 | 2024/06/04 17:15:55.548 FIG10b: Memory usage without flush after 37342 tasks: 1036.9609375 MB +worker1-1 | 2024/06/04 17:15:56.552 FIG10b: Memory usage without flush after 37472 tasks: 1038.1953125 MB +worker1-1 | 2024/06/04 17:15:57.563 FIG10b: Memory usage without flush after 37612 tasks: 1038.1171875 MB +worker1-1 | 2024/06/04 17:15:58.577 FIG10b: Memory usage without flush after 37740 tasks: 1039.08984375 MB +worker1-1 | [worker]2024/06/04 17:15:59.582 FIG10b: Memory usage without flush after 37885 tasks: 1041.8984375 MB +worker1-1 | 2024/06/04 17:16:00.588 FIG10b: Memory usage without flush after 38018 tasks: 1045.99609375 MB +worker1-1 | 2024/06/04 17:16:01.5942024/06/04 17:16:01.594 FIG10b: Memory usage without flush after 38155 tasks: 1048.20703125 MB +worker1-1 | Executed!2024/06/04 17:16:02.597 FIG10b: Memory usage without flush after 38297 tasks: 1046.03515625 MB +worker1-1 | 2024/06/04 17:16:03.597 FIG10b: Memory usage without flush after 38446 tasks: 1052.859375 MB +worker1-1 | 2024/06/04 17:16:04.600 FIG10b: Memory usage without flush after 38574 tasks: 1053.140625 MB +worker1-1 | 2024/06/04 17:16:05.602 FIG10b: Memory usage without flush after 38708 tasks: 1055.57421875 MB +worker1-1 | 2024/06/04 17:16:06.605 FIG10b: Memory usage without flush after 38842 tasks: 1054.19921875 MB +worker1-1 | 2024/06/04 17:16:07.6082024/06/04 17:16:07.609 FIG10b: Memory usage without flush after 38979 tasks: 1060.4921875 MB +worker1-1 | 2024/06/04 17:16:08.6222024/06/04 17:16:08.623 FIG10b: Memory usage without flush after 39112 tasks: 1065.421875 MB +worker1-1 | 2024/06/04 17:16:09.632 FIG10b: Memory usage without flush after 39233 tasks: 1069.72265625 MB +worker1-1 | 2024/06/04 17:16:10.634 FIG10b: Memory usage without flush after 39369 tasks: 1069.12890625 MB +worker1-1 | 2024/06/04 17:16:11.636 FIG10b: Memory usage without flush after 39500 tasks: 1071.12890625 MB +worker1-1 | 2024/06/04 17:16:12.638 FIG10b: Memory usage without flush after 39640 tasks: 1072.25 MB +worker1-1 | 2024/06/04 17:16:13.640 FIG10b: Memory usage without flush after 39768 tasks: 1081.2890625 MB +worker1-1 | 2024/06/04 17:16:14.644 FIG10b: Memory usage without flush after 39923 tasks: 1080.45703125 MB +worker1-1 | 2024/06/04 17:16:15.646 FIG10b: Memory usage without flush after 40079 tasks: 1086.22265625 MB +worker1-1 | 2024/06/04 17:16:16.648 FIG10b: Memory usage without flush after 40228 tasks: 1088.953125 MB +worker1-1 | 2024/06/04 17:16:46.651 FIG10b: Memory usage without flush after 44386 tasks: 1117.3046875 MB +worker1-1 | 2024/06/04 17:17:16.6552024/06/04 17:17:16.655 [worker] in Run() rnFIG10b: Memory usage without flush after 48572 tasks: 1165.1796875 MB +worker1-1 | 2024/06/04 17:17:46.666 FIG10b: Memory usage without flush after 52603 tasks: 1202.19921875 MB +worker1-1 | 2024/06/04 17:18:16.6612024/06/04 17:18:16.661 FIG10b: Memory usage without flush after 56539 tasks: 1269.734375 MB +worker1-1 | 2024/06/04 17:18:46.664 FIG10b: Memory usage without flush after 60671 tasks: 1289.60546875 MB +worker1-1 | 2024/06/04 17:19:16.666 FIG10b: Memory usage without flush after 64929 tasks: 1303.59375 MB +worker1-1 | 2024/06/04 17:19:46.668 FIG10b: Memory usage without flush after 68960 tasks: 1330.65625 MB +worker1-1 | 2024/06/04 17:20:16.673 FIG10b: Memory usage without flush after 72910 tasks: 1324.15234375 MB +worker1-1 | 2024/06/04 17:20:46.674 FIG10b: Memory usage without flush after 77160 tasks: 1337.40625 MB +worker1-1 | 2024/06/04 17:21:16.676 FIG10b: Memory usage without flush after 81358 tasks: 1349.7890625 MB +worker1-1 | 2024/06/04 17:21:46.677 FIG10b: Memory usage without flush after 85642 tasks: 1381.5703125 MB +worker1-1 | 2024/06/04 17:22:16.684 FIG10b: Memory usage without flush after 89999 tasks: 1412.875 MB +worker1-1 | 2024/06/04 17:22:46.688 FIG10b: Memory usage without flush after 94223 tasks: 1424.2890625 MB +worker1-1 | 2024/06/04 17:23:16.688 FIG10b: Memory usage without flush after 98242 tasks: 1456.50390625 MB +worker1-1 | 2024/06/04 17:23:46.690 FIG10b: Memory usage without flush after 102675 tasks: 1455.07421875 MB +worker1-1 | 2024/06/04 17:24:16.692 FIG10b: Memory usage without flush after 106843 tasks: 1491.23046875 MB +worker1-1 | 2024/06/04 17:24:46.695 FIG10b: Memory usage without flush after 110260 tasks: 1501.765625 MB +worker1-1 | 2024/06/04 17:25:16.695 FIG10b: Memory usage without flush after 114276 tasks: 1513.46875 MB +worker1-1 | 2024/06/04 17:25:46.699 FIG10b: Memory usage without flush after 118268 tasks: 1487.48828125 MB +worker1-1 | 2024/06/04 17:26:16.702 FIG10b: Memory usage without flush after 122774 tasks: 1504.36328125 MB +worker1-1 | 2024/06/04 17:26:46.706 FIG10b: Memory usage without flush after 126885 tasks: 1515.48046875 MB +worker1-1 | 2024/06/04 17:27:16.710 FIG10b: Memory usage without flush after 131135 tasks: 1559.76171875 MB +worker1-1 | 2024/06/04 17:27:46.713 FIG10b: Memory usage without flush after 135333 tasks: 1614.8671875 MB +worker1-1 | 2024/06/04 17:28:16.715 FIG10b: Memory usage without flush after 139528 tasks: 1599.36328125 MB +worker1-1 | 2024/06/04 17:28:46.721 FIG10b: Memory usage without flush after 143095 tasks: 1575.1171875 MB +worker1-1 | 2024/06/04 17:29:16.721 FIG10b: Memory usage without flush after 146849 tasks: 1575.37890625 MB +worker1-1 | 2024/06/04 17:29:46.724 FIG10b: Memory usage without flush after 150948 tasks: 1580.12109375 MB +worker1-1 | 2024/06/04 17:30:16.730 FIG10b: Memory usage without flush after 155098 tasks: 1594.60546875 MB +worker1-1 | 2024/06/04 17:30:46.732 FIG10b: Memory usage without flush after 159007 tasks: 1606.80078125 MB +worker1-1 | 2024/06/04 17:31:16.734 FIG10b: Memory usage without flush after 162942 tasks: 1613.8203125 MB +worker1-1 | 2024/06/04 17:31:46.738 FIG10b: Memory usage without flush after 166558 tasks: 1650.86328125 MB +worker1-1 | 2024/06/04 17:32:16.744 FIG10b: Memory usage without flush after 170461 tasks: 1624.8828125 MB +worker1-1 | 2024/06/04 17:32:46.746 FIG10b: Memory usage without flush after 173707 tasks: 1655.79296875 MB +worker1-1 | 2024/06/04 17:33:16.748 FIG10b: Memory usage without flush after 177633 tasks: 1629.8515625 MB +worker1-1 | 2024/06/04 17:33:46.749 FIG10b: Memory usage without flush after 181601 tasks: 1632.0 MB +worker1-1 | 2024/06/04 17:34:16.752 FIG10b: Memory usage without flush after 185073 tasks: 1665.92578125 MB +worker1-1 | 2024/06/04 17:34:46.755 FIG10b: Memory usage without flush after 189301 tasks: 1641.40625 MB +worker1-1 | 2024/06/04 17:35:16.757 FIG10b: Memory usage without flush after 193460 tasks: 1651.3671875 MB +worker1-1 | 2024/06/04 17:35:46.762 FIG10b: Memory usage without flush after 197325 tasks: 1679.046875 MB +worker1-1 | 2024/06/04 17:36:16.764 FIG10b: Memory usage without flush after 201556 tasks: 1700.41796875 MB +worker1-1 | 2024/06/04 17:36:46.764 FIG10b: Memory usage without flush after 206485 tasks: 1742.9765625 MB +worker1-1 | 2024/06/04 17:37:16.765 FIG10b: Memory usage without flush after 209976 tasks: 1769.1015625 MB +worker1-1 | 2024/06/04 17:37:46.768 FIG10b: Memory usage without flush after 214258 tasks: 1834.1015625 MB +worker1-1 | 2024/06/04 17:38:16.772 FIG10b: Memory usage without flush after 218638 tasks: 1843.01953125 MB +worker1-1 | 2024/06/04 17:38:46.777 FIG10b: Memory usage without flush after 222805 tasks: 1836.50390625 MB +worker1-1 | 2024/06/04 17:39:16.780 FIG10b: Memory usage without flush after 227560 tasks: 1847.5 MB +worker1-1 | 2024/06/04 17:39:46.784 FIG10b: Memory usage without flush after 232082 tasks: 1855.3125 MB +worker1-1 | 2024/06/04 17:40:16.791 FIG10b: Memory usage without flush after 236496 tasks: 1852.5625 MB +worker1-1 | 2024/06/04 17:40:46.793 FIG10b: Memory usage without flush after 240574 tasks: 1874.36328125 MB +worker1-1 | 2024/06/04 17:41:16.795 FIG10b: Memory usage without flush after 245130 tasks: 1870.08203125 MB +worker1-1 | 2024/06/04 17:41:46.798 FIG10b: Memory usage without flush after 249358 tasks: 1878.80078125 MB +worker1-1 | 2024/06/04 17:42:16.798 FIG10b: Memory usage without flush after 253381 tasks: 1861.08203125 MB +worker1-1 | 2024/06/04 17:42:46.816 FIG10b: Memory usage without flush after 257740 tasks: 1869.875 MB +worker1-1 | 2024/06/04 17:43:16.819 FIG10b: Memory usage without flush after 261253 tasks: 1898.765625 MB +worker1-1 | 2024/06/04 17:43:46.822 FIG10b: Memory usage without flush after 265729 tasks: 1860.5234375 MB +worker1-1 | 2024/06/04 17:44:16.828 FIG10b: Memory usage without flush after 270384 tasks: 1857.2265625 MB +worker1-1 | 2024/06/04 17:44:46.829 FIG10b: Memory usage without flush after 274364 tasks: 1858.05859375 MB +worker1-1 | 2024/06/04 17:45:16.830 FIG10b: Memory usage without flush after 278751 tasks: 1870.82421875 MB +worker1-1 | 2024/06/04 17:45:46.835 FIG10b: Memory usage without flush after 283240 tasks: 1900.47265625 MB +worker1-1 | 2024/06/04 17:46:16.837 FIG10b: Memory usage without flush after 287670 tasks: 1909.76953125 MB diff --git a/analysis/output_10b_run5_buggy?_1s_flush_use_AOF_no_garbage_yet.txt b/analysis/output_10b_run5_buggy?_1s_flush_use_AOF_no_garbage_yet.txt new file mode 100644 index 0000000..3bba31b --- /dev/null +++ b/analysis/output_10b_run5_buggy?_1s_flush_use_AOF_no_garbage_yet.txt @@ -0,0 +1,314 @@ +worker1-1 | 2024/06/04 17:51:35.961 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 17:51:36.000 FIG10b: Starting tasks submission... +worker1-1 | 2024/06/04 17:51:37.003 FIG10b: Memory usage without flush after 111 tasks: 1066.48046875 MB +worker1-1 | 2024/06/04 17:51:38.0042024/06/04 17:51:38.005 FIG10b: Memory usage without flush after 225 tasks: 1080.74609375 MB +worker1-1 | Executed!2024/06/04 17:51:39.009 FIG10b: Memory usage without flush after 341 tasks: 1081.58203125 MB +worker1-1 | 2024/06/04 17:51:40.011 FIG10b: Memory usage without flush after 461 tasks: 1082.59765625 MB +worker1-1 | 2024/06/04 17:51:41.024 FIG10b: Memory usage without flush after 575 tasks: 1087.3671875 MB +worker1-1 | FIG10b: Memory usage without flush after 686 tasks: 1092.6953125 MB +worker1-1 | FIG10b: Memory usage without flush after 801 tasks: 1094.02734375 MB +worker1-1 | 2024/06/04 17:51:44.051 FIG10b: Memory usage without flush after 905 tasks: 1093.44921875 MB +worker1-1 | 2024/06/04 17:51:45.053 FIG10b: Memory usage without flush after 1009 tasks: 1094.83984375 MB +worker1-1 | 2024/06/04 17:51:46.054 FIG10b: Memory usage without flush after 1105 tasks: 1096.97265625 MB +worker1-1 | 2024/06/04 17:51:47.062 FIG10b: Memory usage without flush after 1226 tasks: 1100.5546875 MB +worker1-1 | [worker]2024/06/04 17:51:48.063 FIG10b: Memory usage without flush after 1343 tasks: 1101.35546875 MB +worker1-1 | 2024/06/04 17:51:49.071 FIG10b: Memory usage without flush after 1451 tasks: 1103.0859375 MB +worker1-1 | 2024/06/04 17:51:50.074 FIG10b: Memory usage without flush after 1577 tasks: 1103.578125 MB +worker1-1 | [worker] 2024/06/04 17:51:51.076 FIG10b: Memory usage without flush after 1696 tasks: 1104.5546875 MB +worker1-1 | 2024/06/04 17:51:52.077 FIG10b: Memory usage without flush after 1803 tasks: 1105.09375 MB +worker1-1 | 2024/06/04 17:51:53.079 FIG10b: Memory usage without flush after 1927 tasks: 1104.55078125 MB +worker1-1 | [worker] 2024/06/04 17:51:54.079 FIG10b: Memory usage without flush after 2036 tasks: 1105.546875 MB +worker1-1 | [worker] stored...2024/06/04 17:51:55.113 FIG10b: Memory usage without flush after 2151 tasks: 1110.0390625 MB +worker1-1 | stored...2024/06/04 17:51:56.116 FIG10b: Memory usage without flush after 2238 tasks: 1110.578125 MB +worker1-1 | 2024/06/04 17:51:57.119 FIG10b: Memory usage without flush after 2354 tasks: 1109.95703125 MB +worker1-1 | 2024/06/04 17:51:58.1212024/06/04 17:51:58.121 FIG10b: Memory usage without flush after 2437 tasks: 1109.3125 MB +worker1-1 | 2024/06/04 17:51:59.125 FIG10b: Memory usage without flush after 2521 tasks: 1111.52734375 MB +worker1-1 | 2024/06/04 17:52:00.1282024/06/04 17:52:00.128 [worker] stored...FIG10b: Memory usage without flush after 2620 tasks: 1112.5 MB +worker1-1 | 2024/06/04 17:52:01.130 FIG10b: Memory usage without flush after 2730 tasks: 1113.73046875 MB +worker1-1 | [worker]2024/06/04 17:52:02.134in Run() rn FIG10b: Memory usage without flush after 2854 tasks: 1113.42578125 MB +worker1-1 | 2024/06/04 17:52:03.1382024/06/04 17:52:03.138 FIG10b: Memory usage without flush after 2975 tasks: 1113.96484375 MB +worker1-1 | 2024/06/04 17:52:04.146 FIG10b: Memory usage without flush after 3098 tasks: 1113.765625 MB +worker1-1 | 2024/06/04 17:52:05.1492024/06/04 17:52:05.149 FIG10b: Memory usage without flush after 3217 tasks: 1115.2421875 MB +worker1-1 | [worker] 2024/06/04 17:52:06.1582024/06/04 17:52:06.158 FIG10b: Memory usage without flush after 3341 tasks: 1115.1484375 MB +worker1-1 | 2024/06/04 17:52:07.161 2024/06/04 17:52:07.162 FIG10b: Memory usage without flush after 3435 tasks: 1119.1953125 MB +worker1-1 | Executed! 2024/06/04 17:52:08.170 FIG10b: Memory usage without flush after 3562 tasks: 1119.265625 MB +worker1-1 | 2024/06/04 17:52:09.179 FIG10b: Memory usage without flush after 3686 tasks: 1120.51953125 MB +worker1-1 | 2024/06/04 17:52:10.187 FIG10b: Memory usage without flush after 3813 tasks: 1120.15234375 MB +worker1-1 | in Run() rn2024/06/04 17:52:11.188 FIG10b: Memory usage without flush after 3934 tasks: 1120.69140625 MB +worker1-1 | 2024/06/04 17:52:12.204 FIG10b: Memory usage without flush after 4040 tasks: 1123.9296875 MB +worker1-1 | 2024/06/04 17:52:13.204 FIG10b: Memory usage without flush after 4158 tasks: 1125.33984375 MB +worker1-1 | [worker] 2024/06/04 17:52:14.206 FIG10b: Memory usage without flush after 4279 tasks: 1126.125 MB +worker1-1 | 2024/06/04 17:52:15.2102024/06/04 17:52:15.210 FIG10b: Memory usage without flush after 4398 tasks: 1126.41796875 MB +worker1-1 | FIG10b: Memory usage without flush after 4509 tasks: 1126.48828125 MB +worker1-1 | 2024/06/04 17:52:17.217 FIG10b: Memory usage without flush after 4636 tasks: 1127.2734375 MB +worker1-1 | 2024/06/04 17:52:18.222 2024/06/04 17:52:18.223 FIG10b: Memory usage without flush after 4761 tasks: 1127.14453125 MB +worker1-1 | 2024/06/04 17:52:19.227 FIG10b: Memory usage without flush after 4884 tasks: 1126.45703125 MB +worker1-1 | [worker]2024/06/04 17:52:20.229[worker]2024/06/04 17:52:20.229 FIG10b: Memory usage without flush after 5006 tasks: 1126.51953125 MB +worker1-1 | 2024/06/04 17:52:21.233 FIG10b: Memory usage without flush after 5128 tasks: 1126.5625 MB +worker1-1 | took this many seconds: 0.01422024/06/04 17:52:22.244 FIG10b: Memory usage without flush after 5247 tasks: 1127.1328125 MB +worker1-1 | 2024/06/04 17:52:23.2582024/06/04 17:52:23.258 FIG10b: Memory usage without flush after 5383 tasks: 1128.5703125 MB +worker1-1 | 2024/06/04 17:52:24.259 FIG10b: Memory usage without flush after 5510 tasks: 1128.6015625 MB +worker1-1 | 2024/06/04 17:52:25.2672024/06/04 17:52:25.267 FIG10b: Memory usage without flush after 5644 tasks: 1129.32421875 MB +worker1-1 | 2024/06/04 17:52:26.273 FIG10b: Memory usage without flush after 5764 tasks: 1129.88671875 MB +worker1-1 | [worker]gonna store now2024/06/04 17:52:27.274 FIG10b: Memory usage without flush after 5887 tasks: 1129.6875 MB +worker1-1 | 2024/06/04 17:52:28.2752024/06/04 17:52:28.275 2024/06/04 17:52:28.276 FIG10b: Memory usage without flush after 6014 tasks: 1129.9609375 MB +worker1-1 | 2024/06/04 17:52:29.281 FIG10b: Memory usage without flush after 6145 tasks: 1130.45703125 MB +worker1-1 | took this many seconds: 0.01622024/06/04 17:52:30.288 FIG10b: Memory usage without flush after 6271 tasks: 1130.7109375 MB +worker1-1 | 2024/06/04 17:52:31.297 FIG10b: Memory usage without flush after 6396 tasks: 1130.2578125 MB +worker1-1 | [worker]in Run() rn 2024/06/04 17:52:32.304 FIG10b: Memory usage without flush after 6516 tasks: 1130.8046875 MB +worker1-1 | 2024/06/04 17:52:33.305 FIG10b: Memory usage without flush after 6638 tasks: 1131.09375 MB +worker1-1 | [worker] 2024/06/04 17:52:34.305 FIG10b: Memory usage without flush after 6759 tasks: 1131.171875 MB +worker1-1 | 2024/06/04 17:52:35.306 2024/06/04 17:52:35.307 [worker]FIG10b: Memory usage without flush after 6890 tasks: 1131.17578125 MB +worker1-1 | FIG10b: Memory usage without flush after 7021 tasks: 1131.42578125 MB +worker1-1 | 2024/06/04 17:52:37.311 FIG10b: Memory usage without flush after 7131 tasks: 1131.3046875 MB +worker1-1 | 2024/06/04 17:52:38.320 FIG10b: Memory usage without flush after 7258 tasks: 1131.81640625 MB2024/06/04 17:52:38.320 [lobs] STORED!!! +worker1-1 | 2024/06/04 17:52:39.325 FIG10b: Memory usage without flush after 7392 tasks: 1131.5703125 MB +worker1-1 | took this many seconds: 0.0175[worker]2024/06/04 17:52:40.328 FIG10b: Memory usage without flush after 7519 tasks: 1132.1015625 MB +worker1-1 | 2024/06/04 17:52:41.329 [worker]2024/06/04 17:52:41.330 FIG10b: Memory usage without flush after 7644 tasks: 1131.64453125 MB +worker1-1 | 2024/06/04 17:52:42.336 FIG10b: Memory usage without flush after 7769 tasks: 1131.93359375 MB +worker1-1 | 2024/06/04 17:52:43.337 FIG10b: Memory usage without flush after 7866 tasks: 1132.62109375 MB +worker1-1 | 2024/06/04 17:52:44.339 FIG10b: Memory usage without flush after 7964 tasks: 1132.5703125 MB +worker1-1 | [worker]2024/06/04 17:52:45.359 FIG10b: Memory usage without flush after 8064 tasks: 1133.00390625 MB +worker1-1 | 2024/06/04 17:52:46.721 FIG10b: Memory usage without flush after 8078 tasks: 1133.15234375 MB +worker1-1 | 2024/06/04 17:52:47.669 [worker] 2024/06/04 17:52:47.679 FIG10b: Memory usage without flush after 8093 tasks: 1133.328125 MB +worker1-1 | 2024/06/04 17:52:48.682 2024/06/04 17:52:48.683 FIG10b: Memory usage without flush after 8175 tasks: 1132.875 MB +worker1-1 | 2024/06/04 17:52:49.699 FIG10b: Memory usage without flush after 8270 tasks: 1132.80859375 MB +worker1-1 | 2024/06/04 17:52:50.761 FIG10b: Memory usage without flush after 8363 tasks: 1133.30078125 MB +worker1-1 | 2024/06/04 17:52:51.767 FIG10b: Memory usage without flush after 8466 tasks: 1134.76171875 MB +worker1-1 | 2024/06/04 17:52:52.775 FIG10b: Memory usage without flush after 8586 tasks: 1135.74609375 MB +worker1-1 | 2024/06/04 17:52:53.783 FIG10b: Memory usage without flush after 8691 tasks: 1135.81640625 MB +worker1-1 | 2024/06/04 17:52:54.784 FIG10b: Memory usage without flush after 8799 tasks: 1135.265625 MB +worker1-1 | 2024/06/04 17:52:55.788 FIG10b: Memory usage without flush after 8908 tasks: 1135.0546875 MB +worker1-1 | [worker]2024/06/04 17:52:56.792 FIG10b: Memory usage without flush after 9029 tasks: 1136.5078125 MB +worker1-1 | stored... 2024/06/04 17:52:57.795 FIG10b: Memory usage without flush after 9152 tasks: 1136.1328125 MB +worker1-1 | FIG10b: Memory usage without flush after 9275 tasks: 1135.734375 MB +worker1-1 | 2024/06/04 17:52:59.800 FIG10b: Memory usage without flush after 9401 tasks: 1136.77734375 MB +worker1-1 | 2024/06/04 17:53:00.803 FIG10b: Memory usage without flush after 9508 tasks: 1137.28125 MB +worker1-1 | 2024/06/04 17:53:01.806 FIG10b: Memory usage without flush after 9622 tasks: 1138.0078125 MB +worker1-1 | 2024/06/04 17:53:02.808 FIG10b: Memory usage without flush after 9749 tasks: 1137.2109375 MB +worker1-1 | 2024/06/04 17:53:03.820 FIG10b: Memory usage without flush after 9878 tasks: 1137.984375 MB +worker1-1 | 2024/06/04 17:53:04.8202024/06/04 17:53:04.821 FIG10b: Memory usage without flush after 10001 tasks: 1138.98046875 MB +worker1-1 | 2024/06/04 17:53:05.8242024/06/04 17:53:05.824 FIG10b: Memory usage without flush after 10122 tasks: 1139.765625 MB +worker1-1 | 2024/06/04 17:53:06.827 FIG10b: Memory usage without flush after 10240 tasks: 1138.9453125 MB +worker1-1 | [worker]2024/06/04 17:53:07.828 FIG10b: Memory usage without flush after 10357 tasks: 1139.26171875 MB +worker1-1 | 2024/06/04 17:53:08.832 FIG10b: Memory usage without flush after 10481 tasks: 1139.578125 MB +worker1-1 | 2024/06/04 17:53:09.841 FIG10b: Memory usage without flush after 10602 tasks: 1139.87109375 MB +worker1-1 | [worker]2024/06/04 17:53:10.843 FIG10b: Memory usage without flush after 10719 tasks: 1140.90234375 MB +worker1-1 | 2024/06/04 17:53:11.849 FIG10b: Memory usage without flush after 10834 tasks: 1141.93359375 MB +worker1-1 | 2024/06/04 17:53:12.859 FIG10b: Memory usage without flush after 10930 tasks: 1141.7578125 MB +worker1-1 | 2024/06/04 17:53:13.862 [worker] 2024/06/04 17:53:13.862 FIG10b: Memory usage without flush after 11014 tasks: 1142.51953125 MB +worker1-1 | [worker] 2024/06/04 17:53:14.866 FIG10b: Memory usage without flush after 11122 tasks: 1142.07421875 MB +worker1-1 | in Run() rn2024/06/04 17:53:15.866 FIG10b: Memory usage without flush after 11255 tasks: 1140.20703125 MB +worker1-1 | 2024/06/04 17:53:16.870 FIG10b: Memory usage without flush after 11366 tasks: 1140.734375 MB +worker1-1 | 2024/06/04 17:53:17.872 FIG10b: Memory usage without flush after 11473 tasks: 1140.51171875 MB +worker1-1 | 2024/06/04 17:53:18.877 FIG10b: Memory usage without flush after 11591 tasks: 1142.72265625 MB +worker1-1 | 2024/06/04 17:53:19.879 [worker]2024/06/04 17:53:19.880 FIG10b: Memory usage without flush after 11693 tasks: 1140.58984375 MB +worker1-1 | 2024/06/04 17:53:20.936 FIG10b: Memory usage without flush after 11792 tasks: 1141.75 MB +worker1-1 | 2024/06/04 17:53:21.938 2024/06/04 17:53:21.938 FIG10b: Memory usage without flush after 11912 tasks: 1142.65625 MB +worker1-1 | 2024/06/04 17:53:22.941 FIG10b: Memory usage without flush after 12036 tasks: 1143.30078125 MB +worker1-1 | 2024/06/04 17:53:23.958 FIG10b: Memory usage without flush after 12161 tasks: 1143.16796875 MB +worker1-1 | 2024/06/04 17:53:24.9622024/06/04 17:53:24.9632024/06/04 17:53:24.963 FIG10b: Memory usage without flush after 12281 tasks: 1143.296875 MB +worker1-1 | 2024/06/04 17:53:25.966 FIG10b: Memory usage without flush after 12399 tasks: 1143.90234375 MB +worker1-1 | 2024/06/04 17:53:26.968 FIG10b: Memory usage without flush after 12510 tasks: 1143.60546875 MB +worker1-1 | 2024/06/04 17:53:27.969in Run() rn FIG10b: Memory usage without flush after 12632 tasks: 1143.95703125 MB +worker1-1 | [worker]2024/06/04 17:53:28.972 FIG10b: Memory usage without flush after 12746 tasks: 1144.84375 MB +worker1-1 | [worker] 2024/06/04 17:53:29.977 FIG10b: Memory usage without flush after 12876 tasks: 1144.6796875 MB +worker1-1 | [worker][worker]2024/06/04 17:53:30.984 FIG10b: Memory usage without flush after 12996 tasks: 1146.03515625 MB +worker1-1 | 2024/06/04 17:53:31.9842024/06/04 17:53:31.984 [worker] FIG10b: Memory usage without flush after 13126 tasks: 1145.625 MBin Run() rn +worker1-1 | 2024/06/04 17:53:32.986 FIG10b: Memory usage without flush after 13234 tasks: 1146.79296875 MB +worker1-1 | 2024/06/04 17:53:33.988 FIG10b: Memory usage without flush after 13349 tasks: 1154.3125 MB +worker1-1 | FIG10b: Memory usage without flush after 13456 tasks: 1154.5703125 MB +worker1-1 | 2024/06/04 17:53:36.004 FIG10b: Memory usage without flush after 13576 tasks: 1156.3984375 MB +worker1-1 | FIG10b: Memory usage without flush after 13698 tasks: 1156.79296875 MB +worker1-1 | 2024/06/04 17:53:38.010 FIG10b: Memory usage without flush after 13827 tasks: 1151.52734375 MB +worker1-1 | 2024/06/04 17:53:39.015 FIG10b: Memory usage without flush after 13957 tasks: 1151.87109375 MB +worker1-1 | 2024/06/04 17:53:40.017 FIG10b: Memory usage without flush after 14074 tasks: 1149.55078125 MB +worker1-1 | 2024/06/04 17:53:41.020 FIG10b: Memory usage without flush after 14199 tasks: 1150.41796875 MB +worker1-1 | 2024/06/04 17:53:42.032 FIG10b: Memory usage without flush after 14326 tasks: 1150.734375 MB +worker1-1 | 2024/06/04 17:53:43.034 FIG10b: Memory usage without flush after 14451 tasks: 1150.578125 MB +worker1-1 | 2024/06/04 17:53:44.035 2024/06/04 17:53:44.035 FIG10b: Memory usage without flush after 14573 tasks: 1150.9375 MB +worker1-1 | 2024/06/04 17:53:45.035 2024/06/04 17:53:45.036 FIG10b: Memory usage without flush after 14696 tasks: 1151.0703125 MB +worker1-1 | 2024/06/04 17:53:46.041 FIG10b: Memory usage without flush after 14809 tasks: 1151.46484375 MB +worker1-1 | in Run() rn2024/06/04 17:53:47.042 FIG10b: Memory usage without flush after 14931 tasks: 1151.5703125 MB +worker1-1 | 2024/06/04 17:53:48.045 FIG10b: Memory usage without flush after 15053 tasks: 1151.9296875 MB +worker1-1 | 2024/06/04 17:53:49.046 FIG10b: Memory usage without flush after 15175 tasks: 1152.3125 MB +worker1-1 | 2024/06/04 17:53:50.047 FIG10b: Memory usage without flush after 15301 tasks: 1144.1484375 MB +worker1-1 | 2024/06/04 17:53:51.051 FIG10b: Memory usage without flush after 15421 tasks: 1145.4765625 MB +worker1-1 | 2024/06/04 17:53:52.055 FIG10b: Memory usage without flush after 15550 tasks: 1146.1015625 MB +worker1-1 | 2024/06/04 17:53:53.062 FIG10b: Memory usage without flush after 15671 tasks: 1146.44921875 MB +worker1-1 | 2024/06/04 17:53:54.063 FIG10b: Memory usage without flush after 15786 tasks: 1146.38671875 MB +worker1-1 | 2024/06/04 17:53:55.072 FIG10b: Memory usage without flush after 15913 tasks: 1146.64453125 MB +worker1-1 | [worker]2024/06/04 17:53:56.077 FIG10b: Memory usage without flush after 16025 tasks: 1145.60546875 MB +worker1-1 | 2024/06/04 17:53:57.089 2024/06/04 17:53:57.090 FIG10b: Memory usage without flush after 16137 tasks: 1146.7109375 MB +worker1-1 | 2024/06/04 17:53:58.091 FIG10b: Memory usage without flush after 16233 tasks: 1146.19140625 MB +worker1-1 | 2024/06/04 17:53:59.094 FIG10b: Memory usage without flush after 16290 tasks: 1147.15625 MB +worker1-1 | 2024/06/04 17:54:00.098 FIG10b: Memory usage without flush after 16404 tasks: 1137.625 MB +worker1-1 | 2024/06/04 17:54:01.105 FIG10b: Memory usage without flush after 16507 tasks: 1137.79296875 MB +worker1-1 | 2024/06/04 17:54:02.120 FIG10b: Memory usage without flush after 16621 tasks: 1138.90625 MB +worker1-1 | 2024/06/04 17:54:03.122 FIG10b: Memory usage without flush after 16739 tasks: 1139.12890625 MB +worker1-1 | 2024/06/04 17:54:04.126 [worker] stored...2024/06/04 17:54:04.1262024/06/04 17:54:04.127 FIG10b: Memory usage without flush after 16849 tasks: 1138.421875 MB +worker1-1 | [worker]2024/06/04 17:54:05.129 FIG10b: Memory usage without flush after 16966 tasks: 1139.22265625 MB +worker1-1 | gonna store now 2024/06/04 17:54:06.1302024/06/04 17:54:06.130 FIG10b: Memory usage without flush after 17089 tasks: 1140.7890625 MB +worker1-1 | stored...2024/06/04 17:54:07.130 FIG10b: Memory usage without flush after 17208 tasks: 1140.51171875 MB +worker1-1 | 2024/06/04 17:54:08.1362024/06/04 17:54:08.137 FIG10b: Memory usage without flush after 17324 tasks: 1140.02734375 MB +worker1-1 | 2024/06/04 17:54:09.150 FIG10b: Memory usage without flush after 17420 tasks: 1139.7578125 MB +worker1-1 | 2024/06/04 17:54:10.160 FIG10b: Memory usage without flush after 17508 tasks: 1141.15234375 MB +worker1-1 | [worker]2024/06/04 17:54:11.170 FIG10b: Memory usage without flush after 17603 tasks: 1139.875 MB +worker1-1 | [worker] 2024/06/04 17:54:12.172 FIG10b: Memory usage without flush after 17700 tasks: 1140.3125 MB +worker1-1 | [worker]2024/06/04 17:54:13.178 FIG10b: Memory usage without flush after 17810 tasks: 1140.9921875 MB +worker1-1 | 2024/06/04 17:54:14.182 FIG10b: Memory usage without flush after 17911 tasks: 1140.7421875 MB +worker1-1 | 2024/06/04 17:54:15.185 FIG10b: Memory usage without flush after 18022 tasks: 1140.453125 MB +worker1-1 | 2024/06/04 17:54:16.186 FIG10b: Memory usage without flush after 18128 tasks: 1141.609375 MB +worker1-1 | 2024/06/04 17:54:17.1862024/06/04 17:54:17.186 FIG10b: Memory usage without flush after 18243 tasks: 1141.55078125 MB +worker1-1 | 2024/06/04 17:54:18.192 2024/06/04 17:54:18.193[worker]2024/06/04 17:54:18.193 FIG10b: Memory usage without flush after 18350 tasks: 1141.2890625 MB +worker1-1 | FIG10b: Memory usage without flush after 18464 tasks: 1141.28125 MB +worker1-1 | 2024/06/04 17:54:20.2002024/06/04 17:54:20.200 FIG10b: Memory usage without flush after 18577 tasks: 1141.4765625 MB +worker1-1 | [worker] 2024/06/04 17:54:21.208 FIG10b: Memory usage without flush after 18696 tasks: 1142.69140625 MB +worker1-1 | 2024/06/04 17:54:22.218 FIG10b: Memory usage without flush after 18816 tasks: 1143.0 MB +worker1-1 | 2024/06/04 17:54:23.219 FIG10b: Memory usage without flush after 18939 tasks: 1143.05859375 MB +worker1-1 | 2024/06/04 17:54:24.227in Run() rn2024/06/04 17:54:24.228 FIG10b: Memory usage without flush after 19052 tasks: 1143.6640625 MB +worker1-1 | 2024/06/04 17:54:25.231 FIG10b: Memory usage without flush after 19178 tasks: 1143.70703125 MB +worker1-1 | stored...2024/06/04 17:54:26.234 FIG10b: Memory usage without flush after 19301 tasks: 1144.0078125 MB +worker1-1 | 2024/06/04 17:54:27.2532024/06/04 17:54:27.253 [worker] Executed!FIG10b: Memory usage without flush after 19426 tasks: 1144.29296875 MB +worker1-1 | 2024/06/04 17:54:28.256 FIG10b: Memory usage without flush after 19541 tasks: 1144.40234375 MB +worker1-1 | 2024/06/04 17:54:29.260 2024/06/04 17:54:29.261 FIG10b: Memory usage without flush after 19636 tasks: 1144.85546875 MB +worker1-1 | 2024/06/04 17:54:30.266 FIG10b: Memory usage without flush after 19714 tasks: 1145.16015625 MB +worker1-1 | 2024/06/04 17:54:31.268 FIG10b: Memory usage without flush after 19825 tasks: 1147.01953125 MB +worker1-1 | 2024/06/04 17:54:32.320 FIG10b: Memory usage without flush after 19938 tasks: 1146.8828125 MB +worker1-1 | 2024/06/04 17:54:33.341 FIG10b: Memory usage without flush after 20060 tasks: 1146.9375 MB +worker1-1 | 2024/06/04 17:54:34.346 FIG10b: Memory usage without flush after 20156 tasks: 1146.41796875 MB +worker1-1 | 2024/06/04 17:54:35.395 FIG10b: Memory usage without flush after 20260 tasks: 1147.53125 MB +worker1-1 | took this many seconds: 0.02142024/06/04 17:54:36.409 FIG10b: Memory usage without flush after 20364 tasks: 1147.203125 MB +worker1-1 | 2024/06/04 17:54:37.413 FIG10b: Memory usage without flush after 20475 tasks: 1147.32421875 MB +worker1-1 | 2024/06/04 17:54:38.418 [worker]2024/06/04 17:54:38.418 FIG10b: Memory usage without flush after 20557 tasks: 1147.59765625 MB +worker1-1 | took this many seconds: 0.0115 2024/06/04 17:54:39.4182024/06/04 17:54:39.418 FIG10b: Memory usage without flush after 20673 tasks: 1147.4453125 MB +worker1-1 | 2024/06/04 17:54:40.4292024/06/04 17:54:40.429 FIG10b: Memory usage without flush after 20773 tasks: 1147.625 MB +worker1-1 | 2024/06/04 17:54:41.441 FIG10b: Memory usage without flush after 20873 tasks: 1149.32421875 MB +worker1-1 | 2024/06/04 17:54:42.446 FIG10b: Memory usage without flush after 20968 tasks: 1149.02734375 MB +worker1-1 | 2024/06/04 17:54:43.447 2024/06/04 17:54:43.447[worker] FIG10b: Memory usage without flush after 21083 tasks: 1149.37890625 MB +worker1-1 | 2024/06/04 17:54:44.448 FIG10b: Memory usage without flush after 21185 tasks: 1149.0546875 MB +worker1-1 | 2024/06/04 17:54:45.458 FIG10b: Memory usage without flush after 21273 tasks: 1149.53125 MB +worker1-1 | 2024/06/04 17:54:46.4602024/06/04 17:54:46.460 FIG10b: Memory usage without flush after 21372 tasks: 1149.96484375 MB +worker1-1 | gonna store nowin Run() rn2024/06/04 17:54:47.461 FIG10b: Memory usage without flush after 21481 tasks: 1149.6015625 MB +worker1-1 | 2024/06/04 17:54:48.467 FIG10b: Memory usage without flush after 21574 tasks: 1149.79296875 MB +worker1-1 | 2024/06/04 17:54:49.4682024/06/04 17:54:49.468 FIG10b: Memory usage without flush after 21684 tasks: 1149.68359375 MB +worker1-1 | 2024/06/04 17:54:50.471 FIG10b: Memory usage without flush after 21797 tasks: 1150.765625 MB +worker1-1 | [worker]2024/06/04 17:54:51.4752024/06/04 17:54:51.4752024/06/04 17:54:51.475 FIG10b: Memory usage without flush after 21908 tasks: 1150.83203125 MB +worker1-1 | 2024/06/04 17:54:52.476 [worker]gonna store now2024/06/04 17:54:52.477 FIG10b: Memory usage without flush after 22020 tasks: 1151.71875 MB +worker1-1 | 2024/06/04 17:54:53.483 2024/06/04 17:54:53.4842024/06/04 17:54:53.485 FIG10b: Memory usage without flush after 22130 tasks: 1153.14453125 MB +worker1-1 | 2024/06/04 17:54:54.487 FIG10b: Memory usage without flush after 22240 tasks: 1153.86328125 MB +worker1-1 | FIG10b: Memory usage without flush after 22345 tasks: 1152.5078125 MB +worker1-1 | Executed!2024/06/04 17:54:56.494 FIG10b: Memory usage without flush after 22446 tasks: 1152.77734375 MB +worker1-1 | stored...2024/06/04 17:54:57.4982024/06/04 17:54:57.498 FIG10b: Memory usage without flush after 22554 tasks: 1153.44921875 MB +worker1-1 | FIG10b: Memory usage without flush after 22658 tasks: 1154.15234375 MB +worker1-1 | 2024/06/04 17:54:59.5082024/06/04 17:54:59.508 stored...2024/06/04 17:54:59.509 FIG10b: Memory usage without flush after 22764 tasks: 1153.2890625 MB +worker1-1 | 2024/06/04 17:55:00.5132024/06/04 17:55:00.5142024/06/04 17:55:00.514 FIG10b: Memory usage without flush after 22866 tasks: 1153.59375 MB +worker1-1 | 2024/06/04 17:55:01.516 FIG10b: Memory usage without flush after 22966 tasks: 1146.75390625 MB +worker1-1 | 2024/06/04 17:55:02.522 FIG10b: Memory usage without flush after 23071 tasks: 1147.3984375 MB +worker1-1 | 2024/06/04 17:55:03.5252024/06/04 17:55:03.5252024/06/04 17:55:03.525 [worker]2024/06/04 17:55:03.526 FIG10b: Memory usage without flush after 23170 tasks: 1147.75 MB +worker1-1 | 2024/06/04 17:55:04.530 FIG10b: Memory usage without flush after 23265 tasks: 1148.02734375 MB +worker1-1 | 2024/06/04 17:55:05.532gonna store now FIG10b: Memory usage without flush after 23373 tasks: 1148.5625 MB +worker1-1 | stored...2024/06/04 17:55:06.538 FIG10b: Memory usage without flush after 23476 tasks: 1148.30859375 MB +worker1-1 | 2024/06/04 17:55:07.549 FIG10b: Memory usage without flush after 23579 tasks: 1149.0078125 MB +worker1-1 | 2024/06/04 17:55:08.5552024/06/04 17:55:08.555 FIG10b: Memory usage without flush after 23686 tasks: 1149.26953125 MB +worker1-1 | 2024/06/04 17:55:09.555 2024/06/04 17:55:09.5562024/06/04 17:55:09.556[worker] FIG10b: Memory usage without flush after 23793 tasks: 1149.5390625 MB +worker1-1 | FIG10b: Memory usage without flush after 23900 tasks: 1150.2421875 MB +worker1-1 | Executed!2024/06/04 17:55:11.580 2024/06/04 17:55:11.580 FIG10b: Memory usage without flush after 24006 tasks: 1150.45703125 MB +worker1-1 | 2024/06/04 17:55:12.5802024/06/04 17:55:12.581 FIG10b: Memory usage without flush after 24106 tasks: 1150.4765625 MB +worker1-1 | 2024/06/04 17:55:13.586 FIG10b: Memory usage without flush after 24208 tasks: 1151.46484375 MBExecuted! +worker1-1 | 2024/06/04 17:55:14.593 FIG10b: Memory usage without flush after 24318 tasks: 1151.48046875 MB +worker1-1 | 2024/06/04 17:55:15.596 FIG10b: Memory usage without flush after 24429 tasks: 1151.9609375 MB +worker1-1 | [worker] in Run() rn2024/06/04 17:55:16.597 FIG10b: Memory usage without flush after 24529 tasks: 1151.734375 MB +worker1-1 | FIG10b: Memory usage without flush after 24629 tasks: 1152.2421875 MB +worker1-1 | 2024/06/04 17:55:18.602 [worker]2024/06/04 17:55:18.602 FIG10b: Memory usage without flush after 24715 tasks: 1152.42578125 MB +worker1-1 | 2024/06/04 17:55:19.6102024/06/04 17:55:19.611 FIG10b: Memory usage without flush after 24801 tasks: 1152.6953125 MB +worker1-1 | 2024/06/04 17:55:20.617 FIG10b: Memory usage without flush after 24897 tasks: 1153.1640625 MB +worker1-1 | 2024/06/04 17:55:21.618 FIG10b: Memory usage without flush after 24992 tasks: 1151.421875 MB +worker1-1 | 2024/06/04 17:55:22.6212024/06/04 17:55:22.622 FIG10b: Memory usage without flush after 25089 tasks: 1148.91015625 MB +worker1-1 | 2024/06/04 17:55:23.627 FIG10b: Memory usage without flush after 25187 tasks: 1150.38671875 MB +worker1-1 | 2024/06/04 17:55:24.637 FIG10b: Memory usage without flush after 25293 tasks: 1150.31640625 MB +worker1-1 | 2024/06/04 17:55:25.640 FIG10b: Memory usage without flush after 25387 tasks: 1150.5546875 MB +worker1-1 | 2024/06/04 17:55:26.645 FIG10b: Memory usage without flush after 25477 tasks: 1150.83984375 MB +worker1-1 | 2024/06/04 17:55:27.648 FIG10b: Memory usage without flush after 25573 tasks: 1150.34765625 MB +worker1-1 | 2024/06/04 17:55:28.655 FIG10b: Memory usage without flush after 25666 tasks: 1150.8515625 MB +worker1-1 | 2024/06/04 17:55:29.657 FIG10b: Memory usage without flush after 25762 tasks: 1150.8671875 MB +worker1-1 | 2024/06/04 17:55:30.659 2024/06/04 17:55:30.660 FIG10b: Memory usage without flush after 25858 tasks: 1151.12109375 MB +worker1-1 | 2024/06/04 17:55:31.668 FIG10b: Memory usage without flush after 25944 tasks: 1151.1484375 MB +worker1-1 | 2024/06/04 17:55:32.6792024/06/04 17:55:32.679 FIG10b: Memory usage without flush after 26032 tasks: 1153.4609375 MB +worker1-1 | 2024/06/04 17:55:33.6822024/06/04 17:55:33.683 FIG10b: Memory usage without flush after 26120 tasks: 1153.9921875 MB +worker1-1 | [worker] gonna store now2024/06/04 17:55:34.688 FIG10b: Memory usage without flush after 26207 tasks: 1153.80859375 MB +worker1-1 | 2024/06/04 17:55:35.692 FIG10b: Memory usage without flush after 26277 tasks: 1153.6953125 MB +worker1-1 | 2024/06/04 17:55:36.699 FIG10b: Memory usage without flush after 26352 tasks: 1153.81640625 MB +worker1-1 | 2024/06/04 17:55:37.702 FIG10b: Memory usage without flush after 26423 tasks: 1153.94921875 MB +worker1-1 | 2024/06/04 17:55:38.706 FIG10b: Memory usage without flush after 26520 tasks: 1153.72265625 MB +worker1-1 | 2024/06/04 17:55:39.719 FIG10b: Memory usage without flush after 26600 tasks: 1153.56640625 MB +worker1-1 | 2024/06/04 17:55:40.723 FIG10b: Memory usage without flush after 26699 tasks: 1163.3046875 MB +worker1-1 | 2024/06/04 17:55:41.724 FIG10b: Memory usage without flush after 26786 tasks: 1164.203125 MB +worker1-1 | 2024/06/04 17:55:42.725 FIG10b: Memory usage without flush after 26880 tasks: 1164.9296875 MB +worker1-1 | 2024/06/04 17:55:43.733 FIG10b: Memory usage without flush after 26984 tasks: 1165.4140625 MB +worker1-1 | [worker]2024/06/04 17:55:44.7372024/06/04 17:55:44.737 FIG10b: Memory usage without flush after 27080 tasks: 1166.13671875 MB +worker1-1 | 2024/06/04 17:55:45.745 FIG10b: Memory usage without flush after 27161 tasks: 1165.96484375 MB +worker1-1 | 2024/06/04 17:55:46.746 FIG10b: Memory usage without flush after 27255 tasks: 1166.48046875 MB +worker1-1 | 2024/06/04 17:55:47.748 FIG10b: Memory usage without flush after 27355 tasks: 1167.19140625 MB +worker1-1 | 2024/06/04 17:55:48.7482024/06/04 17:55:48.748 [worker]2024/06/04 17:55:48.749 FIG10b: Memory usage without flush after 27456 tasks: 1167.42578125 MB +worker1-1 | 2024/06/04 17:55:49.7532024/06/04 17:55:49.753 FIG10b: Memory usage without flush after 27563 tasks: 1172.82421875 MB +worker1-1 | 2024/06/04 17:55:50.755 FIG10b: Memory usage without flush after 27659 tasks: 1172.80078125 MB +worker1-1 | [worker]2024/06/04 17:55:51.758 2024/06/04 17:55:51.759 FIG10b: Memory usage without flush after 27762 tasks: 1173.64453125 MB +worker1-1 | 2024/06/04 17:55:52.768 FIG10b: Memory usage without flush after 27859 tasks: 1175.81640625 MB +worker1-1 | 2024/06/04 17:55:53.772 FIG10b: Memory usage without flush after 27947 tasks: 1177.01953125 MB +worker1-1 | 2024/06/04 17:55:54.777 2024/06/04 17:55:54.777 FIG10b: Memory usage without flush after 28036 tasks: 1177.4609375 MB +worker1-1 | 2024/06/04 17:55:55.7832024/06/04 17:55:55.783 FIG10b: Memory usage without flush after 28128 tasks: 1177.91796875 MB +worker1-1 | 2024/06/04 17:55:56.7862024/06/04 17:55:56.786 FIG10b: Memory usage without flush after 28230 tasks: 1177.03125 MB +worker1-1 | 2024/06/04 17:55:57.7862024/06/04 17:55:57.787 FIG10b: Memory usage without flush after 28334 tasks: 1176.80078125 MB +worker1-1 | 2024/06/04 17:55:58.791 FIG10b: Memory usage without flush after 28427 tasks: 1177.3046875 MB +worker1-1 | FIG10b: Memory usage without flush after 28494 tasks: 1177.78125 MB +worker1-1 | 2024/06/04 17:56:00.803 FIG10b: Memory usage without flush after 28573 tasks: 1177.72265625 MB +worker1-1 | 2024/06/04 17:56:01.807 FIG10b: Memory usage without flush after 28662 tasks: 1177.25390625 MB +worker1-1 | 2024/06/04 17:56:02.809 [worker] 2024/06/04 17:56:02.810 FIG10b: Memory usage without flush after 28756 tasks: 1177.80078125 MB +worker1-1 | 2024/06/04 17:56:03.820 FIG10b: Memory usage without flush after 28851 tasks: 1181.1171875 MB +worker1-1 | 2024/06/04 17:56:04.8352024/06/04 17:56:04.835 FIG10b: Memory usage without flush after 28946 tasks: 1181.34765625 MB +worker1-1 | 2024/06/04 17:56:05.8412024/06/04 17:56:05.841 FIG10b: Memory usage without flush after 29033 tasks: 1181.546875 MB +worker1-1 | 2024/06/04 17:56:06.857 FIG10b: Memory usage without flush after 29129 tasks: 1181.5859375 MB +worker1-1 | 2024/06/04 17:56:07.857 FIG10b: Memory usage without flush after 29212 tasks: 1182.09375 MB +worker1-1 | 2024/06/04 17:56:08.863 FIG10b: Memory usage without flush after 29300 tasks: 1181.640625 MB +worker1-1 | 2024/06/04 17:56:09.868 FIG10b: Memory usage without flush after 29414 tasks: 1182.57421875 MB +worker1-1 | 2024/06/04 17:56:10.873 FIG10b: Memory usage without flush after 29521 tasks: 1183.7265625 MB +worker1-1 | 2024/06/04 17:56:11.875 FIG10b: Memory usage without flush after 29623 tasks: 1183.546875 MB +worker1-1 | 2024/06/04 17:56:12.8832024/06/04 17:56:12.883 [worker] gonna store nowFIG10b: Memory usage without flush after 29732 tasks: 1183.8046875 MB +worker1-1 | 2024/06/04 17:56:13.893 FIG10b: Memory usage without flush after 29839 tasks: 1179.33984375 MB +worker1-1 | 2024/06/04 17:56:14.894 FIG10b: Memory usage without flush after 29939 tasks: 1179.7890625 MB +worker1-1 | 2024/06/04 17:56:15.905 FIG10b: Memory usage without flush after 30042 tasks: 1180.2890625 MB +worker1-1 | [worker]2024/06/04 17:56:16.920 FIG10b: Memory usage without flush after 30122 tasks: 1180.04296875 MB +worker1-1 | 2024/06/04 17:56:17.9212024/06/04 17:56:17.922 FIG10b: Memory usage without flush after 30206 tasks: 1179.5859375 MB +worker1-1 | 2024/06/04 17:56:18.923 FIG10b: Memory usage without flush after 30310 tasks: 1180.7734375 MB +worker1-1 | [worker]2024/06/04 17:56:19.930 FIG10b: Memory usage without flush after 30416 tasks: 1180.55859375 MB +worker1-1 | 2024/06/04 17:56:20.930 [worker] stored...2024/06/04 17:56:20.931 FIG10b: Memory usage without flush after 30518 tasks: 1180.31640625 MB +worker1-1 | [worker]2024/06/04 17:56:21.938 FIG10b: Memory usage without flush after 30626 tasks: 1181.28515625 MB +worker1-1 | 2024/06/04 17:56:22.942 FIG10b: Memory usage without flush after 30726 tasks: 1180.00390625 MB +worker1-1 | 2024/06/04 17:56:23.954 FIG10b: Memory usage without flush after 30815 tasks: 1182.046875 MB +worker1-1 | 2024/06/04 17:56:24.958 FIG10b: Memory usage without flush after 30874 tasks: 1183.4375 MB +worker1-1 | 2024/06/04 17:56:25.976 [worker]2024/06/04 17:56:25.977 FIG10b: Memory usage without flush after 30966 tasks: 1183.21875 MB +worker1-1 | 2024/06/04 17:56:26.986[worker] FIG10b: Memory usage without flush after 31058 tasks: 1183.97265625 MB +worker1-1 | [worker][worker]2024/06/04 17:56:27.987 FIG10b: Memory usage without flush after 31155 tasks: 1183.51171875 MB +worker1-1 | gonna store nowtook this many seconds: 0.02612024/06/04 17:56:28.998 FIG10b: Memory usage without flush after 31251 tasks: 1184.0078125 MB +worker1-1 | 2024/06/04 17:56:30.006 FIG10b: Memory usage without flush after 31365 tasks: 1183.65625 MB +worker1-1 | 2024/06/04 17:56:31.0152024/06/04 17:56:31.015 FIG10b: Memory usage without flush after 31473 tasks: 1184.14453125 MB +worker1-1 | 2024/06/04 17:56:32.0182024/06/04 17:56:32.018 FIG10b: Memory usage without flush after 31576 tasks: 1182.58984375 MB +worker1-1 | 2024/06/04 17:56:33.022 FIG10b: Memory usage without flush after 31680 tasks: 1182.53125 MB +worker1-1 | 2024/06/04 17:56:34.030 FIG10b: Memory usage without flush after 31787 tasks: 1186.67578125 MB +worker1-1 | 2024/06/04 17:56:35.033 FIG10b: Memory usage without flush after 31900 tasks: 1185.92578125 MB +worker1-1 | 2024/06/04 17:56:36.033 FIG10b: Memory usage without flush after 32003 tasks: 1174.68359375 MB +worker1-1 | 2024/06/04 17:57:06.033 2024/06/04 17:57:06.0342024/06/04 17:57:06.034 FIG10b: Memory usage without flush after 35229 tasks: 1180.8984375 MB +worker1-1 | 2024/06/04 17:57:36.0392024/06/04 17:57:36.040 FIG10b: Memory usage without flush after 38298 tasks: 1184.65234375 MB +worker1-1 | 2024/06/04 17:58:06.046 FIG10b: Memory usage without flush after 41191 tasks: 1191.8828125 MB +worker1-1 | [worker] 2024/06/04 17:58:36.050 FIG10b: Memory usage without flush after 44182 tasks: 1196.6171875 MB +worker1-1 | 2024/06/04 17:59:06.058 FIG10b: Memory usage without flush after 47052 tasks: 1204.25390625 MB +worker1-1 | 2024/06/04 17:59:36.059 FIG10b: Memory usage without flush after 49891 tasks: 1207.82421875 MB +worker1-1 | 2024/06/04 18:00:06.0692024/06/04 18:00:06.069 FIG10b: Memory usage without flush after 52681 tasks: 1214.05859375 MB +worker1-1 | 2024/06/04 18:00:36.072 2024/06/04 18:00:36.072 FIG10b: Memory usage without flush after 55884 tasks: 1232.2421875 MB +worker1-1 | 2024/06/04 18:01:06.075 FIG10b: Memory usage without flush after 59003 tasks: 1239.90234375 MB +worker1-1 | 2024/06/04 18:01:36.080 FIG10b: Memory usage without flush after 62247 tasks: 1239.94140625 MB +worker1-1 | 2024/06/04 18:02:06.084 [worker]2024/06/04 18:02:06.085 FIG10b: Memory usage without flush after 65488 tasks: 1240.8671875 MB +worker1-1 | FIG10b: Memory usage without flush after 68727 tasks: 1243.9453125 MB +worker1-1 | 2024/06/04 18:03:06.099[worker]2024/06/04 18:03:06.100 FIG10b: Memory usage without flush after 71805 tasks: 1250.90234375 MB +worker1-1 | 2024/06/04 18:03:36.1072024/06/04 18:03:36.107 FIG10b: Memory usage without flush after 74937 tasks: 1256.14453125 MB diff --git a/analysis/output_10b_run6_flush.txt b/analysis/output_10b_run6_flush.txt new file mode 100644 index 0000000..52c51c4 --- /dev/null +++ b/analysis/output_10b_run6_flush.txt @@ -0,0 +1,212 @@ +gcs-1 | 2024/06/04 18:35:55.978 0.4712982177734375 +gcs-1 | 2024/06/04 18:36:00.979 0.6151580810546875 +gcs-1 | 2024/06/04 18:36:05.980 0.730987548828125 +gcs-1 | 2024/06/04 18:36:10.980 0.8366241455078125 +gcs-1 | 2024/06/04 18:36:15.982 0.7630157470703125 +gcs-1 | 2024/06/04 18:36:20.984 0.8029861450195312 +gcs-1 | 2024/06/04 18:36:25.985 0.806304931640625 +gcs-1 | 2024/06/04 18:36:30.986 0.9662704467773438 +gcs-1 | 2024/06/04 18:36:35.986 0.882080078125 +gcs-1 | 2024/06/04 18:36:40.990 0.8871078491210938 +gcs-1 | 2024/06/04 18:36:45.991 0.8853683471679688 +gcs-1 | 2024/06/04 18:36:50.992 0.8856353759765625 +gcs-1 | 2024/06/04 18:36:55.993 0.903106689453125 +gcs-1 | 2024/06/04 18:37:00.994 1.2060394287109375 +gcs-1 | 2024/06/04 18:37:05.995 1.0459136962890625 +gcs-1 | 2024/06/04 18:37:10.998 1.0422439575195312 +gcs-1 | 2024/06/04 18:37:16.001 1.0460891723632812 +gcs-1 | 2024/06/04 18:37:21.002 1.0367889404296875 +gcs-1 | 2024/06/04 18:37:26.005 1.0477371215820312 +gcs-1 | 2024/06/04 18:37:31.007 1.0630111694335938 +gcs-1 | 2024/06/04 18:37:36.008 1.0644607543945312 +gcs-1 | 2024/06/04 18:37:41.008 1.06597900390625 +gcs-1 | 2024/06/04 18:37:46.010 1.07586669921875 +gcs-1 | 2024/06/04 18:37:51.010 1.0880126953125 +gcs-1 | 2024/06/04 18:37:56.011 1.7011642456054688 +gcs-1 | 2024/06/04 18:38:01.012 1.6937637329101562 +gcs-1 | 2024/06/04 18:38:06.015 1.35693359375 +gcs-1 | 2024/06/04 18:38:11.016 1.3575210571289062 +gcs-1 | 2024/06/04 18:38:16.016 1.3592376708984375 +gcs-1 | 2024/06/04 18:38:21.017 1.355438232421875 +gcs-1 | 2024/06/04 18:38:26.018 1.3642654418945312 +gcs-1 | 2024/06/04 18:38:31.019 1.3612899780273438 +gcs-1 | 2024/06/04 18:38:36.020 1.35736083984375 +gcs-1 | 2024/06/04 18:38:41.021 1.36566162109375 +gcs-1 | 2024/06/04 18:38:46.025 1.3725051879882812 +gcs-1 | 2024/06/04 18:38:51.027 1.3767852783203125 +gcs-1 | 2024/06/04 18:38:56.028 1.3851394653320312 +gcs-1 | 2024/06/04 18:39:01.029 1.391815185546875 +gcs-1 | 2024/06/04 18:39:06.030 1.3970184326171875 +gcs-1 | 2024/06/04 18:39:11.032 1.4043045043945312 +gcs-1 | 2024/06/04 18:39:16.034 1.413177490234375 +gcs-1 | 2024/06/04 18:39:21.034 1.4253005981445312 +gcs-1 | 2024/06/04 18:39:26.036 1.4256057739257812 +gcs-1 | 2024/06/04 18:39:31.038 1.4508438110351562 +gcs-1 | 2024/06/04 18:39:36.038 1.4605636596679688 +gcs-1 | 2024/06/04 18:39:41.040 2.6618728637695312 +gcs-1 | 2024/06/04 18:39:46.042 2.6613845825195312 +gcs-1 | 2024/06/04 18:39:51.043 2.6645889282226562 +gcs-1 | 2024/06/04 18:39:56.044 2.64996337890625 +gcs-1 | 2024/06/04 18:40:01.046 2.6588897705078125 +gcs-1 | 2024/06/04 18:40:06.048 1.9665908813476562 +gcs-1 | 2024/06/04 18:40:11.049 1.9753646850585938 +gcs-1 | 2024/06/04 18:40:16.050 1.9753799438476562 +gcs-1 | 2024/06/04 18:40:21.050 1.9930038452148438 +gcs-1 | 2024/06/04 18:40:26.051 1.9843673706054688 +gcs-1 | 2024/06/04 18:40:31.052 1.9817047119140625 +gcs-1 | 2024/06/04 18:40:36.053 1.9836349487304688 +gcs-1 | 2024/06/04 18:40:41.053 1.9745941162109375 +gcs-1 | 2024/06/04 18:40:46.057 1.9837646484375 +gcs-1 | 2024/06/04 18:40:51.059 1.9785537719726562 +gcs-1 | 2024/06/04 18:40:56.059 1.990203857421875 +gcs-1 | 2024/06/04 18:41:01.061 1.7728424072265625 +gcs-1 | 2024/06/04 18:41:06.063 1.79400634765625 +gcs-1 | 2024/06/04 18:41:11.065 1.7856674194335938 +gcs-1 | 2024/06/04 18:41:16.066 1.793914794921875 +gcs-1 | 2024/06/04 18:41:21.067 1.7599563598632812 +gcs-1 | 2024/06/04 18:41:26.069 1.794189453125 +gcs-1 | 2024/06/04 18:41:31.070 1.7983551025390625 +gcs-1 | 2024/06/04 18:41:36.072 1.7949447631835938 +gcs-1 | 2024/06/04 18:41:41.073 1.8064117431640625 +gcs-1 | 2024/06/04 18:41:46.077 1.8233795166015625 +gcs-1 | 2024/06/04 18:41:51.079 1.81158447265625 +gcs-1 | 2024/06/04 18:41:56.080 1.8395156860351562 +gcs-1 | 2024/06/04 18:42:01.081 1.827056884765625 +gcs-1 | 2024/06/04 18:42:06.082 1.8432846069335938 +gcs-1 | 2024/06/04 18:42:11.083 1.8462371826171875 +gcs-1 | 2024/06/04 18:42:16.083 1.8495559692382812 +gcs-1 | 2024/06/04 18:42:21.084 1.8671875 +gcs-1 | 2024/06/04 18:42:26.085 1.8811187744140625 +gcs-1 | 2024/06/04 18:42:31.086 1.895721435546875 +gcs-1 | 2024/06/04 18:42:36.087 1.8936309814453125 +gcs-1 | 2024/06/04 18:42:41.088 1.8928680419921875 +gcs-1 | 2024/06/04 18:42:46.093 1.9055862426757812 +gcs-1 | 2024/06/04 18:42:51.094 1.9277496337890625 +gcs-1 | 2024/06/04 18:42:56.095 1.9287490844726562 +gcs-1 | 2024/06/04 18:43:01.097 1.937286376953125 +gcs-1 | 2024/06/04 18:43:06.097 1.9503936767578125 +gcs-1 | 2024/06/04 18:43:11.099 1.9545745849609375 +gcs-1 | 2024/06/04 18:43:16.101 1.988128662109375 +gcs-1 | 2024/06/04 18:43:21.102 4.376167297363281 +gcs-1 | 2024/06/04 18:43:26.103 4.376502990722656 +gcs-1 | 2024/06/04 18:43:31.105 4.377189636230469 +gcs-1 | 2024/06/04 18:43:36.106 4.394599914550781 +gcs-1 | 2024/06/04 18:43:41.106 4.396209716796875 +gcs-1 | 2024/06/04 18:43:46.109 4.377449035644531 +gcs-1 | 2024/06/04 18:43:51.112 4.3657379150390625 +gcs-1 | 2024/06/04 18:43:56.114 4.391792297363281 +gcs-1 | 2024/06/04 18:44:01.116 4.393150329589844 +gcs-1 | 2024/06/04 18:44:06.118 4.379981994628906 +gcs-1 | 2024/06/04 18:44:11.118 3.0070571899414062 +gcs-1 | 2024/06/04 18:44:16.122 3.0190658569335938 +gcs-1 | 2024/06/04 18:44:21.124 3.0153961181640625 +gcs-1 | 2024/06/04 18:44:26.126 3.0196151733398438 +gcs-1 | 2024/06/04 18:44:31.127 3.0075531005859375 +gcs-1 | 2024/06/04 18:44:36.129 3.0203094482421875 +gcs-1 | 2024/06/04 18:44:41.130 3.0154876708984375 +gcs-1 | 2024/06/04 18:44:46.132 3.0289764404296875 +gcs-1 | 2024/06/04 18:44:51.133 3.02105712890625 +gcs-1 | 2024/06/04 18:44:56.135 3.0169143676757812 +gcs-1 | 2024/06/04 18:45:01.152 3.0152587890625 +gcs-1 | 2024/06/04 18:45:06.153 3.0446090698242188 +gcs-1 | 2024/06/04 18:45:11.156 3.0236053466796875 +gcs-1 | 2024/06/04 18:45:16.158 3.037078857421875 +gcs-1 | 2024/06/04 18:45:21.159 3.031402587890625 +gcs-1 | 2024/06/04 18:45:26.162 3.00067138671875 +gcs-1 | 2024/06/04 18:45:31.163 3.0385284423828125 +gcs-1 | 2024/06/04 18:45:36.164 3.0350189208984375 +gcs-1 | 2024/06/04 18:45:41.166 3.0415420532226562 +gcs-1 | 2024/06/04 18:45:46.169 3.027008056640625 +gcs-1 | 2024/06/04 18:45:51.170 3.0510482788085938 +gcs-1 | 2024/06/04 18:45:56.171 3.0405120849609375 +gcs-1 | 2024/06/04 18:46:01.173 3.040008544921875 +gcs-1 | 2024/06/04 18:46:06.174 3.0400543212890625 +gcs-1 | 2024/06/04 18:46:11.176 3.0528411865234375 +gcs-1 | 2024/06/04 18:46:16.179 3.0454940795898438 +gcs-1 | 2024/06/04 18:46:21.180 3.0629653930664062 +gcs-1 | 2024/06/04 18:46:26.181 3.064178466796875 +gcs-1 | 2024/06/04 18:46:31.183 3.056488037109375 +gcs-1 | 2024/06/04 18:46:36.184 3.0559768676757812 +gcs-1 | 2024/06/04 18:46:41.186 3.049346923828125 +gcs-1 | 2024/06/04 18:46:46.187 3.0491104125976562 +gcs-1 | 2024/06/04 18:46:51.188 3.070220947265625 +gcs-1 | 2024/06/04 18:46:56.189 3.0629425048828125 +gcs-1 | 2024/06/04 18:47:01.190 3.0534286499023438 +gcs-1 | 2024/06/04 18:47:06.191 3.0541229248046875 +gcs-1 | 2024/06/04 18:47:11.193 3.063262939453125 +gcs-1 | 2024/06/04 18:47:16.197 3.066314697265625 +gcs-1 | 2024/06/04 18:47:21.198 3.0680618286132812 +gcs-1 | 2024/06/04 18:47:26.199 3.0597457885742188 +gcs-1 | 2024/06/04 18:47:31.213 3.02667236328125 +gcs-1 | 2024/06/04 18:47:36.214 3.0867996215820312 +gcs-1 | 2024/06/04 18:47:41.215 3.0983428955078125 +gcs-1 | 2024/06/04 18:47:46.217 3.0848159790039062 +gcs-1 | 2024/06/04 18:47:51.218 3.1069259643554688 +gcs-1 | 2024/06/04 18:47:56.219 3.1214981079101562 +gcs-1 | 2024/06/04 18:48:01.220 3.0757522583007812 +gcs-1 | 2024/06/04 18:48:06.221 3.1439437866210938 +gcs-1 | 2024/06/04 18:48:11.224 3.1282882690429688 +gcs-1 | 2024/06/04 18:48:16.226 3.134124755859375 +gcs-1 | 2024/06/04 18:48:21.227 3.1048660278320312 +gcs-1 | 2024/06/04 18:48:26.227 3.1502838134765625 +gcs-1 | 2024/06/04 18:48:31.228 3.1458282470703125 +gcs-1 | 2024/06/04 18:48:36.229 3.1410675048828125 +gcs-1 | 2024/06/04 18:48:41.237 3.1520156860351562 +gcs-1 | 2024/06/04 18:48:46.239 3.1770706176757812 +gcs-1 | 2024/06/04 18:48:51.240 3.1921844482421875 +gcs-1 | 2024/06/04 18:48:56.241 3.1827392578125 +gcs-1 | 2024/06/04 18:49:01.243 3.1824722290039062 +gcs-1 | 2024/06/04 18:49:06.244 3.2339859008789062 +gcs-1 | 2024/06/04 18:49:11.245 3.1619110107421875 +gcs-1 | 2024/06/04 18:49:16.247 3.2211456298828125 +gcs-1 | 2024/06/04 18:49:21.270 3.2269287109375 +gcs-1 | 2024/06/04 18:49:26.271 3.24176025390625 +gcs-1 | 2024/06/04 18:49:31.272 3.265350341796875 +gcs-1 | 2024/06/04 18:49:36.274 3.272674560546875 +gcs-1 | 2024/06/04 18:49:41.276 3.2856063842773438 +gcs-1 | 2024/06/04 18:49:46.277 3.3036575317382812 +gcs-1 | 2024/06/04 18:49:51.278 3.3234634399414062 +gcs-1 | 2024/06/04 18:49:56.279 3.321624755859375 +gcs-1 | 2024/06/04 18:50:01.281 3.3337173461914062 +gcs-1 | 2024/06/04 18:50:06.282 3.352447509765625 +gcs-1 | 2024/06/04 18:50:11.283 3.3528671264648438 +gcs-1 | 2024/06/04 18:50:16.285 3.3671417236328125 +gcs-1 | 2024/06/04 18:50:21.287 3.3683624267578125 +gcs-1 | 2024/06/04 18:50:26.288 3.394073486328125 +gcs-1 | 2024/06/04 18:50:31.290 3.3882980346679688 +gcs-1 | 2024/06/04 18:50:36.291 3.4113845825195312 +gcs-1 | 2024/06/04 18:50:41.298 3.3700027465820312 +gcs-1 | 2024/06/04 18:50:46.298 3.4124832153320312 +gcs-1 | 2024/06/04 18:50:51.300 3.3822402954101562 +gcs-1 | 2024/06/04 18:50:56.302 3.4241943359375 +gcs-1 | 2024/06/04 18:51:01.303 3.4392852783203125 +gcs-1 | 2024/06/04 18:51:06.310 3.3993606567382812 +gcs-1 | 2024/06/04 18:51:11.313 3.456268310546875 +gcs-1 | 2024/06/04 18:51:16.316 3.444488525390625 +gcs-1 | 2024/06/04 18:51:21.317 3.4685821533203125 +gcs-1 | 2024/06/04 18:51:26.318 8.228248596191406 +gcs-1 | 2024/06/04 18:51:31.385 8.261550903320312 +gcs-1 | 2024/06/04 18:51:36.387 8.248344421386719 +gcs-1 | 2024/06/04 18:51:41.388 8.290199279785156 +gcs-1 | 2024/06/04 18:51:46.389 8.281494140625 +gcs-1 | 2024/06/04 18:51:51.390 8.303192138671875 +gcs-1 | 2024/06/04 18:51:56.391 8.26129150390625 +gcs-1 | 2024/06/04 18:52:01.393 8.272514343261719 +gcs-1 | 2024/06/04 18:52:06.396 8.297195434570312 +gcs-1 | 2024/06/04 18:52:11.418 8.257728576660156 +gcs-1 | 2024/06/04 18:52:16.421 8.254081726074219 +gcs-1 | 2024/06/04 18:52:21.422 8.299392700195312 +gcs-1 | 2024/06/04 18:52:26.424 8.278854370117188 +gcs-1 | 2024/06/04 18:52:31.425 8.324920654296875 +gcs-1 | 2024/06/04 18:52:36.426 8.291427612304688 +gcs-1 | 2024/06/04 18:52:41.427 8.329216003417969 +gcs-1 | 2024/06/04 18:52:46.428 8.279190063476562 +gcs-1 | 2024/06/04 18:52:51.429 8.315376281738281 +gcs-1 | 2024/06/04 18:52:56.430 8.319190979003906 +gcs-1 | 2024/06/04 18:53:01.431 8.326942443847656 +gcs-1 | 2024/06/04 18:53:06.433 8.319160461425781 +gcs-1 | 2024/06/04 18:53:11.433 8.320426940917969 +gcs-1 | 2024/06/04 18:53:16.435 8.341056823730469 +gcs-1 | 2024/06/04 18:53:21.437 5.562217712402344 +gcs-1 | 2024/06/04 18:53:26.440 5.5629730224609375 +gcs-1 | 2024/06/04 18:53:31.441 5.592796325683594 diff --git a/analysis/output_10b_run7_BADRESULT_dontflush.txt b/analysis/output_10b_run7_BADRESULT_dontflush.txt new file mode 100644 index 0000000..36f5def --- /dev/null +++ b/analysis/output_10b_run7_BADRESULT_dontflush.txt @@ -0,0 +1,114 @@ +gcs-1 | 2024/06/04 18:58:14 server listening at [::]:50000 +gcs-1 | 2024/06/04 18:58:14.414 server listening at [::]:50001 +gcs-1 | 2024/06/04 18:58:14.416 0.4748687744140625 +gcs-1 | 2024/06/04 18:58:19.417 0.7540206909179688 +gcs-1 | 2024/06/04 18:58:24.418 1.065032958984375 +gcs-1 | 2024/06/04 18:58:29.419 1.0816116333007812 +gcs-1 | 2024/06/04 18:58:34.420 1.4495162963867188 +gcs-1 | 2024/06/04 18:58:39.422 1.53350830078125 +gcs-1 | 2024/06/04 18:58:44.423 1.61444091796875 +gcs-1 | 2024/06/04 18:58:49.424 2.6472244262695312 +gcs-1 | 2024/06/04 18:58:54.425 2.9524612426757812 +gcs-1 | 2024/06/04 18:58:59.426 2.374725341796875 +gcs-1 | 2024/06/04 18:59:04.427 2.5830154418945312 +gcs-1 | 2024/06/04 18:59:09.429 2.9528121948242188 +gcs-1 | 2024/06/04 18:59:14.432 3.2568206787109375 +gcs-1 | 2024/06/04 18:59:19.433 3.0411911010742188 +gcs-1 | 2024/06/04 18:59:24.433 2.9720611572265625 +gcs-1 | 2024/06/04 18:59:29.435 3.58203125 +gcs-1 | 2024/06/04 18:59:34.436 2.4860000610351562 +gcs-1 | 2024/06/04 18:59:39.437 3.1765060424804688 +gcs-1 | 2024/06/04 18:59:44.439 2.0571365356445312 +gcs-1 | 2024/06/04 18:59:49.441 2.7280654907226562 +gcs-1 | 2024/06/04 18:59:54.441 3.5274429321289062 +gcs-1 | 2024/06/04 18:59:59.442 3.6893539428710938 +gcs-1 | 2024/06/04 19:00:04.444 2.2642440795898438 +gcs-1 | 2024/06/04 19:00:09.446 2.71295166015625 +gcs-1 | 2024/06/04 19:00:14.447 6.010200500488281 +gcs-1 | 2024/06/04 19:00:19.449 5.968513488769531 +gcs-1 | 2024/06/04 19:00:24.450 3.9825515747070312 +gcs-1 | 2024/06/04 19:00:29.450 3.7501296997070312 +gcs-1 | 2024/06/04 19:00:34.451 3.5324172973632812 +gcs-1 | 2024/06/04 19:00:39.453 3.3953094482421875 +gcs-1 | 2024/06/04 19:00:44.456 3.1057891845703125 +gcs-1 | 2024/06/04 19:00:49.458 2.8861236572265625 +gcs-1 | 2024/06/04 19:00:54.459 5.432861328125 +gcs-1 | 2024/06/04 19:00:59.460 5.0107574462890625 +gcs-1 | 2024/06/04 19:01:04.461 4.550201416015625 +gcs-1 | 2024/06/04 19:01:09.462 4.207878112792969 +gcs-1 | 2024/06/04 19:01:14.462 3.9461746215820312 +gcs-1 | 2024/06/04 19:01:19.463 3.7596893310546875 +gcs-1 | 2024/06/04 19:01:24.465 3.5427474975585938 +gcs-1 | 2024/06/04 19:01:29.465 3.2588424682617188 +gcs-1 | 2024/06/04 19:01:34.466 5.787635803222656 +gcs-1 | 2024/06/04 19:01:39.467 5.4891204833984375 +gcs-1 | 2024/06/04 19:01:44.469 5.175819396972656 +gcs-1 | 2024/06/04 19:01:49.470 4.7996063232421875 +gcs-1 | 2024/06/04 19:01:54.471 4.439849853515625 +gcs-1 | 2024/06/04 19:01:59.471 7.001838684082031 +gcs-1 | 2024/06/04 19:02:04.473 9.541595458984375 +gcs-1 | 2024/06/04 19:02:09.474 12.116615295410156 +gcs-1 | 2024/06/04 19:02:14.477 8.2061767578125 +gcs-1 | 2024/06/04 19:02:19.479 10.763824462890625 +gcs-1 | 2024/06/04 19:02:24.480 5.2975616455078125 +gcs-1 | 2024/06/04 19:02:29.481 7.9277191162109375 +gcs-1 | 2024/06/04 19:02:34.482 6.046714782714844 +gcs-1 | 2024/06/04 19:02:39.483 8.58746337890625 +gcs-1 | 2024/06/04 19:02:44.483 6.536651611328125 +gcs-1 | 2024/06/04 19:02:49.484 8.646133422851562 +gcs-1 | 2024/06/04 19:02:54.485 6.558677673339844 +gcs-1 | 2024/06/04 19:02:59.486 9.062904357910156 +gcs-1 | 2024/06/04 19:03:04.488 6.9549560546875 +gcs-1 | 2024/06/04 19:03:09.490 4.9092559814453125 +gcs-1 | 2024/06/04 19:03:14.491 7.502021789550781 +gcs-1 | 2024/06/04 19:03:19.492 5.123695373535156 +gcs-1 | 2024/06/04 19:03:24.493 7.7623138427734375 +gcs-1 | 2024/06/04 19:03:29.495 5.789070129394531 +gcs-1 | 2024/06/04 19:03:34.497 8.377128601074219 +gcs-1 | 2024/06/04 19:03:39.498 6.492942810058594 +gcs-1 | 2024/06/04 19:03:44.501 4.676490783691406 +gcs-1 | 2024/06/04 19:03:49.502 7.218788146972656 +gcs-1 | 2024/06/04 19:03:54.503 5.489463806152344 +gcs-1 | 2024/06/04 19:03:59.504 8.080680847167969 +gcs-1 | 2024/06/04 19:04:04.505 6.2022552490234375 +gcs-1 | 2024/06/04 19:04:09.506 8.754859924316406 +gcs-1 | 2024/06/04 19:04:14.509 6.789039611816406 +gcs-1 | 2024/06/04 19:04:19.510 9.362159729003906 +gcs-1 | 2024/06/04 19:04:24.512 7.197380065917969 +gcs-1 | 2024/06/04 19:04:29.513 5.1575927734375 +gcs-1 | 2024/06/04 19:04:34.514 7.810218811035156 +gcs-1 | 2024/06/04 19:04:39.516 5.3560943603515625 +gcs-1 | 2024/06/04 19:04:44.647 5.6973419189453125 +gcs-1 | 2024/06/04 19:04:49.667 7.562110900878906 +gcs-1 | 2024/06/04 19:04:54.669 5.394920349121094 +gcs-1 | 2024/06/04 19:04:59.673 7.6736907958984375 +gcs-1 | 2024/06/04 19:10:15.133 6.317527770996094 +gcs-1 | 2024/06/04 19:10:20.135 7.560111999511719 +gcs-1 | 2024/06/04 19:10:25.151 9.531242370605469 +gcs-1 | 2024/06/04 19:10:30.279 9.783424377441406 +gcs-1 | 2024/06/04 19:10:35.300 5.319740295410156 +gcs-1 | 2024/06/04 19:10:40.301 5.928047180175781 +gcs-1 | 2024/06/04 19:10:45.148 6.508811950683594 +gcs-1 | 2024/06/04 19:10:50.152 7.157768249511719 +gcs-1 | 2024/06/04 19:10:55.246 7.735542297363281 +gcs-1 | 2024/06/04 19:12:29.466 7.840553283691406 +gcs-1 | 2024/06/04 19:12:34.468 5.777587890625 +gcs-1 | 2024/06/04 19:12:39.469 8.519912719726562 +gcs-1 | 2024/06/04 19:12:44.471 9.306671142578125 +gcs-1 | 2024/06/04 19:12:49.476 9.999237060546875 +gcs-1 | 2024/06/04 19:12:54.488 5.756217956542969 +gcs-1 | 2024/06/04 19:12:59.491 6.480560302734375 +gcs-1 | 2024/06/04 19:18:21.785 6.981689453125 +gcs-1 | 2024/06/04 19:18:26.785 7.7218780517578125 +gcs-1 | 2024/06/04 19:19:00.886 10.090690612792969 +gcs-1 | 2024/06/04 19:19:05.890 14.087242126464844 +gcs-1 | 2024/06/04 19:19:10.892 15.348991394042969 +gcs-1 | 2024/06/04 19:19:15.894 16.777236938476562 +gcs-1 | 2024/06/04 19:19:20.895 19.112342834472656 +gcs-1 | 2024/06/04 19:19:25.896 14.24517822265625 +gcs-1 | 2024/06/04 19:19:30.902 16.9776611328125 +gcs-1 | 2024/06/04 19:19:35.905 19.4801025390625 +gcs-1 | 2024/06/04 19:19:40.906 21.99610137939453 +gcs-1 | 2024/06/04 19:19:45.988 9.858772277832031 +gcs-1 | 2024/06/04 19:19:50.993 11.033058166503906 +gcs-1 | 2024/06/04 19:19:55.995 13.123298645019531 diff --git a/analysis/output_10b_run8_dontflush.txt b/analysis/output_10b_run8_dontflush.txt new file mode 100644 index 0000000..f2fa4a8 --- /dev/null +++ b/analysis/output_10b_run8_dontflush.txt @@ -0,0 +1,192 @@ +gcs-1 | 2024/06/04 19:21:27.363 0.47129058837890625 +gcs-1 | 2024/06/04 19:21:32.370 0.7487564086914062 +gcs-1 | 2024/06/04 19:21:37.371 0.8387985229492188 +gcs-1 | 2024/06/04 19:21:42.372 3.419403076171875 +gcs-1 | 2024/06/04 19:21:47.373 3.4430770874023438 +gcs-1 | 2024/06/04 19:21:52.373 1.003143310546875 +gcs-1 | 2024/06/04 19:21:57.374 0.9983901977539062 +gcs-1 | 2024/06/04 19:22:02.375 2.0103912353515625 +gcs-1 | 2024/06/04 19:22:07.377 2.3386611938476562 +gcs-1 | 2024/06/04 19:22:12.378 2.3165206909179688 +gcs-1 | 2024/06/04 19:22:17.380 2.5964279174804688 +gcs-1 | 2024/06/04 19:22:22.381 2.657073974609375 +gcs-1 | 2024/06/04 19:22:27.382 2.8733978271484375 +gcs-1 | 2024/06/04 19:22:32.385 2.5427169799804688 +gcs-1 | 2024/06/04 19:22:37.387 2.2776947021484375 +gcs-1 | 2024/06/04 19:22:42.388 2.8539962768554688 +gcs-1 | 2024/06/04 19:22:47.390 3.3134765625 +gcs-1 | 2024/06/04 19:22:52.391 2.057342529296875 +gcs-1 | 2024/06/04 19:22:57.392 2.6971817016601562 +gcs-1 | 2024/06/04 19:23:02.393 3.2243499755859375 +gcs-1 | 2024/06/04 19:23:07.395 3.4471054077148438 +gcs-1 | 2024/06/04 19:23:12.395 1.925506591796875 +gcs-1 | 2024/06/04 19:23:17.397 2.5724411010742188 +gcs-1 | 2024/06/04 19:23:22.398 2.974029541015625 +gcs-1 | 2024/06/04 19:23:27.399 3.4613037109375 +gcs-1 | 2024/06/04 19:23:32.399 6.0098876953125 +gcs-1 | 2024/06/04 19:23:37.400 5.898078918457031 +gcs-1 | 2024/06/04 19:23:42.402 3.8481826782226562 +gcs-1 | 2024/06/04 19:23:47.403 3.360137939453125 +gcs-1 | 2024/06/04 19:23:52.404 3.246429443359375 +gcs-1 | 2024/06/04 19:23:57.407 3.1309356689453125 +gcs-1 | 2024/06/04 19:24:02.410 3.0131988525390625 +gcs-1 | 2024/06/04 19:24:07.410 2.9094314575195312 +gcs-1 | 2024/06/04 19:24:12.411 5.452568054199219 +gcs-1 | 2024/06/04 19:24:17.412 4.906715393066406 +gcs-1 | 2024/06/04 19:24:22.413 4.477287292480469 +gcs-1 | 2024/06/04 19:24:27.415 4.377899169921875 +gcs-1 | 2024/06/04 19:24:32.418 4.270896911621094 +gcs-1 | 2024/06/04 19:24:37.419 4.158775329589844 +gcs-1 | 2024/06/04 19:24:42.420 4.088165283203125 +gcs-1 | 2024/06/04 19:24:47.420 3.9782638549804688 +gcs-1 | 2024/06/04 19:24:52.422 3.7663192749023438 +gcs-1 | 2024/06/04 19:24:57.423 3.4873809814453125 +gcs-1 | 2024/06/04 19:25:02.426 3.2296524047851562 +gcs-1 | 2024/06/04 19:25:07.427 5.927833557128906 +gcs-1 | 2024/06/04 19:25:12.428 5.4492340087890625 +gcs-1 | 2024/06/04 19:25:17.429 7.936981201171875 +gcs-1 | 2024/06/04 19:25:22.432 10.604972839355469 +gcs-1 | 2024/06/04 19:25:27.432 6.833213806152344 +gcs-1 | 2024/06/04 19:25:32.434 9.40277099609375 +gcs-1 | 2024/06/04 19:25:37.435 12.065711975097656 +gcs-1 | 2024/06/04 19:25:42.436 6.60052490234375 +gcs-1 | 2024/06/04 19:25:47.437 4.7006988525390625 +gcs-1 | 2024/06/04 19:25:52.439 7.328437805175781 +gcs-1 | 2024/06/04 19:25:57.440 5.4693603515625 +gcs-1 | 2024/06/04 19:26:02.439 7.106376647949219 +gcs-1 | 2024/06/04 19:26:07.440 4.885688781738281 +gcs-1 | 2024/06/04 19:26:12.441 7.247566223144531 +gcs-1 | 2024/06/04 19:26:17.442 5.273231506347656 +gcs-1 | 2024/06/04 19:26:22.443 7.742225646972656 +gcs-1 | 2024/06/04 19:26:27.444 5.845916748046875 +gcs-1 | 2024/06/04 19:26:32.448 8.254135131835938 +gcs-1 | 2024/06/04 19:26:37.449 6.340934753417969 +gcs-1 | 2024/06/04 19:26:42.449 8.846717834472656 +gcs-1 | 2024/06/04 19:26:47.450 6.9485931396484375 +gcs-1 | 2024/06/04 19:26:52.452 5.043464660644531 +gcs-1 | 2024/06/04 19:26:57.453 7.529380798339844 +gcs-1 | 2024/06/04 19:27:02.454 5.5513153076171875 +gcs-1 | 2024/06/04 19:27:07.455 7.921234130859375 +gcs-1 | 2024/06/04 19:27:12.455 5.634376525878906 +gcs-1 | 2024/06/04 19:27:17.457 8.166412353515625 +gcs-1 | 2024/06/04 19:27:22.460 5.962432861328125 +gcs-1 | 2024/06/04 19:27:27.460 8.487312316894531 +gcs-1 | 2024/06/04 19:27:32.462 6.260498046875 +gcs-1 | 2024/06/04 19:27:37.463 8.73968505859375 +gcs-1 | 2024/06/04 19:27:42.464 6.531364440917969 +gcs-1 | 2024/06/04 19:27:47.465 9.033126831054688 +gcs-1 | 2024/06/04 19:27:52.468 6.6288299560546875 +gcs-1 | 2024/06/04 19:27:57.469 9.079505920410156 +gcs-1 | 2024/06/04 19:28:02.471 6.7116241455078125 +gcs-1 | 2024/06/04 19:28:07.473 9.220306396484375 +gcs-1 | 2024/06/04 19:28:12.676 6.7909088134765625 +gcs-1 | 2024/06/04 19:28:17.677 8.9853515625 +gcs-1 | 2024/06/04 19:28:22.679 6.484397888183594 +gcs-1 | 2024/06/04 19:28:27.680 8.861106872558594 +gcs-1 | 2024/06/04 19:28:32.681 6.506874084472656 +gcs-1 | 2024/06/04 19:28:37.682 9.062728881835938 +gcs-1 | 2024/06/04 19:28:42.683 6.430274963378906 +gcs-1 | 2024/06/04 19:28:47.684 8.7381591796875 +gcs-1 | 2024/06/04 19:28:52.685 6.049659729003906 +gcs-1 | 2024/06/04 19:28:57.687 8.569046020507812 +gcs-1 | 2024/06/04 19:29:02.685 13.198020935058594 +gcs-1 | 2024/06/04 19:29:07.686 15.540107727050781 +gcs-1 | 2024/06/04 19:29:12.687 17.877342224121094 +gcs-1 | 2024/06/04 19:29:17.688 12.938446044921875 +gcs-1 | 2024/06/04 19:29:22.690 15.421417236328125 +gcs-1 | 2024/06/04 19:29:27.690 17.76190948486328 +gcs-1 | 2024/06/04 19:29:32.694 20.038368225097656 +gcs-1 | 2024/06/04 19:29:37.696 22.490615844726562 +gcs-1 | 2024/06/04 19:29:42.697 10.978309631347656 +gcs-1 | 2024/06/04 19:29:47.698 13.442184448242188 +gcs-1 | 2024/06/04 19:29:52.699 15.766464233398438 +gcs-1 | 2024/06/04 19:29:57.701 18.161224365234375 +gcs-1 | 2024/06/04 19:30:02.703 9.691146850585938 +gcs-1 | 2024/06/04 19:30:07.704 11.990638732910156 +gcs-1 | 2024/06/04 19:30:12.706 14.3824462890625 +gcs-1 | 2024/06/04 19:30:17.708 8.759857177734375 +gcs-1 | 2024/06/04 19:30:22.709 11.142829895019531 +gcs-1 | 2024/06/04 19:30:27.710 13.604194641113281 +gcs-1 | 2024/06/04 19:30:32.712 16.05791473388672 +gcs-1 | 2024/06/04 19:30:37.713 10.415145874023438 +gcs-1 | 2024/06/04 19:30:42.713 12.878974914550781 +gcs-1 | 2024/06/04 19:30:47.714 15.367881774902344 +gcs-1 | 2024/06/04 19:30:52.715 9.372314453125 +gcs-1 | 2024/06/04 19:30:57.718 11.754783630371094 +gcs-1 | 2024/06/04 19:31:02.719 14.211502075195312 +gcs-1 | 2024/06/04 19:31:07.720 16.67999267578125 +gcs-1 | 2024/06/04 19:31:12.721 10.782791137695312 +gcs-1 | 2024/06/04 19:31:17.722 13.280471801757812 +gcs-1 | 2024/06/04 19:31:22.724 15.736763000488281 +gcs-1 | 2024/06/04 19:31:27.725 10.063522338867188 +gcs-1 | 2024/06/04 19:31:32.728 12.499763488769531 +gcs-1 | 2024/06/04 19:31:37.730 14.886199951171875 +gcs-1 | 2024/06/04 19:31:42.731 9.108489990234375 +gcs-1 | 2024/06/04 19:31:47.733 11.585563659667969 +gcs-1 | 2024/06/04 19:31:52.735 14.075881958007812 +gcs-1 | 2024/06/04 19:31:57.736 16.557769775390625 +gcs-1 | 2024/06/04 19:32:02.737 10.810867309570312 +gcs-1 | 2024/06/04 19:32:07.739 13.184783935546875 +gcs-1 | 2024/06/04 19:32:12.740 15.646293640136719 +gcs-1 | 2024/06/04 19:32:17.740 9.857810974121094 +gcs-1 | 2024/06/04 19:32:22.742 12.212295532226562 +gcs-1 | 2024/06/04 19:32:27.743 14.596710205078125 +gcs-1 | 2024/06/04 19:32:32.744 9.035865783691406 +gcs-1 | 2024/06/04 19:32:37.745 11.638259887695312 +gcs-1 | 2024/06/04 19:32:42.746 14.198501586914062 +gcs-1 | 2024/06/04 19:32:47.748 16.848388671875 +gcs-1 | 2024/06/04 19:32:52.748 11.137123107910156 +gcs-1 | 2024/06/04 19:32:57.749 13.722923278808594 +gcs-1 | 2024/06/04 19:33:02.751 16.373641967773438 +gcs-1 | 2024/06/04 19:33:07.753 9.919609069824219 +gcs-1 | 2024/06/04 19:33:12.754 12.312255859375 +gcs-1 | 2024/06/04 19:33:17.756 14.829246520996094 +gcs-1 | 2024/06/04 19:33:22.757 9.050392150878906 +gcs-1 | 2024/06/04 19:33:27.758 11.583648681640625 +gcs-1 | 2024/06/04 19:33:32.760 14.016876220703125 +gcs-1 | 2024/06/04 19:33:37.761 16.556686401367188 +gcs-1 | 2024/06/04 19:33:42.773 10.625007629394531 +gcs-1 | 2024/06/04 19:33:47.775 13.024528503417969 +gcs-1 | 2024/06/04 19:33:52.776 15.596565246582031 +gcs-1 | 2024/06/04 19:33:57.777 9.745750427246094 +gcs-1 | 2024/06/04 19:34:02.778 12.34716796875 +gcs-1 | 2024/06/04 19:34:07.779 14.959617614746094 +gcs-1 | 2024/06/04 19:34:12.780 17.524765014648438 +gcs-1 | 2024/06/04 19:34:17.782 11.546844482421875 +gcs-1 | 2024/06/04 19:34:22.784 14.118873596191406 +gcs-1 | 2024/06/04 19:34:27.785 16.70843505859375 +gcs-1 | 2024/06/04 19:34:32.784 10.193046569824219 +gcs-1 | 2024/06/04 19:34:37.814 12.485702514648438 +gcs-1 | 2024/06/04 19:34:42.816 14.736854553222656 +gcs-1 | 2024/06/04 19:34:47.817 17.065345764160156 +gcs-1 | 2024/06/04 19:34:52.818 10.659378051757812 +gcs-1 | 2024/06/04 19:34:57.819 13.1251220703125 +gcs-1 | 2024/06/04 19:35:02.823 15.710685729980469 +gcs-1 | 2024/06/04 19:35:07.824 18.30474090576172 +gcs-1 | 2024/06/04 19:35:12.825 12.003318786621094 +gcs-1 | 2024/06/04 19:35:17.827 14.653106689453125 +gcs-1 | 2024/06/04 19:35:22.828 17.357589721679688 +gcs-1 | 2024/06/04 19:35:27.829 11.02642822265625 +gcs-1 | 2024/06/04 19:35:32.830 13.667793273925781 +gcs-1 | 2024/06/04 19:35:37.831 16.328025817871094 +gcs-1 | 2024/06/04 19:35:42.832 9.955978393554688 +gcs-1 | 2024/06/04 19:35:47.833 12.561546325683594 +gcs-1 | 2024/06/04 19:35:52.836 15.2049560546875 +gcs-1 | 2024/06/04 19:35:57.838 17.817832946777344 +gcs-1 | 2024/06/04 19:36:02.840 11.227790832519531 +gcs-1 | 2024/06/04 19:36:07.841 13.8360595703125 +gcs-1 | 2024/06/04 19:36:12.842 16.432937622070312 +gcs-1 | 2024/06/04 19:36:17.843 19.054405212402344 +gcs-1 | 2024/06/04 19:36:22.844 12.429069519042969 +gcs-1 | 2024/06/04 19:36:27.846 15.116043090820312 +gcs-1 | 2024/06/04 19:36:32.848 26.552085876464844 +gcs-1 | 2024/06/04 19:36:37.849 29.160964965820312 +gcs-1 | 2024/06/04 19:36:42.850 31.53943634033203 +gcs-1 | 2024/06/04 19:36:47.851 34.08723449707031 +gcs-1 | 2024/06/04 19:36:52.852 36.65630340576172 +gcs-1 | 2024/06/04 19:36:57.853 39.16461181640625 +gcs-1 | 2024/06/04 19:37:02.855 41.622657775878906 +gcs-1 | 2024/06/04 19:37:07.856 25.137908935546875 +gcs-1 | 2024/06/04 19:37:12.857 27.570457458496094 +gcs-1 | 2024/06/04 19:37:17.858 30.110198974609375 +gcs-1 | 2024/06/04 19:37:22.860 32.5474853515625 diff --git a/analysis/output_10b_run9_flush_mysql.txt b/analysis/output_10b_run9_flush_mysql.txt new file mode 100644 index 0000000..d97db98 --- /dev/null +++ b/analysis/output_10b_run9_flush_mysql.txt @@ -0,0 +1,270 @@ +gcs-1 | 2024/06/04 19:42:41 server listening at [::]:50000 +gcs-1 | 2024/06/04 19:42:41.847 server listening at [::]:50001 +gcs-1 | 2024/06/04 19:42:41.851 0.541778564453125 +gcs-1 | 2024/06/04 19:42:42.849 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:43.848 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:44.850 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:45.849 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:46.848 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:46.851 0.7560653686523438 +gcs-1 | 2024/06/04 19:42:51.853 1.3027725219726562 +gcs-1 | 2024/06/04 19:42:56.853 1.3161163330078125 +gcs-1 | 2024/06/04 19:43:01.861 1.3270416259765625 +gcs-1 | 2024/06/04 19:43:06.863 1.3676834106445312 +gcs-1 | 2024/06/04 19:43:11.864 1.397064208984375 +gcs-1 | 2024/06/04 19:43:16.873 1.5728912353515625 +gcs-1 | 2024/06/04 19:43:21.874 1.5522232055664062 +gcs-1 | 2024/06/04 19:43:26.876 1.4987716674804688 +gcs-1 | 2024/06/04 19:43:31.879 1.5016555786132812 +gcs-1 | 2024/06/04 19:43:36.881 1.460968017578125 +gcs-1 | 2024/06/04 19:43:41.881 1.4944305419921875 +gcs-1 | 2024/06/04 19:43:46.883 1.909912109375 +gcs-1 | 2024/06/04 19:43:51.883 1.6782989501953125 +gcs-1 | 2024/06/04 19:43:56.885 1.6936721801757812 +gcs-1 | 2024/06/04 19:44:01.887 1.6722183227539062 +gcs-1 | 2024/06/04 19:44:06.890 1.7350997924804688 +gcs-1 | 2024/06/04 19:44:11.891 1.7082443237304688 +gcs-1 | 2024/06/04 19:44:16.893 1.725311279296875 +gcs-1 | 2024/06/04 19:44:20.160 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:44:21.895 1.7982254028320312 +gcs-1 | 2024/06/04 19:44:26.897 1.6686325073242188 +gcs-1 | 2024/06/04 19:44:31.900 1.7048492431640625 +gcs-1 | 2024/06/04 19:44:36.900 1.8752365112304688 +gcs-1 | 2024/06/04 19:44:41.902 2.2953033447265625 +gcs-1 | 2024/06/04 19:44:46.903 2.2530364990234375 +gcs-1 | 2024/06/04 19:44:51.903 1.9637451171875 +gcs-1 | 2024/06/04 19:44:56.905 1.8707351684570312 +gcs-1 | 2024/06/04 19:45:01.906 2.0524215698242188 +gcs-1 | 2024/06/04 19:45:06.907 2.0872573852539062 +gcs-1 | 2024/06/04 19:45:11.909 1.9512405395507812 +gcs-1 | 2024/06/04 19:45:16.910 2.0131683349609375 +gcs-1 | 2024/06/04 19:45:21.911 1.9726409912109375 +gcs-1 | 2024/06/04 19:45:26.912 1.9984664916992188 +gcs-1 | 2024/06/04 19:45:31.915 2.1372756958007812 +gcs-1 | 2024/06/04 19:45:36.917 2.053253173828125 +gcs-1 | 2024/06/04 19:45:41.919 2.1457443237304688 +gcs-1 | 2024/06/04 19:45:46.920 2.1833419799804688 +gcs-1 | 2024/06/04 19:45:51.923 2.18670654296875 +gcs-1 | 2024/06/04 19:45:56.924 2.1562881469726562 +gcs-1 | 2024/06/04 19:46:01.926 2.2276992797851562 +gcs-1 | 2024/06/04 19:46:06.929 2.1830291748046875 +gcs-1 | 2024/06/04 19:46:11.931 2.2578582763671875 +gcs-1 | 2024/06/04 19:46:16.932 2.3423690795898438 +gcs-1 | 2024/06/04 19:46:21.933 3.4385833740234375 +gcs-1 | 2024/06/04 19:46:26.934 3.4460372924804688 +gcs-1 | 2024/06/04 19:46:31.936 3.4427566528320312 +gcs-1 | 2024/06/04 19:46:36.937 3.405426025390625 +gcs-1 | 2024/06/04 19:46:41.939 2.6701431274414062 +gcs-1 | 2024/06/04 19:46:46.939 2.7786788940429688 +gcs-1 | 2024/06/04 19:46:51.942 2.7306900024414062 +gcs-1 | 2024/06/04 19:46:56.943 2.7115554809570312 +gcs-1 | 2024/06/04 19:47:01.946 2.7729339599609375 +gcs-1 | 2024/06/04 19:47:06.947 2.6181869506835938 +gcs-1 | 2024/06/04 19:47:11.948 2.6241455078125 +gcs-1 | 2024/06/04 19:47:16.950 2.7991256713867188 +gcs-1 | 2024/06/04 19:47:21.951 2.685302734375 +gcs-1 | 2024/06/04 19:47:26.953 2.6424407958984375 +gcs-1 | 2024/06/04 19:47:31.954 2.83978271484375 +gcs-1 | 2024/06/04 19:47:36.955 2.7719802856445312 +gcs-1 | 2024/06/04 19:47:41.957 2.825897216796875 +gcs-1 | 2024/06/04 19:47:46.958 2.7015762329101562 +gcs-1 | 2024/06/04 19:47:51.961 2.6591110229492188 +gcs-1 | 2024/06/04 19:47:56.964 2.637481689453125 +gcs-1 | 2024/06/04 19:48:01.966 2.5851974487304688 +gcs-1 | 2024/06/04 19:48:06.967 2.6280364990234375 +gcs-1 | 2024/06/04 19:48:11.967 2.8046035766601562 +gcs-1 | 2024/06/04 19:48:16.968 2.7517776489257812 +gcs-1 | 2024/06/04 19:48:21.969 2.7795562744140625 +gcs-1 | 2024/06/04 19:48:26.970 2.7211380004882812 +gcs-1 | 2024/06/04 19:48:31.973 2.813690185546875 +gcs-1 | 2024/06/04 19:48:36.974 2.6896209716796875 +gcs-1 | 2024/06/04 19:48:41.975 2.9103164672851562 +gcs-1 | 2024/06/04 19:48:46.977 2.8389129638671875 +gcs-1 | 2024/06/04 19:48:51.978 2.7414321899414062 +gcs-1 | 2024/06/04 19:48:56.979 2.7215194702148438 +gcs-1 | 2024/06/04 19:49:01.980 2.7321243286132812 +gcs-1 | 2024/06/04 19:49:06.981 2.8215179443359375 +gcs-1 | 2024/06/04 19:49:11.982 2.7779006958007812 +gcs-1 | 2024/06/04 19:49:16.984 2.6975250244140625 +gcs-1 | 2024/06/04 19:49:21.988 2.7195587158203125 +gcs-1 | 2024/06/04 19:49:26.991 2.7353515625 +gcs-1 | 2024/06/04 19:49:31.996 5.162193298339844 +gcs-1 | 2024/06/04 19:49:36.997 5.352867126464844 +gcs-1 | 2024/06/04 19:49:41.998 5.135749816894531 +gcs-1 | 2024/06/04 19:49:47.001 5.236991882324219 +gcs-1 | 2024/06/04 19:49:52.002 5.279327392578125 +gcs-1 | 2024/06/04 19:49:57.003 5.358299255371094 +gcs-1 | 2024/06/04 19:50:02.005 5.383903503417969 +gcs-1 | 2024/06/04 19:50:07.007 5.256683349609375 +gcs-1 | 2024/06/04 19:50:12.008 3.896575927734375 +gcs-1 | 2024/06/04 19:50:17.009 3.8473968505859375 +gcs-1 | 2024/06/04 19:50:22.010 3.86724853515625 +gcs-1 | 2024/06/04 19:50:27.011 3.9947586059570312 +gcs-1 | 2024/06/04 19:50:32.010 4.004676818847656 +gcs-1 | 2024/06/04 19:50:37.010 3.9845046997070312 +gcs-1 | 2024/06/04 19:50:42.013 3.6995315551757812 +gcs-1 | 2024/06/04 19:50:47.015 3.89434814453125 +gcs-1 | 2024/06/04 19:50:52.015 3.8594207763671875 +gcs-1 | 2024/06/04 19:50:57.016 3.8529205322265625 +gcs-1 | 2024/06/04 19:51:02.021 3.822540283203125 +gcs-1 | 2024/06/04 19:51:07.022 3.98785400390625 +gcs-1 | 2024/06/04 19:51:12.023 3.8061752319335938 +gcs-1 | 2024/06/04 19:51:17.025 3.90289306640625 +gcs-1 | 2024/06/04 19:51:22.026 3.8698196411132812 +gcs-1 | 2024/06/04 19:51:27.028 3.9017105102539062 +gcs-1 | 2024/06/04 19:51:32.029 3.9386215209960938 +gcs-1 | 2024/06/04 19:51:37.031 3.9354934692382812 +gcs-1 | 2024/06/04 19:51:42.032 3.8931045532226562 +gcs-1 | 2024/06/04 19:51:47.033 4.007530212402344 +gcs-1 | 2024/06/04 19:51:52.034 3.92266845703125 +gcs-1 | 2024/06/04 19:51:57.036 3.8879776000976562 +gcs-1 | 2024/06/04 19:52:02.038 3.94140625 +gcs-1 | 2024/06/04 19:52:07.039 3.96728515625 +gcs-1 | 2024/06/04 19:52:12.040 4.01123046875 +gcs-1 | 2024/06/04 19:52:17.042 4.004814147949219 +gcs-1 | 2024/06/04 19:52:22.043 3.931427001953125 +gcs-1 | 2024/06/04 19:52:27.044 3.9497604370117188 +gcs-1 | 2024/06/04 19:52:32.046 4.100624084472656 +gcs-1 | 2024/06/04 19:52:37.049 3.8479232788085938 +gcs-1 | 2024/06/04 19:52:42.051 3.8843002319335938 +gcs-1 | 2024/06/04 19:52:47.054 3.8787612915039062 +gcs-1 | 2024/06/04 19:52:49.385 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:52:52.055 3.9020767211914062 +gcs-1 | 2024/06/04 19:52:53.949 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:52:57.057 3.3466644287109375 +gcs-1 | 2024/06/04 19:53:02.106 3.7256851196289062 +gcs-1 | 2024/06/04 19:53:07.109 3.9135284423828125 +gcs-1 | 2024/06/04 19:53:12.109 3.94085693359375 +gcs-1 | 2024/06/04 19:53:17.112 3.9469451904296875 +gcs-1 | 2024/06/04 19:53:22.113 4.1459503173828125 +gcs-1 | 2024/06/04 19:53:27.114 4.072853088378906 +gcs-1 | 2024/06/04 19:53:32.115 3.9909515380859375 +gcs-1 | 2024/06/04 19:53:37.116 3.9355239868164062 +gcs-1 | 2024/06/04 19:53:42.117 3.9438705444335938 +gcs-1 | 2024/06/04 19:53:47.119 4.1521148681640625 +gcs-1 | 2024/06/04 19:53:52.120 4.1165771484375 +gcs-1 | 2024/06/04 19:53:57.122 3.9645462036132812 +gcs-1 | 2024/06/04 19:54:02.123 4.008697509765625 +gcs-1 | 2024/06/04 19:54:07.126 3.9752349853515625 +gcs-1 | 2024/06/04 19:54:12.127 4.1572265625 +gcs-1 | 2024/06/04 19:54:14.593 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:54:17.135 3.5205612182617188 +gcs-1 | 2024/06/04 19:54:19.404 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:54:22.136 4.158515930175781 +gcs-1 | 2024/06/04 19:54:27.137 4.0652313232421875 +gcs-1 | 2024/06/04 19:54:31.919 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:54:32.127 3.5928192138671875 +gcs-1 | 2024/06/04 19:54:37.133 3.4572525024414062 +gcs-1 | 2024/06/04 19:54:42.134 4.091117858886719 +gcs-1 | 2024/06/04 19:54:47.136 4.213432312011719 +gcs-1 | 2024/06/04 19:54:52.136 4.0318450927734375 +gcs-1 | 2024/06/04 19:54:57.138 4.234596252441406 +gcs-1 | 2024/06/04 19:55:02.159 4.1728057861328125 +gcs-1 | 2024/06/04 19:55:07.161 4.115531921386719 +gcs-1 | 2024/06/04 19:55:12.161 4.257545471191406 +gcs-1 | 2024/06/04 19:55:17.164 4.3677978515625 +gcs-1 | 2024/06/04 19:55:22.164 4.366447448730469 +gcs-1 | 2024/06/04 19:55:27.166 4.489662170410156 +gcs-1 | 2024/06/04 19:55:32.168 4.238433837890625 +gcs-1 | 2024/06/04 19:55:37.170 4.231101989746094 +gcs-1 | 2024/06/04 19:55:42.171 4.193916320800781 +gcs-1 | 2024/06/04 19:55:47.172 4.2407073974609375 +gcs-1 | 2024/06/04 19:55:52.175 4.24365234375 +gcs-1 | 2024/06/04 19:55:57.178 4.167854309082031 +gcs-1 | 2024/06/04 19:56:02.179 9.159080505371094 +gcs-1 | 2024/06/04 19:56:07.180 9.027717590332031 +gcs-1 | 2024/06/04 19:56:11.936 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:56:12.181 8.815505981445312 +gcs-1 | 2024/06/04 19:56:13.885 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:56:17.183 9.644462585449219 +gcs-1 | 2024/06/04 19:56:22.184 9.179283142089844 +gcs-1 | 2024/06/04 19:56:27.185 9.074821472167969 +gcs-1 | 2024/06/04 19:56:32.193 9.318168640136719 +gcs-1 | 2024/06/04 19:56:37.194 9.010856628417969 +gcs-1 | 2024/06/04 19:56:42.195 9.003768920898438 +gcs-1 | 2024/06/04 19:56:47.197 9.423927307128906 +gcs-1 | 2024/06/04 19:56:52.199 9.155105590820312 +gcs-1 | 2024/06/04 19:56:57.200 9.069099426269531 +gcs-1 | 2024/06/04 19:57:02.201 9.386749267578125 +gcs-1 | 2024/06/04 19:57:07.203 9.30413818359375 +gcs-1 | 2024/06/04 19:57:12.206 9.159759521484375 +gcs-1 | 2024/06/04 19:57:17.207 6.3763427734375 +gcs-1 | 2024/06/04 19:57:22.208 6.4787445068359375 +gcs-1 | 2024/06/04 19:57:27.208 6.401405334472656 +gcs-1 | 2024/06/04 19:57:32.219 5.834190368652344 +gcs-1 | 2024/06/04 19:57:37.221 6.562278747558594 +gcs-1 | 2024/06/04 19:57:42.223 6.282386779785156 +gcs-1 | 2024/06/04 19:57:47.223 6.442680358886719 +gcs-1 | 2024/06/04 19:57:52.226 6.404937744140625 +gcs-1 | 2024/06/04 19:57:57.226 6.2392578125 +gcs-1 | 2024/06/04 19:58:02.228 6.125709533691406 +gcs-1 | 2024/06/04 19:58:07.253 6.2435150146484375 +gcs-1 | 2024/06/04 19:58:09.190 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:58:12.255 6.359710693359375 +gcs-1 | 2024/06/04 19:58:17.258 6.2640838623046875 +gcs-1 | 2024/06/04 19:58:22.259 6.612174987792969 +gcs-1 | 2024/06/04 19:58:27.261 6.179771423339844 +gcs-1 | 2024/06/04 19:58:32.263 6.527679443359375 +gcs-1 | 2024/06/04 19:58:37.266 6.354972839355469 +gcs-1 | 2024/06/04 19:58:42.267 6.275825500488281 +gcs-1 | 2024/06/04 19:58:47.268 6.4571990966796875 +gcs-1 | 2024/06/04 19:58:52.273 6.233741760253906 +gcs-1 | 2024/06/04 19:58:54.893 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:58:57.278 6.1600494384765625 +gcs-1 | 2024/06/04 19:59:02.277 6.147499084472656 +gcs-1 | 2024/06/04 19:59:07.279 6.4554595947265625 +gcs-1 | 2024/06/04 19:59:12.280 6.414337158203125 +gcs-1 | 2024/06/04 19:59:17.281 6.2513885498046875 +gcs-1 | 2024/06/04 19:59:22.282 6.390953063964844 +gcs-1 | 2024/06/04 19:59:27.284 6.281791687011719 +gcs-1 | 2024/06/04 19:59:32.289 6.332313537597656 +gcs-1 | 2024/06/04 19:59:37.375 6.100608825683594 +gcs-1 | 2024/06/04 19:59:37.945 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:59:39.899 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:59:42.376 5.796142578125 +gcs-1 | 2024/06/04 19:59:47.377 6.586235046386719 +gcs-1 | 2024/06/04 19:59:52.378 6.6221466064453125 +gcs-1 | 2024/06/04 19:59:57.389 6.4645538330078125 +gcs-1 | 2024/06/04 20:00:02.391 6.424186706542969 +gcs-1 | 2024/06/04 20:00:07.392 6.462791442871094 +gcs-1 | 2024/06/04 20:00:09.892 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:12.394 6.034141540527344 +gcs-1 | 2024/06/04 20:00:17.395 6.5565948486328125 +gcs-1 | 2024/06/04 20:00:22.396 6.491447448730469 +gcs-1 | 2024/06/04 20:00:27.397 6.366546630859375 +gcs-1 | 2024/06/04 20:00:32.401 5.9578704833984375 +gcs-1 | 2024/06/04 20:00:35.903 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:37.404 6.703330993652344 +gcs-1 | 2024/06/04 20:00:42.407 5.6876983642578125 +gcs-1 | 2024/06/04 20:00:47.407 6.803955078125 +gcs-1 | 2024/06/04 20:00:52.038 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:52.409 6.2969512939453125 +gcs-1 | 2024/06/04 20:00:53.894 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:56.003 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:57.410 7.14398193359375 +gcs-1 | 2024/06/04 20:00:57.945 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:01:02.412 6.350189208984375 +gcs-1 | 2024/06/04 20:01:07.413 5.6882781982421875 +gcs-1 | 2024/06/04 20:01:12.427 5.788665771484375 +gcs-1 | 2024/06/04 20:01:17.429 5.781158447265625 +gcs-1 | 2024/06/04 20:01:22.431 5.6903533935546875 +gcs-1 | 2024/06/04 20:01:27.432 6.453605651855469 +gcs-1 | 2024/06/04 20:01:32.437 6.3587493896484375 +gcs-1 | 2024/06/04 20:01:37.438 5.68914794921875 +gcs-1 | 2024/06/04 20:01:42.439 6.609962463378906 +gcs-1 | 2024/06/04 20:01:47.440 5.8092193603515625 +gcs-1 | 2024/06/04 20:01:52.441 6.6129302978515625 +gcs-1 | 2024/06/04 20:01:57.441 5.7250823974609375 +gcs-1 | 2024/06/04 20:02:02.442 5.694068908691406 +gcs-1 | 2024/06/04 20:02:07.442 6.5864715576171875 +gcs-1 | 2024/06/04 20:02:12.443 5.6991119384765625 +gcs-1 | 2024/06/04 20:02:17.444 6.4114227294921875 +gcs-1 | 2024/06/04 20:02:22.445 6.061088562011719 +gcs-1 | 2024/06/04 20:02:27.448 6.7945404052734375 +gcs-1 | 2024/06/04 20:02:32.451 5.9176483154296875 +gcs-1 | 2024/06/04 20:02:37.453 6.7691192626953125 +gcs-1 | 2024/06/04 20:02:42.113 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:02:42.456 6.22845458984375 +gcs-1 | 2024/06/04 20:02:47.457 6.676582336425781 +gcs-1 | 2024/06/04 20:02:52.459 6.405731201171875 +gcs-1 | 2024/06/04 20:02:57.460 5.7739105224609375 diff --git a/analysis/parse_10b.py b/analysis/parse_10b.py new file mode 100644 index 0000000..6c60916 --- /dev/null +++ b/analysis/parse_10b.py @@ -0,0 +1,54 @@ +import re +import csv +from datetime import datetime + +def parse_log_line(line): + pattern = r'^worker\d+\-\d+\s+\|\s+(\d{4}\/\d{2}\/\d{2}\s+\d{2}:\d{2}:\d{2}\.\d{3})\s+FIG10b:\s+Memory\s+usage\s+without\s+flush\s+after\s+(\d+)\s+tasks:\s+(\d+\.\d+)\s+MB' + match = re.match(pattern, line) + if match: + timestamp = datetime.strptime(match.group(1), '%Y/%m/%d %H:%M:%S.%f') + tasks = int(match.group(2)) + memory_usage = float(match.group(3)) + return (timestamp, memory_usage) + else: + return None + +def parse_logs(log_data): + result = [] + for line in log_data.split('\n'): + parsed = parse_log_line(line) + if parsed: + result.append(parsed) + return result + +def write_to_csv(data, filename): + with open(filename, 'w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(['Timestamp', 'Memory Usage (MB)']) + for timestamp, memory_usage in data: + writer.writerow([timestamp, memory_usage]) + +# Unit test +def test_parse_log_line(): + test_line = 'worker1-1 | 2024/06/04 05:02:21.134 FIG10b: Memory usage without flush after 1459 tasks: 1060.06640625 MB' + expected_output = (datetime(2024, 6, 4, 5, 2, 21, 134000), 1060.06640625) + assert parse_log_line(test_line) == expected_output + +# Example usage +if __name__ == '__main__': + test_parse_log_line() # Run the unit test + print("unit test worked") + + with open('output_10b_run2_300s_flush_2b_memory_limit.txt', 'r') as f: + log_data = f.read() + + parsed_logs = parse_logs(log_data) + write_to_csv(parsed_logs, 'output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv') + + # ================================= + + with open('output_10b_run3_40s_flush_2gb_memory_limit.txt', 'r') as f: + log_data = f.read() + + parsed_logs = parse_logs(log_data) + write_to_csv(parsed_logs, 'output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv') \ No newline at end of file diff --git a/analysis/plot_10b.py b/analysis/plot_10b.py new file mode 100644 index 0000000..614f1f0 --- /dev/null +++ b/analysis/plot_10b.py @@ -0,0 +1,114 @@ +import matplotlib.pyplot as plt +import csv +from datetime import datetime + +# Read data from the CSV file +x1 = [] # Elapsed time (seconds) +y1 = [] # GCS Used Memory (MB) + +with open('output_10b_run1_cleaned.csv', 'r') as csvfile: + plots = csv.reader(csvfile) + header = next(plots) # Skip the header row + timestamp_baseline = None + memory_baseline = None + + for row in plots: + timestamp_str, memory = row + timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + + if timestamp_baseline is None: + timestamp_baseline = timestamp + memory_baseline = float(memory) + + elapsed_time = (timestamp - timestamp_baseline).total_seconds() + memory_used = float(memory) - memory_baseline + + x1.append(elapsed_time) + y1.append(memory_used) + +# ===== +# Read data from the CSV file +x2 = [] # Elapsed time (seconds) +y2 = [] # GCS Used Memory (MB) + +with open('output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv', 'r') as csvfile: + plots = csv.reader(csvfile) + header = next(plots) # Skip the header row + timestamp_baseline = None + memory_baseline = None + + for row in plots: + timestamp_str, memory = row + timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + + if timestamp_baseline is None: + timestamp_baseline = timestamp + memory_baseline = float(memory) + + elapsed_time = (timestamp - timestamp_baseline).total_seconds() + memory_used = float(memory) - memory_baseline + + x2.append(elapsed_time) + y2.append(memory_used) + +# ===== + +# Read data from the CSV file +x3 = [] # Elapsed time (seconds) +y3 = [] # GCS Used Memory (MB) + +with open('output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv', 'r') as csvfile: + plots = csv.reader(csvfile) + header = next(plots) # Skip the header row + timestamp_baseline = None + memory_baseline = None + + for row in plots: + timestamp_str, memory = row + timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + + if timestamp_baseline is None: + timestamp_baseline = timestamp + memory_baseline = float(memory) + + elapsed_time = (timestamp - timestamp_baseline).total_seconds() + memory_used = float(memory) - memory_baseline + + x3.append(elapsed_time) + y3.append(memory_used) + +# ===== + +def attempt1(): + # Create the plot + plt.plot(x1, y1, label="Baby Ray, no GCS flush", linestyle='--', color='blue') + plt.plot(x2, y2, label="Baby Ray, GCS flush per 40s", color='orange') + + # Customize the plot + plt.xlabel("Elapsed time (seconds)") + plt.ylabel("GCS Used Memory (MB)") + plt.title("Memory Usage Over Time") + plt.legend(loc="lower right") + + # Show the plot + plt.grid(True) + plt.show() + +def attempt2(): + # Create the plot + plt.plot(x1, y1, label="Baby Ray, no GCS flush", linestyle='--', color='blue') + plt.plot(x3, y3, label="Baby Ray, GCS flush per 300s", color='orange') + + # Customize the plot + plt.xlabel("Elapsed time (seconds)") + plt.ylabel("GCS Used Memory (MB)") + plt.title("50 million no-op tasks") + plt.legend(loc="lower right") + + # Show the plot + # Save the plot as a high-quality PNG + plt.savefig("fig10b_graphs/attempt2_sharp.png", dpi=300) + plt.show() + +if __name__ == '__main__': + attempt2() \ No newline at end of file diff --git a/config/app_config.yaml b/config/app_config.yaml index e38bf7b..2509d0d 100644 --- a/config/app_config.yaml +++ b/config/app_config.yaml @@ -14,3 +14,5 @@ ports: global_scheduler: 50000 dns: node_prefix: "node" +gcs: + flush_interval_sec: 1 # put -1 to never flush diff --git a/docker-compose.fig10b.yml b/docker-compose.fig10b.yml new file mode 100644 index 0000000..3d4fb12 --- /dev/null +++ b/docker-compose.fig10b.yml @@ -0,0 +1,11 @@ +version: '3' + +services: + worker1: + command: ["/bin/sh", "-c", "python3 -u -m pythonserver.worker & ./go/bin/localobjstore & ./go/bin/localscheduler & sleep 5 && python3 -u scripts/fig10b.py & sleep infinity"] + depends_on: + # - zookeeper + - worker2 + - worker3 + - gcs + - global_scheduler \ No newline at end of file diff --git a/go/cmd/gcsobjtable/db.go b/go/cmd/gcsobjtable/db.go new file mode 100644 index 0000000..540a211 --- /dev/null +++ b/go/cmd/gcsobjtable/db.go @@ -0,0 +1,129 @@ +package main + +import ( + "database/sql" + "strings" +) + +func setCacheSizeToZero(db *sql.DB) error { + _, err := db.Exec("PRAGMA cache_size = 0;") + return err +} + +func createObjectLocationsTable(db *sql.DB) error { + createTableSQL := `CREATE TABLE IF NOT EXISTS object_locations ( + "object_uid" INTEGER NOT NULL, + "node_id" INTEGER NOT NULL, + PRIMARY KEY (object_uid, node_id) + );` + + createIndexSQL := `CREATE INDEX IF NOT EXISTS idx_object_uid_object_locations ON object_locations (object_uid);` + + // Create object_locations table + statement, err := db.Prepare(createTableSQL) + if err != nil { + return err + } + _, err = statement.Exec() + if err != nil { + return err + } + + // Create index on object_uid for object_locations table + statement, err = db.Prepare(createIndexSQL) + if err != nil { + return err + } + _, err = statement.Exec() + if err != nil { + return err + } + + return nil +} + +// We assume that, for any object_uid in the possession of memory, it has the latest state +// on all of the locations that it can be found, and hence it is imperate to overwrite the +// corresponding "state" in the database. That is, the database might have a location which in fact +// no longer exists for this object_uid, so we need to eliminate that from the database. +func insertOrUpdateObjectLocations(db *sql.DB, objectLocations map[uint64][]uint64) error { + // NOTE ON IMPLEMENTATION DECISION: + // We DELETE all rows that have object_uid and then INSERT anew because this is a lot more + // efficient than having to query the db repeatedly to do comparisons about which rows + // no longer exist in the state, and trying to selectively delete those rows. + + // Begin a transaction + tx, err := db.Begin() + if err != nil { + return err + } + + // Prepare the bulk delete statement + objectUIDs := make([]string, 0, len(objectLocations)) + deleteArgs := make([]interface{}, 0, len(objectLocations)) + for objectUID := range objectLocations { + objectUIDs = append(objectUIDs, "?") + deleteArgs = append(deleteArgs, objectUID) + } + deleteSQL := `DELETE FROM object_locations WHERE object_uid IN (` + strings.Join(objectUIDs, ",") + `);` + + // Prepare the bulk insert statement + insertSQL := `INSERT INTO object_locations (object_uid, node_id) VALUES ` + valueStrings := []string{} + valueArgs := []interface{}{} + + for objectUID, nodeIDs := range objectLocations { + for _, nodeID := range nodeIDs { + valueStrings = append(valueStrings, "(?, ?)") + valueArgs = append(valueArgs, objectUID, nodeID) + } + } + + insertSQL = insertSQL + strings.Join(valueStrings, ",") + + // Execute the bulk delete and insert at the end of preparation + if _, err := tx.Exec(deleteSQL, deleteArgs...); err != nil { + tx.Rollback() + return err + } + + if _, err := tx.Exec(insertSQL, valueArgs...); err != nil { + tx.Rollback() + return err + } + + // Commit the transaction + err = tx.Commit() + if err != nil { + return err + } + + return nil +} + +// returns an empty []uint64 if nothing found +func getObjectLocations(db *sql.DB, objectUID uint64) ([]uint64, error) { + querySQL := `SELECT node_id FROM object_locations WHERE object_uid = ?;` + + rows, err := db.Query(querySQL, objectUID) + if err != nil { + return nil, err + } + defer rows.Close() + + var nodeIDs []uint64 + for rows.Next() { + var nodeID uint64 + if err := rows.Scan(&nodeID); err != nil { + return nil, err + } + nodeIDs = append(nodeIDs, nodeID) + } + + // Check for errors from iterating over rows. + if err = rows.Err(); err != nil { + return nil, err + } + + return nodeIDs, nil +} diff --git a/go/cmd/gcsobjtable/db_test.go b/go/cmd/gcsobjtable/db_test.go new file mode 100644 index 0000000..97fef9c --- /dev/null +++ b/go/cmd/gcsobjtable/db_test.go @@ -0,0 +1,216 @@ +package main + +import ( + "database/sql" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +func TestSetCacheSizeToZero(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("Failed to open database: %v", err) + } + defer db.Close() + + err = setCacheSizeToZero(db) + if err != nil { + t.Errorf("Failed to set cache size to zero: %v", err) + } + + var cacheSize int + err = db.QueryRow("PRAGMA cache_size;").Scan(&cacheSize) + if err != nil { + t.Fatalf("Failed to query cache size: %v", err) + } + + if cacheSize != 0 { + t.Errorf("Cache size is not zero, got: %d", cacheSize) + } +} + +func TestCreateObjectLocationsTable(t *testing.T) { + // Use an in-memory SQLite database for testing + database, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("Failed to open database: %v", err) + } + defer database.Close() + + // Call the function to create the object_locations table and index + if err := createObjectLocationsTable(database); err != nil { + t.Fatalf("Failed to create object_locations table: %v", err) + } + + // Check if object_locations table exists + var tableName string + err = database.QueryRow("SELECT name FROM sqlite_master WHERE type='table' AND name='object_locations';").Scan(&tableName) + if err != nil { + t.Fatalf("object_locations table was not created: %v", err) + } + + // Check if index on object_uid for object_locations table exists + err = database.QueryRow("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_object_uid_object_locations';").Scan(&tableName) + if err != nil { + t.Fatalf("Index on object_uid for object_locations table was not created: %v", err) + } +} + +func TestInsertOrUpdateObjectLocations(t *testing.T) { + // Use an in-memory SQLite database for testing + database, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("Failed to open database: %v", err) + } + defer database.Close() + + // Call the function to create the object_locations table + if err := createObjectLocationsTable(database); err != nil { + t.Fatalf("Failed to create object_locations table: %v", err) + } + + // Initial example data + objectLocations := map[uint64][]uint64{ + 1: {100, 101}, + 2: {102, 103}, + } + + // Insert initial data + if err := insertOrUpdateObjectLocations(database, objectLocations); err != nil { + t.Fatalf("Failed to insert/update object locations: %v", err) + } + + // Verify initial data + verifyObjectLocations(t, database, objectLocations) + + // Update example data + objectLocations = map[uint64][]uint64{ + 1: {100, 102}, // Change node_ids for object_uid 1 + 3: {104}, // Add new object_uid 3 + } + + // Insert updated data + if err := insertOrUpdateObjectLocations(database, objectLocations); err != nil { + t.Fatalf("Failed to insert/update object locations: %v", err) + } + + // Verify updated data + verifyObjectLocations(t, database, objectLocations) +} + +func verifyObjectLocations(t *testing.T, db *sql.DB, expected map[uint64][]uint64) { + // Extract the keys from the expected map + keys := make([]uint64, 0, len(expected)) + for k := range expected { + keys = append(keys, k) + } + + // Build the query with placeholders + query := "SELECT object_uid, node_id FROM object_locations WHERE object_uid IN (" + for i := range keys { + if i > 0 { + query += "," + } + query += "?" + } + query += ")" + + // Convert keys to a slice of interface{} for the query arguments + args := make([]interface{}, len(keys)) + for i, v := range keys { + args[i] = v + } + + // Query the database + rows, err := db.Query(query, args...) + if err != nil { + t.Fatalf("Failed to query object_locations: %v", err) + } + defer rows.Close() + + result := make(map[uint64][]uint64) + for rows.Next() { + var objectUID uint64 + var nodeID uint64 + if err := rows.Scan(&objectUID, &nodeID); err != nil { + t.Fatalf("Failed to scan row: %v", err) + } + result[objectUID] = append(result[objectUID], nodeID) + } + + // Compare results + if len(result) != len(expected) { + t.Fatalf("Expected %d entries, got %d", len(expected), len(result)) + } + + for objectUID, nodeIDs := range expected { + if len(result[objectUID]) != len(nodeIDs) { + t.Fatalf("For objectUID %d, expected %d nodeIDs, got %d", objectUID, len(nodeIDs), len(result[objectUID])) + } + for i, nodeID := range nodeIDs { + if result[objectUID][i] != nodeID { + t.Fatalf("For objectUID %d, at index %d, expected nodeID %d, got %d", objectUID, i, nodeID, result[objectUID][i]) + } + } + } +} + +func TestGetObjectLocations(t *testing.T) { + // Create an in-memory SQLite database + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("Failed to open database: %v", err) + } + defer db.Close() + + // Create the object_locations table + err = createObjectLocationsTable(db) + if err != nil { + t.Fatalf("Failed to create object_locations table: %v", err) + } + + // Insert test data + objectLocations := map[uint64][]uint64{ + 1: {101, 102, 103}, + 2: {201, 202}, + } + + err = insertOrUpdateObjectLocations(db, objectLocations) + if err != nil { + t.Fatalf("Failed to insert test data: %v", err) + } + + // Test getObjectLocations + tests := []struct { + objectUID uint64 + expected []uint64 + }{ + {1, []uint64{101, 102, 103}}, + {2, []uint64{201, 202}}, + {3, []uint64{}}, // Test for an objectUID that does not exist + } + + for _, test := range tests { + nodeIDs, err := getObjectLocations(db, test.objectUID) + if err != nil { + t.Errorf("Error getting object locations for objectUID %d: %v", test.objectUID, err) + } + if !equal(nodeIDs, test.expected) { + t.Errorf("Expected %v, got %v for objectUID %d", test.expected, nodeIDs, test.objectUID) + } + } +} + +// Helper function to compare two slices for equality +func equal(a, b []uint64) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} diff --git a/go/cmd/gcsobjtable/main.go b/go/cmd/gcsobjtable/main.go index aa6aac9..c6b9ceb 100644 --- a/go/cmd/gcsobjtable/main.go +++ b/go/cmd/gcsobjtable/main.go @@ -1,26 +1,35 @@ package main import ( + "bufio" "context" + "os" "time" // "errors" + "fmt" "log" "math/rand" "net" + "runtime" "strconv" "sync" - "fmt" + "github.com/rodrigo-castellon/babyray/config" - "github.com/rodrigo-castellon/babyray/util" "github.com/rodrigo-castellon/babyray/customlog" pb "github.com/rodrigo-castellon/babyray/pkg" + "github.com/rodrigo-castellon/babyray/util" "google.golang.org/grpc" + // "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/peer" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "database/sql" + + _ "github.com/mattn/go-sqlite3" ) var cfg *config.Config @@ -37,6 +46,7 @@ func LocalLog(format string, v ...interface{}) { } func main() { + /* Set up GCS Object Table */ customlog.Init() cfg = config.GetConfig() // Load configuration address := ":" + strconv.Itoa(cfg.Ports.GCSObjectTable) // Prepare the network address @@ -47,8 +57,8 @@ func main() { } _ = lis s := grpc.NewServer(util.GetServerOptions()...) - pb.RegisterGCSObjServer(s, NewGCSObjServer()) - //log.Printf("server listening at %v", lis.Addr()) + pb.RegisterGCSObjServer(s, NewGCSObjServer(cfg.GCS.FlushIntervalSec)) + // log.Printf("server listening at %v", lis.Addr()) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } @@ -57,53 +67,191 @@ func main() { // server is used to implement your gRPC service. type GCSObjServer struct { pb.UnimplementedGCSObjServer - objectLocations map[uint64][]uint64 // object uid -> list of nodeIds as uint64 - waitlist map[uint64][]string // object uid -> list of IP addresses as string - mu sync.Mutex // lock should be used for both objectLocations and waitlist - lineageMu sync.RWMutex // lock should be used for s.lineage - objectSizes map[uint64]uint64 - lineage map[uint64]*pb.GlobalScheduleRequest + objectLocations map[uint64][]uint64 // object uid -> list of nodeIds as uint64 + waitlist map[uint64][]string // object uid -> list of IP addresses as string + mu sync.Mutex // lock should be used for both objectLocations and waitlist + database *sql.DB // connection to SQLite persistent datastore + ticker *time.Ticker // for GCS flushing + lineageMu sync.RWMutex // lock should be used for s.lineage + objectSizes map[uint64]uint64 + lineage map[uint64]*pb.GlobalScheduleRequest globalSchedulerClient SchedulerClient - liveNodes map[uint64]bool - generating map[uint64]uint64 //object uid -> node id of original generator - //used to determine when an original creation of a uid should be restarted + liveNodes map[uint64]bool + generating map[uint64]uint64 //object uid -> node id of original generator + //used to determine when an original creation of a uid should be restarted globalCtx context.Context } type SchedulerClient interface { - Schedule(ctx context.Context , req *pb.GlobalScheduleRequest, opts ...grpc.CallOption ) (*pb.StatusResponse, error) - Heartbeat(ctx context.Context, req *pb.HeartbeatRequest, opts ...grpc.CallOption ) (*pb.StatusResponse, error) - + Schedule(ctx context.Context, req *pb.GlobalScheduleRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error) + Heartbeat(ctx context.Context, req *pb.HeartbeatRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error) } -func NewGCSObjServer() *GCSObjServer { +/* set flushIntervalSec to -1 to disable GCS flushing */ +func NewGCSObjServer(flushIntervalSec int) *GCSObjServer { + /* Set up SQLite */ + // Note: You don't need to call database.Close() in Golang: https://stackoverflow.com/a/50788205 + database, err := sql.Open("sqlite3", "./gcsobjtable.db") // TODO: Remove hardcode to config + if err != nil { + log.Fatal(err) + } + setCacheSizeToZero(database) + // Creates table if it doesn't already exist + createObjectLocationsTable(database) + + /* Create server object */ + globalSchedulerAddress := fmt.Sprintf("%s%d:%d", cfg.DNS.NodePrefix, cfg.NodeIDs.GlobalScheduler, cfg.Ports.GlobalScheduler) conn, _ := grpc.Dial(globalSchedulerAddress, grpc.WithInsecure()) globalSchedulerClient := pb.NewGlobalSchedulerClient(conn) server := &GCSObjServer{ - objectLocations: make(map[uint64][]uint64), - waitlist: make(map[uint64][]string), - mu: sync.Mutex{}, - objectSizes: make(map[uint64]uint64), - lineage: make(map[uint64]*pb.GlobalScheduleRequest), + objectLocations: make(map[uint64][]uint64), + waitlist: make(map[uint64][]string), + mu: sync.Mutex{}, + database: database, + ticker: nil, + objectSizes: make(map[uint64]uint64), + lineage: make(map[uint64]*pb.GlobalScheduleRequest), globalSchedulerClient: globalSchedulerClient, - liveNodes: make(map[uint64]bool), - generating: make(map[uint64]uint64), - globalCtx: context.Background(), + liveNodes: make(map[uint64]bool), + generating: make(map[uint64]uint64), + globalCtx: context.Background(), + } + if flushIntervalSec != -1 { + // Launch periodic disk flushing + interval := time.Duration(flushIntervalSec) * time.Second + server.ticker = time.NewTicker(interval) + go func() { + for range server.ticker.C { + err := server.flushToDisk() + if err != nil { + log.Printf("Error flushing to disk: %v", err) + } else { + //log.Printf("Successfully flushed to disk!") + } + } + }() + } + + logMemoryUsage := true // TODO: REMOVE HARDCODE + if logMemoryUsage { + go func() { + for { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + // HeapAlloc: Bytes of allocated heap objects (heap memory in use). in MB + log.Printf("%v\n", bToMb(memStats.HeapAlloc)) + + // Sleep for 5 seconds before the next iteration + time.Sleep(5 * time.Second) + } + }() } + return server } +func bToMb(b uint64) float64 { + return float64(b) / 1024 / 1024 +} + +func (s *GCSObjServer) flushToDisk() error { + s.mu.Lock() + defer s.mu.Unlock() + garbage_collect := true // TODO: REMOVE HARDCODED + flush_to_AOF := false // TODO: REMOVE HARDCODED + aof_filename := "./aof.txt" // TODO: REMOVE HARDCODED + + // Can either flush to append-only file or to DB + if flush_to_AOF { + err := WriteObjectLocationsToAOF(aof_filename, s.objectLocations) + if err != nil { + return err + } + //log.Printf("Successfully flushed to AOF!") + } else { + // Flush to SQLite3 Disk Database + err := insertOrUpdateObjectLocations(s.database, s.objectLocations) + if err != nil { + return err + } + } + + // Completely delete the current map in memory and start blank + s.objectLocations = make(map[uint64][]uint64) // orphaning the old map will get it garbage collected + // Manually trigger garbage collection if desired + if garbage_collect { + runtime.GC() + //log.Println("Garbage collection triggered") + } + return nil +} + +// WriteObjectLocationsToAOF appends the contents of the objectLocations map to a file +func WriteObjectLocationsToAOF(filename string, objectLocations map[uint64][]uint64) error { + // Open file in append mode, create if it doesn't exist + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("error opening file: %w", err) + } + defer file.Close() + + // Create a buffered writer + writer := bufio.NewWriter(file) + + // Iterate over the map and write to file + for key, values := range objectLocations { + // Convert key to string + keyStr := strconv.FormatUint(key, 10) + + // Convert values slice to a comma-separated string + var valuesStr string + for i, val := range values { + if i > 0 { + valuesStr += "," + } + valuesStr += strconv.FormatUint(val, 10) + } + + // Write key and values to file + _, err := writer.WriteString(fmt.Sprintf("%s: [%s]\n", keyStr, valuesStr)) + if err != nil { + return fmt.Errorf("error writing to file: %w", err) + } + } + + // Flush the buffered writer + if err := writer.Flush(); err != nil { + return fmt.Errorf("error flushing buffer: %w", err) + } + + return nil +} + /* -Returns a nodeId that has object uid. If it has never been added to objectLocations, -then return false. Otherwise, return True. +Returns a nodeId that has object uid. If it has never been added to objectLocations, +then return false. Otherwise, return True. Assumes that s's mutex is locked. +The return values are the location, boolean if it exists, and error */ -func (s *GCSObjServer) getNodeId(uid uint64) (*uint64) { +func (s *GCSObjServer) getNodeId(uid uint64) (*uint64, bool, error) { nodeIds, exists := s.objectLocations[uid] if !exists || len(nodeIds) == 0 { - return nil + // Not found in memory; it's a cache miss so let's try disk + var err error + nodeIds, err = getObjectLocations(s.database, uid) + if err != nil { + return nil, false, err + } + // Move state into cache, even if it's empty so we know how to answer in the future + s.objectLocations[uid] = nodeIds + if len(nodeIds) == 0 { + // Not found + return nil, false, nil + } + // Proceed below } nodesToReturn := make([]uint64, 1, 1) @@ -114,13 +262,13 @@ func (s *GCSObjServer) getNodeId(uid uint64) (*uint64) { } if len(nodesToReturn) == 0 { - return nil + return nil, false, nil } // Note: policy is to pick a random one; in the future it will need to be locality-based randomIndex := rand.Intn(len(nodesToReturn)) nodeId := &nodesToReturn[randomIndex] - return nodeId + return nodeId, true, nil } // sendCallback sends a location found callback to the local object store client @@ -132,7 +280,7 @@ func (s *GCSObjServer) sendCallback(clientAddress string, uid uint64, nodeId uin conn, err := grpc.Dial(clientAddress, util.GetDialOptions()...) if err != nil { // Log the error instead of returning it - // log.Printf("Failed to connect back to client at %s: %v", clientAddress, err) + // log.Printf("Failed to connect back to client at %s: %v", clientAddress, err) return } defer conn.Close() // TODO: remove in some eventual universe @@ -145,7 +293,7 @@ func (s *GCSObjServer) sendCallback(clientAddress string, uid uint64, nodeId uin // Call LocationFound and handle any potential error _, err = localObjStoreClient.LocationFound(ctx, &pb.LocationFoundCallback{Uid: uid, Location: nodeId}) if err != nil { - // log.Printf("Failed to send LocationFound callback for UID %d to client at %s: %v", uid, clientAddress, err) + // log.Printf("Failed to send LocationFound callback for UID %d to client at %s: %v", uid, clientAddress, err) } } @@ -153,8 +301,6 @@ func (s *GCSObjServer) NotifyOwns(ctx context.Context, req *pb.NotifyOwnsRequest s.mu.Lock() defer s.mu.Unlock() - //log.Printf("WAS JUST NOTIFYOWNS()ED") - uid, nodeId := req.Uid, req.NodeId delete(s.generating, uid) // Append the nodeId to the list for the given object uid @@ -187,11 +333,6 @@ func (s *GCSObjServer) RequestLocation(ctx context.Context, req *pb.RequestLocat return nil, status.Error(codes.Internal, "could not get peer information; hence, failed to extract client address") } - // ///// - // log.SetOutput(os.Stdout) - // log.Println(p.Addr.String()) - // ////////// - host, _, err := net.SplitHostPort(p.Addr.String()) // Strip out the ephemeral port if err != nil { //return nil, status.Error(codes.Internal, "could not split host and port") @@ -202,11 +343,12 @@ func (s *GCSObjServer) RequestLocation(ctx context.Context, req *pb.RequestLocat clientAddress := net.JoinHostPort(host, clientPort) uid := req.Uid - // LocalLog("Starting get Node ID") - nodeId := s.getNodeId(uid) - // LocalLog("finished get node ID") - // LocalLog("node id = %v", nodeId) - if nodeId == nil { + nodeId, exists, err := s.getNodeId(uid) + if err != nil { + return nil, status.Error(codes.Internal, "could not get node id: "+err.Error()) + } + + if !exists { // Add client to waiting list if _, waiting := s.waitlist[uid]; !waiting { s.waitlist[uid] = []string{} // Initialize slice if it doesn't exist @@ -230,14 +372,9 @@ func (s *GCSObjServer) RequestLocation(ctx context.Context, req *pb.RequestLocat func (s *GCSObjServer) GetObjectLocations(ctx context.Context, req *pb.ObjectLocationsRequest) (*pb.ObjectLocationsResponse, error) { locations := make(map[uint64]*pb.LocationByteTuple) - // log.Printf("DEEP PRINT!") - // log.Printf("length = %v", len(s.objectLocations)) - // for k, v := range s.objectLocations { - // log.Printf("s.objectLocations[%v] = %v", k, v) - // } for _, u := range req.Args { - if _,ok := s.objectLocations[uint64(u)]; ok { + if _, ok := s.objectLocations[uint64(u)]; ok { locations[uint64(u)] = &pb.LocationByteTuple{Locations: s.objectLocations[uint64(u)], Bytes: s.objectSizes[uint64(u)]} } } @@ -258,7 +395,7 @@ func (s *GCSObjServer) RegisterLineage(ctx context.Context, req *pb.GlobalSchedu } func (s *GCSObjServer) getNodes(uid uint64) []uint64 { - var nodes[]uint64 + var nodes []uint64 if node, ok := s.generating[uid]; ok { nodes = append(nodes, node) } @@ -291,7 +428,7 @@ func (s *GCSObjServer) RegisterLiveNodes(ctx context.Context, req *pb.LiveNodesR // then reschedule. but if there is at least one live node that is either // gonna generate it or storing it then don't - var regenerateList[]uint64 + var regenerateList []uint64 allUids := make(map[uint64]bool) for uid, _ := range s.generating { diff --git a/go/cmd/gcsobjtable/main_test.go b/go/cmd/gcsobjtable/main_test.go index b9b62a4..8bcc17a 100644 --- a/go/cmd/gcsobjtable/main_test.go +++ b/go/cmd/gcsobjtable/main_test.go @@ -3,15 +3,21 @@ package main import ( "bytes" "context" + "database/sql" + "errors" "fmt" + "io/ioutil" "log" "math/rand" "net" + "os" "strconv" + "strings" "sync" "testing" "time" + "github.com/DATA-DOG/go-sqlmock" "github.com/rodrigo-castellon/babyray/config" pb "github.com/rodrigo-castellon/babyray/pkg" "google.golang.org/grpc" @@ -27,7 +33,7 @@ func init() { cfg = config.GetConfig() // Load configuration lis = bufconn.Listen(bufSize) s := grpc.NewServer() - pb.RegisterGCSObjServer(s, NewGCSObjServer()) + pb.RegisterGCSObjServer(s, NewGCSObjServer(-1)) go func() { if err := s.Serve(lis); err != nil { log.Fatalf("Server exited with error: %v", err) @@ -39,20 +45,61 @@ func bufDialer(context.Context, string) (net.Conn, error) { return lis.Dial() } +func TestWriteObjectLocationsToAOF(t *testing.T) { + // Create a temporary file + tmpfile, err := ioutil.TempFile("", "test_object_locations_*.txt") + if err != nil { + t.Fatalf("Unable to create temp file: %v", err) + } + defer os.Remove(tmpfile.Name()) // Clean up the temp file after the test + + // Close the file so WriteObjectLocations can open it + tmpfile.Close() + + // Sample map for testing + objectLocations := map[uint64][]uint64{ + 1: {10, 20, 30}, + 2: {40, 50, 60}, + 3: {70, 80, 90}, + } + + // Call the function to write the map to the file + err = WriteObjectLocationsToAOF(tmpfile.Name(), objectLocations) + if err != nil { + t.Fatalf("WriteObjectLocations failed: %v", err) + } + + // Read the file content + content, err := ioutil.ReadFile(tmpfile.Name()) + if err != nil { + t.Fatalf("Unable to read temp file: %v", err) + } + + // Expected content + expectedContent := `1: [10,20,30] +2: [40,50,60] +3: [70,80,90] +` + + // Check if the content matches the expected content + if strings.TrimSpace(string(content)) != strings.TrimSpace(expectedContent) { + t.Errorf("Content mismatch\nExpected:\n%s\nGot:\n%s", expectedContent, string(content)) + } +} + func TestGetNodeId(t *testing.T) { // Seed the random number generator for reproducibility in tests rand.Seed(1) // Initialize the GCSObjServer - server := &GCSObjServer{ - objectLocations: make(map[uint64][]uint64), - waitlist: make(map[uint64][]string), - mu: sync.Mutex{}, - } + server := NewGCSObjServer(-1) // Test case: UID exists with multiple NodeIds server.objectLocations[1] = []uint64{100, 101, 102} - nodeId, exists := server.getNodeId(1) + nodeId, exists, err := server.getNodeId(1) + if err != nil { + t.Fatalf("failed test with unexpected error: %v", err) + } if !exists { t.Errorf("Expected UID 1 to exist") } @@ -62,7 +109,10 @@ func TestGetNodeId(t *testing.T) { // Test case: UID exists with a single NodeId server.objectLocations[2] = []uint64{200} - nodeId, exists = server.getNodeId(2) + nodeId, exists, err = server.getNodeId(2) + if err != nil { + t.Fatalf("failed test with unexpected error: %v", err) + } if !exists { t.Errorf("Expected UID 2 to exist") } @@ -71,7 +121,10 @@ func TestGetNodeId(t *testing.T) { } // Test case: UID does not exist - nodeId, exists = server.getNodeId(3) + nodeId, exists, err = server.getNodeId(3) + if err != nil { + t.Fatalf("failed test with unexpected error: %v", err) + } if exists { t.Errorf("Expected UID 3 to not exist") } @@ -81,7 +134,10 @@ func TestGetNodeId(t *testing.T) { // Test case: UID exists but with an empty NodeId list server.objectLocations[4] = []uint64{} - nodeId, exists = server.getNodeId(4) + nodeId, exists, err = server.getNodeId(4) + if err != nil { + t.Fatalf("failed test with unexpected error: %v", err) + } if exists { t.Errorf("Expected UID 4 to not exist due to empty NodeId list") } @@ -93,7 +149,7 @@ func TestGetNodeId(t *testing.T) { // Test the getNodeId function func TestGetNodeId_2(t *testing.T) { // Initialize the server - server := NewGCSObjServer() + server := NewGCSObjServer(-1) // Seed the random number generator to produce consistent results rand.Seed(time.Now().UnixNano()) @@ -117,7 +173,10 @@ func TestGetNodeId_2(t *testing.T) { } // Call getNodeId - nodeId, exists := server.getNodeId(tc.uid) + nodeId, exists, err := server.getNodeId(tc.uid) + if err != nil { + t.Fatalf("failed test with unexpected error: %v", err) + } // Check if the result is nil or not as expected if tc.expectNil && nodeId != nil { @@ -140,6 +199,187 @@ func TestGetNodeId_2(t *testing.T) { } } +func TestGetNodeId_CacheMissWithError(t *testing.T) { + // Create sqlmock database connection and mock object + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("An error '%s' was not expected when opening a stub database connection", err) + } + defer db.Close() + + // Define the mock expectation: when querying the database, it should return an error + mock.ExpectQuery("SELECT node_id FROM object_locations WHERE object_uid = ?"). + WithArgs(3). + WillReturnError(errors.New("database error")) + + // Initialize the server with the mocked database + server := &GCSObjServer{ + objectLocations: make(map[uint64][]uint64), + waitlist: make(map[uint64][]string), + mu: sync.Mutex{}, + database: db, + } + + // Test case: Cache miss with error + _, exists, err := server.getNodeId(3) + if err == nil { + t.Errorf("Expected error, but got nil") + } + if exists { + t.Errorf("Expected UID 3 to not exist") + } + + // Ensure all expectations were met + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("There were unfulfilled expectations: %s", err) + } +} + +func TestGetNodeId_CacheMissWithEmptyResult(t *testing.T) { + // Create sqlmock database connection and mock object + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("An error '%s' was not expected when opening a stub database connection", err) + } + defer db.Close() + + // Define the mock expectation: when querying the database, it should return no rows + mock.ExpectQuery("SELECT node_id FROM object_locations WHERE object_uid = ?"). + WithArgs(2). + WillReturnRows(sqlmock.NewRows([]string{"node_id"})) + + // Initialize the server with the mocked database + server := &GCSObjServer{ + objectLocations: make(map[uint64][]uint64), + waitlist: make(map[uint64][]string), + mu: sync.Mutex{}, + database: db, + } + + // Test case: Cache miss with empty result + nodeId, exists, err := server.getNodeId(2) + if err != nil { + t.Errorf("Expected no error, but got %v", err) + } + if exists { + t.Errorf("Expected UID 2 to not exist") + } + if nodeId != nil { + t.Errorf("Expected nodeId to be nil, got %v", nodeId) + } + + // Ensure all expectations were met + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("There were unfulfilled expectations: %s", err) + } +} + +func TestGetNodeId_CacheMissWithResult(t *testing.T) { + // Create sqlmock database connection and mock object + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("An error '%s' was not expected when opening a stub database connection", err) + } + defer db.Close() + + // Define the mock expectation: when querying the database, it should return specific rows + rows := sqlmock.NewRows([]string{"node_id"}).AddRow(100).AddRow(101).AddRow(102) + mock.ExpectQuery("SELECT node_id FROM object_locations WHERE object_uid = ?"). + WithArgs(1). + WillReturnRows(rows) + + // Initialize the server with the mocked database + server := &GCSObjServer{ + objectLocations: make(map[uint64][]uint64), + waitlist: make(map[uint64][]string), + mu: sync.Mutex{}, + database: db, + } + + // Test case: Cache miss with result + nodeId, exists, err := server.getNodeId(1) + if err != nil { + t.Errorf("Expected no error, but got %v", err) + } + if !exists { + t.Errorf("Expected UID 1 to exist") + } + if nodeId == nil || (*nodeId != 100 && *nodeId != 101 && *nodeId != 102) { + t.Errorf("Expected nodeId to be one of [100, 101, 102], got %v", nodeId) + } + + // Ensure all expectations were met + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("There were unfulfilled expectations: %s", err) + } +} + +func TestGetNodeId_CacheMissWithResult_RealDB(t *testing.T) { + // Set up in-memory SQLite database for testing + database, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("Failed to open SQLite database: %v", err) + } + defer database.Close() + + // Create the table schema + createObjectLocationsTable(database) + + // Insert test data + _, err = database.Exec(`INSERT INTO object_locations (object_uid, node_id) VALUES (1, 100), (1, 101), (1, 102)`) + if err != nil { + t.Fatalf("Failed to insert test data: %v", err) + } + + // Initialize the server + server := NewGCSObjServer(-1) + server.database = database + + // Test case: Cache miss with result + nodeId, exists, err := server.getNodeId(1) + if err != nil { + t.Errorf("Expected no error, but got %v", err) + } + if !exists { + t.Errorf("Expected UID 1 to exist") + } + if nodeId == nil || (*nodeId != 100 && *nodeId != 101 && *nodeId != 102) { + t.Errorf("Expected nodeId to be one of [100, 101, 102], got %v", nodeId) + } +} + +func TestGetNodeId_CacheMissWithEmptyResult_RealDB(t *testing.T) { + // Set up in-memory SQLite database for testing + database, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("Failed to open SQLite database: %v", err) + } + defer database.Close() + + // Create the table schema + createObjectLocationsTable(database) + + // Initialize the server + server := &GCSObjServer{ + objectLocations: make(map[uint64][]uint64), + waitlist: make(map[uint64][]string), + mu: sync.Mutex{}, + database: database, + } + + // Test case: Cache miss with empty result + nodeId, exists, err := server.getNodeId(2) + if err != nil { + t.Errorf("Expected no error, but got %v", err) + } + if exists { + t.Errorf("Expected UID 2 to not exist") + } + if nodeId != nil { + t.Errorf("Expected nodeId to be nil, got %v", nodeId) + } +} + // Mock implementation of the LocalObjStoreServer type mockLocalObjStoreServer struct { pb.UnimplementedLocalObjStoreServer @@ -624,3 +864,74 @@ func contains(slice []uint64, value uint64) bool { } return false } + +func TestFlushToDisk(t *testing.T) { + // Set up in-memory SQLite database for testing + database, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("Failed to open SQLite database: %v", err) + } + defer database.Close() + + // Create the table schema + createObjectLocationsTable(database) + + // Create the server object with a short flush interval for testing + server := &GCSObjServer{ + objectLocations: make(map[uint64][]uint64), + waitlist: make(map[uint64][]string), + mu: sync.Mutex{}, + database: database, + ticker: nil, + } + + // Populate the objectLocations map + server.objectLocations[1] = []uint64{100, 101, 102} + server.objectLocations[2] = []uint64{200, 201} + + // Call the flushToDisk method + err = server.flushToDisk() + if err != nil { + t.Fatalf("flushToDisk failed: %v", err) + } + + // Verify the data has been written to the database + rows, err := database.Query("SELECT object_uid, node_id FROM object_locations") + if err != nil { + t.Fatalf("Failed to query database: %v", err) + } + defer rows.Close() + + // Read the results + results := make(map[uint64][]uint64) + for rows.Next() { + var objectUID uint64 + var nodeID uint64 + if err := rows.Scan(&objectUID, &nodeID); err != nil { + t.Fatalf("Failed to scan row: %v", err) + } + results[objectUID] = append(results[objectUID], nodeID) + } + + // Expected results + expectedResults := map[uint64][]uint64{ + 1: {100, 101, 102}, + 2: {200, 201}, + } + + // Compare the results + if len(results) != len(expectedResults) { + t.Fatalf("Expected %d results, got %d", len(expectedResults), len(results)) + } + + for objectUID, nodeIDs := range expectedResults { + if len(results[objectUID]) != len(nodeIDs) { + t.Fatalf("Expected %d nodeIDs for object %d, got %d", len(nodeIDs), objectUID, len(results[objectUID])) + } + for i, nodeID := range nodeIDs { + if results[objectUID][i] != nodeID { + t.Fatalf("Expected nodeID %d for object %d, got %d", nodeID, objectUID, results[objectUID][i]) + } + } + } +} diff --git a/go/cmd/localscheduler/main.go b/go/cmd/localscheduler/main.go index 928a063..5c12f03 100644 --- a/go/cmd/localscheduler/main.go +++ b/go/cmd/localscheduler/main.go @@ -4,23 +4,22 @@ import ( "context" "fmt" "log" + "math" "math/rand" "net" + "os" "strconv" - "os" "time" "github.com/rodrigo-castellon/babyray/config" "github.com/rodrigo-castellon/babyray/customlog" - "github.com/rodrigo-castellon/babyray/util" pb "github.com/rodrigo-castellon/babyray/pkg" + "github.com/rodrigo-castellon/babyray/util" "google.golang.org/grpc" ) - - - var cfg *config.Config + const HEARTBEAT_WAIT = 100 * time.Millisecond const MAX_TASKS uint32 = 10 @@ -61,8 +60,7 @@ func main() { gcsConn, _ := grpc.Dial(gcsObjAddress, grpc.WithInsecure()) gcsObjClient := pb.NewGCSObjClient(gcsConn) - - server := &server{globalSchedulerClient: globalSchedulerClient, workerClient: workerClient, globalCtx: context.Background(), localNodeID: uint64(nodeId), gcsClient: gcsObjClient, alive: true } + server := &server{globalSchedulerClient: globalSchedulerClient, workerClient: workerClient, globalCtx: context.Background(), localNodeID: uint64(nodeId), gcsClient: gcsObjClient, alive: true} pb.RegisterLocalSchedulerServer(s, server) ctx := context.Background() go server.SendHeartbeats(ctx, globalSchedulerClient, uint64(nodeId)) @@ -72,45 +70,50 @@ func main() { log.Fatalf("failed to serve: %v", err) } - } // server is used to implement your gRPC service. type server struct { pb.UnimplementedLocalSchedulerServer globalSchedulerClient pb.GlobalSchedulerClient - workerClient pb.WorkerClient - gcsClient pb.GCSObjClient - globalCtx context.Context - localNodeID uint64 - alive bool + workerClient pb.WorkerClient + gcsClient pb.GCSObjClient + globalCtx context.Context + localNodeID uint64 + alive bool } -// Implement your service methods here. +// TruncateHighBit truncates the most significant bit of a uint64 value +func TruncateHighBit(u64 uint64) uint64 { + mask := uint64(math.MaxUint64 >> 1) // or mask := uint64(0x7FFFFFFFFFFFFFFF) + return u64 & mask +} +// Implement your service methods here. func (s *server) Schedule(ctx context.Context, req *pb.ScheduleRequest) (*pb.ScheduleResponse, error) { // var worker_id int // worker_id, _ = strconv.Atoi(os.Getenv("NODE_ID")) - uid := rand.Uint64() + uid := rand.Uint64() + uid = TruncateHighBit(uid) // NOTE: THIS IS A WORKAROUND FOR THE FACT THAT SQLite3 IN THE GCS OBJECT TABLE's STORAGE CANNOT TAKE HIGH BIT OF UINT64 - _ , err := s.gcsClient.RegisterLineage(ctx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids}) + _, err := s.gcsClient.RegisterLineage(ctx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids}) if err != nil { // LocalLog("cant hit gcs: %v", err) } // custom behavior if the client itself specifies where we should send this // computation - if (req.NodeId != 0) { + if req.NodeId != 0 { // LocalLog("Doing something special with req.NodeId = %v", req.NodeId) go func() { - _, err := s.globalSchedulerClient.Schedule(s.globalCtx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids, NewObject: true, NodeId: req.NodeId}) - if err != nil { - // LocalLog("cannot contact global scheduler") - } else { + _, err := s.globalSchedulerClient.Schedule(s.globalCtx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids, NewObject: true, NodeId: req.NodeId}) + if err != nil { + // LocalLog("cannot contact global scheduler") + } else { // LocalLog("Just ran it on global!") } - }() + }() return &pb.ScheduleResponse{Uid: uid}, nil } @@ -122,25 +125,25 @@ func (s *server) Schedule(ctx context.Context, req *pb.ScheduleRequest) (*pb.Sch // LocalLog("Just running locally") go func() { s.gcsClient.RegisterGenerating(ctx, &pb.GeneratingRequest{Uid: uid, NodeId: s.localNodeID}) - _, err := s.workerClient.Run(s.globalCtx, &pb.RunRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs}) - if err != nil { - // LocalLog("cannot contact worker %d: %v", worker_id, err) - } else { - // LocalLog("Just ran it!") - } - }() - + _, err := s.workerClient.Run(s.globalCtx, &pb.RunRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs}) + if err != nil { + // LocalLog("cannot contact worker %d: %v", worker_id, err) + } else { + // LocalLog("Just ran it!") + } + }() + } else { // LocalLog("contacting global scheduler") go func() { // LocalLog("THE REQ UIDS AT LOCAL SCHEDULER ARE %v", req.Uids) - _, err := s.globalSchedulerClient.Schedule(s.globalCtx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids, NewObject: true, LocalityFlag: req.LocalityFlag}) - if err != nil { - // LocalLog("cannot contact global scheduler") - } else { + _, err := s.globalSchedulerClient.Schedule(s.globalCtx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids, NewObject: true, LocalityFlag: req.LocalityFlag}) + if err != nil { + // LocalLog("cannot contact global scheduler") + } else { // LocalLog("Just ran it on global!") } - }() + }() } return &pb.ScheduleResponse{Uid: uid}, nil @@ -158,84 +161,82 @@ func (s *server) ReviveServer(ctx context.Context, req *pb.StatusResponse) (*pb. return &pb.StatusResponse{Success: true}, nil } -func (s *server) SendHeartbeats(ctx context.Context, globalSchedulerClient pb.GlobalSchedulerClient, nodeId uint64 ) { +func (s *server) SendHeartbeats(ctx context.Context, globalSchedulerClient pb.GlobalSchedulerClient, nodeId uint64) { workerAddress := fmt.Sprintf("localhost:%d", cfg.Ports.LocalWorkerStart) workerConn, err := grpc.Dial(workerAddress, util.GetDialOptions()...) if err != nil { log.Fatalf("failed to connect to %s: %v", workerAddress, err) } - defer workerConn.Close() + defer workerConn.Close() lobsAddress := fmt.Sprintf("localhost:%d", cfg.Ports.LocalObjectStore) lobsConn, err := grpc.Dial(lobsAddress, util.GetDialOptions()...) if err != nil { log.Fatalf("failed to connect to %s: %v", lobsAddress, err) - - } - defer lobsConn.Close() + } + defer lobsConn.Close() workerClient := pb.NewWorkerClient(workerConn) lobsClient := pb.NewLocalObjStoreClient(lobsConn) for { if s.alive { - backoff := 1 - var status *pb.WorkerStatusResponse - - for { - status, err = workerClient.WorkerStatus(ctx, &pb.StatusResponse{}) - if err != nil { - // LocalLog("got error from WorkerStatus(): %v", err) - // LocalLog("retrying in %v seconds", backoff) - time.Sleep(time.Duration(backoff) * time.Second) - backoff *= 2 - if backoff > 32 { - backoff = 32 // Cap the backoff to 32 seconds + backoff := 1 + var status *pb.WorkerStatusResponse + + for { + status, err = workerClient.WorkerStatus(ctx, &pb.StatusResponse{}) + if err != nil { + // LocalLog("got error from WorkerStatus(): %v", err) + // LocalLog("retrying in %v seconds", backoff) + time.Sleep(time.Duration(backoff) * time.Second) + backoff *= 2 + if backoff > 32 { + backoff = 32 // Cap the backoff to 32 seconds + } + continue } - continue + break } - break - } - numRunningTasks := status.NumRunningTasks - numQueuedTasks := status.NumQueuedTasks - avgRunningTime := status.AverageRunningTime - var avgBandwidth *pb.BandwidthResponse - backoff = 1 - for { - avgBandwidth, err = lobsClient.AvgBandwidth(ctx, &pb.StatusResponse{}) - if err != nil { - // LocalLog("got error from AvgBand(): %v", err) - // LocalLog("retrying in %v seconds", backoff) - time.Sleep(time.Duration(backoff) * time.Second) - backoff *= 2 - if backoff > 32 { - backoff = 32 // Cap the backoff to 32 seconds + numRunningTasks := status.NumRunningTasks + numQueuedTasks := status.NumQueuedTasks + avgRunningTime := status.AverageRunningTime + var avgBandwidth *pb.BandwidthResponse + backoff = 1 + for { + avgBandwidth, err = lobsClient.AvgBandwidth(ctx, &pb.StatusResponse{}) + if err != nil { + // LocalLog("got error from AvgBand(): %v", err) + // LocalLog("retrying in %v seconds", backoff) + time.Sleep(time.Duration(backoff) * time.Second) + backoff *= 2 + if backoff > 32 { + backoff = 32 // Cap the backoff to 32 seconds + } + continue } - continue + break } - break - } - heartbeatRequest := &pb.HeartbeatRequest{ - RunningTasks: numRunningTasks, - QueuedTasks: numQueuedTasks, - AvgRunningTime: avgRunningTime, - AvgBandwidth: avgBandwidth.AvgBandwidth, - NodeId: nodeId } - - // LocalLog("HeartbeatRequest: RunningTasks=%d, QueuedTasks=%d, AvgRunningTime=%.2f, AvgBandwidth=%.2f, NodeId=%d", - // heartbeatRequest.RunningTasks, - // heartbeatRequest.QueuedTasks, - // heartbeatRequest.AvgRunningTime, - // heartbeatRequest.AvgBandwidth, - // heartbeatRequest.NodeId) - - // LocalLog("SENDING GLOBAL SCHEDULER A HEARTBEAT!!!") - globalSchedulerClient.Heartbeat(ctx, heartbeatRequest) + heartbeatRequest := &pb.HeartbeatRequest{ + RunningTasks: numRunningTasks, + QueuedTasks: numQueuedTasks, + AvgRunningTime: avgRunningTime, + AvgBandwidth: avgBandwidth.AvgBandwidth, + NodeId: nodeId} + + // LocalLog("HeartbeatRequest: RunningTasks=%d, QueuedTasks=%d, AvgRunningTime=%.2f, AvgBandwidth=%.2f, NodeId=%d", + // heartbeatRequest.RunningTasks, + // heartbeatRequest.QueuedTasks, + // heartbeatRequest.AvgRunningTime, + // heartbeatRequest.AvgBandwidth, + // heartbeatRequest.NodeId) + + // LocalLog("SENDING GLOBAL SCHEDULER A HEARTBEAT!!!") + globalSchedulerClient.Heartbeat(ctx, heartbeatRequest) } - time.Sleep(HEARTBEAT_WAIT) + time.Sleep(HEARTBEAT_WAIT) } } - diff --git a/go/cmd/localscheduler/main_test.go b/go/cmd/localscheduler/main_test.go new file mode 100644 index 0000000..5c51117 --- /dev/null +++ b/go/cmd/localscheduler/main_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "testing" +) + +// Unit test for TruncateHighBit +func TestTruncateHighBit(t *testing.T) { + tests := []struct { + input uint64 + expected uint64 + }{ + {0xFFFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF}, + {0x8000000000000000, 0x0000000000000000}, + {0x7FFFFFFFFFFFFFFF, 0x7FFFFFFFFFFFFFFF}, + {0x0000000000000001, 0x0000000000000001}, + } + + for _, test := range tests { + result := TruncateHighBit(test.input) + if result != test.expected { + t.Errorf("TruncateHighBit(%d) = %d; expected %d", test.input, result, test.expected) + } + } +} diff --git a/go/config/config.go b/go/config/config.go index b0a8432..d0fec60 100644 --- a/go/config/config.go +++ b/go/config/config.go @@ -1,62 +1,66 @@ package config import ( - "io/ioutil" - "gopkg.in/yaml.v3" - "log" - "os" - "path/filepath" + "io/ioutil" + "log" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" ) var cfg *Config type Config struct { - NumWorkerNodes int `yaml:"num_worker_nodes"` - ClusterSize int `yaml:"cluster_size"` - SimulatedDelay float32 `yaml:"simulated_delay"` - NodeIDs struct { - GCS int `yaml:"gcs"` - GlobalScheduler int `yaml:"global_scheduler"` - Ourself int `yaml:"ourself"` - } `yaml:"node_ids"` - Ports struct { - LocalObjectStore int `yaml:"local_object_store"` - LocalScheduler int `yaml:"local_scheduler"` - LocalWorkerStart int `yaml:"local_worker_start"` - GCSFunctionTable int `yaml:"gcs_function_table"` - GCSObjectTable int `yaml:"gcs_object_table"` - GlobalScheduler int `yaml:"global_scheduler"` - } `yaml:"ports"` - DNS struct { - NodePrefix string `yaml:"node_prefix"` - } `yaml:"dns"` + NumWorkerNodes int `yaml:"num_worker_nodes"` + ClusterSize int `yaml:"cluster_size"` + SimulatedDelay float32 `yaml:"simulated_delay"` + NodeIDs struct { + GCS int `yaml:"gcs"` + GlobalScheduler int `yaml:"global_scheduler"` + Ourself int `yaml:"ourself"` + } `yaml:"node_ids"` + Ports struct { + LocalObjectStore int `yaml:"local_object_store"` + LocalScheduler int `yaml:"local_scheduler"` + LocalWorkerStart int `yaml:"local_worker_start"` + GCSFunctionTable int `yaml:"gcs_function_table"` + GCSObjectTable int `yaml:"gcs_object_table"` + GlobalScheduler int `yaml:"global_scheduler"` + } `yaml:"ports"` + DNS struct { + NodePrefix string `yaml:"node_prefix"` + } `yaml:"dns"` + GCS struct { + FlushIntervalSec int `yaml:"flush_interval_sec"` + } } func LoadConfig() *Config { - var config Config - - // Construct the path to the configuration file. PROJECT_ROOT should somehow - // be set prior to any Go code execution (i.e., in GitHub Actions workflow - // before unit tests are run or in Dockerfile) - rootPath := os.Getenv("PROJECT_ROOT") - - configFile := filepath.Join(rootPath, "config", "app_config.yaml") - - yamlFile, err := ioutil.ReadFile(configFile) - if err != nil { - log.Fatalf("error: %v", err) - } - err = yaml.Unmarshal(yamlFile, &config) - if err != nil { - log.Fatalf("Unmarshal: %v", err) - } - return &config + var config Config + + // Construct the path to the configuration file. PROJECT_ROOT should somehow + // be set prior to any Go code execution (i.e., in GitHub Actions workflow + // before unit tests are run or in Dockerfile) + rootPath := os.Getenv("PROJECT_ROOT") + + configFile := filepath.Join(rootPath, "config", "app_config.yaml") + + yamlFile, err := ioutil.ReadFile(configFile) + if err != nil { + log.Fatalf("error: %v", err) + } + err = yaml.Unmarshal(yamlFile, &config) + if err != nil { + log.Fatalf("Unmarshal: %v", err) + } + return &config } func init() { - cfg = LoadConfig() + cfg = LoadConfig() } func GetConfig() *Config { - return cfg + return cfg } diff --git a/go/go.mod b/go/go.mod index 14c04f4..1e37b15 100644 --- a/go/go.mod +++ b/go/go.mod @@ -9,6 +9,10 @@ require ( ) require ( + github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go/go.sum b/go/go.sum index fce0399..b695b11 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,3 +1,14 @@ +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= diff --git a/go/pkg/rayclient.pb.go b/go/pkg/rayclient.pb.go index 27daebe..ad29be0 100644 --- a/go/pkg/rayclient.pb.go +++ b/go/pkg/rayclient.pb.go @@ -441,6 +441,13 @@ type GetRequest struct { Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` Testing bool `protobuf:"varint,2,opt,name=testing,proto3" json:"testing,omitempty"` + // normally is true, but is false whenever we want to just block + // for a task to complete + Copy bool `protobuf:"varint,3,opt,name=copy,proto3" json:"copy,omitempty"` + // normally is true, but is false whenever we want to copy the + // object over but not cache it (so we can keep measuring the copy + // speed) + Cache bool `protobuf:"varint,4,opt,name=cache,proto3" json:"cache,omitempty"` } func (x *GetRequest) Reset() { @@ -489,6 +496,20 @@ func (x *GetRequest) GetTesting() bool { return false } +func (x *GetRequest) GetCopy() bool { + if x != nil { + return x.Copy + } + return false +} + +func (x *GetRequest) GetCache() bool { + if x != nil { + return x.Cache + } + return false +} + type GetResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -497,6 +518,7 @@ type GetResponse struct { Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` ObjectBytes []byte `protobuf:"bytes,2,opt,name=objectBytes,proto3" json:"objectBytes,omitempty"` Local bool `protobuf:"varint,3,opt,name=local,proto3" json:"local,omitempty"` + Size uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` } func (x *GetResponse) Reset() { @@ -552,6 +574,13 @@ func (x *GetResponse) GetLocal() bool { return false } +func (x *GetResponse) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + type BandwidthResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/scripts/fig10b.py b/scripts/fig10b.py new file mode 100644 index 0000000..43a5100 --- /dev/null +++ b/scripts/fig10b.py @@ -0,0 +1,85 @@ +import sys +from babyray import ( + init, + remote, + get, + Future, +) +import time +from utils import log + +log("FIG10b: HELLO WORLD I AM ANGRY!") + +# Initialize babyray +init() + +# Define a dummy remote function +@remote +def f(): + return 0 + +# Function to measure memory usage using /proc/meminfo +def get_memory_usage(): + with open('/proc/meminfo', 'r') as meminfo: + lines = meminfo.readlines() + meminfo_dict = {} + for line in lines: + parts = line.split(':') + meminfo_dict[parts[0].strip()] = int(parts[1].strip().split()[0]) # Value is in KB + + total_memory = meminfo_dict['MemTotal'] / 1024 # Convert KB to MB + free_memory = meminfo_dict['MemFree'] / 1024 + buffers = meminfo_dict['Buffers'] / 1024 + cached = meminfo_dict['Cached'] / 1024 + + used_memory = total_memory - (free_memory + buffers + cached) + return used_memory + +# Number of tasks to submit +num_tasks = 50_000_000 + +# Submit tasks without GCS flushing and measure memory usage +log("FIG10b: Starting tasks submission...") +start_time = time.time() +elapsed_time = 0 +total_duration = 2 * 60 * 60 # 2 hours in seconds + +# Memory logging intervals +initial_interval = 1 # seconds +second_interval = 30 # seconds after initial period +initial_period = 300 # seconds +max_interval = 600 # Maximum interval of 10 minutes + +current_interval = initial_interval +next_log_time = start_time + current_interval + +i = 0 + +old_way = True +if old_way: + while elapsed_time < total_duration and i < num_tasks: + # Submit a single task + f.remote() + i += 1 + + # Measure memory usage + current_time = time.time() + elapsed_time = current_time - start_time + + usage = 0 + if current_time >= next_log_time: + usage = get_memory_usage() + log(f"FIG10b: Memory usage without flush after {i} tasks: {usage} MB") + + # Schedule next log time + if elapsed_time < initial_period: + current_interval = initial_interval + else: + current_interval = min(second_interval, max_interval) + next_log_time = current_time + current_interval + + if usage > 2000: # Stop if memory usage exceeds 2 GB + break +else: + while True: + f.remote() \ No newline at end of file diff --git a/scripts/run_fig10b.sh b/scripts/run_fig10b.sh new file mode 100755 index 0000000..4b9ca40 --- /dev/null +++ b/scripts/run_fig10b.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# scripts/run_fig10b.sh + +# REALLY ONLY USING THIS; EVERYTHING ELSE IS JUNK +docker-compose -f docker-compose.yml -f docker-compose.fig10b.yml up + + +# ================== + +# docker-compose logs worker1 > out.txt + + + + +# Wait for the services to be up and running +# sleep + +# Get current timestamp +# timestamp=$(date +"%Y%m%d_%H%M%S") + +# # Define the log filename with the timestamp suffix +# log_filename="output_${timestamp}.log" + +# Capture and filter the logs, then save to the timestamped file +# docker-compose logs worker1 | grep FIG10b: | tee "$log_filename" +# docker-compose logs worker1 | tee "$log_filename" +# docker-compose logs worker1 | grep FIG10b: +#echo yolo +# docker-compose logs worker1 > out.txt + +# # Check if "1 passed" is in the logs +# # if grep -q "1 passed" "$log_filename"; then +# if grep -q "1 passed" "output_hello.log"; then +# echo "All tests passed!" +# docker-compose down +# exit 0 +# else +# echo "Some tests failed." +# docker-compose down +# exit 1 +# fi