diff --git a/.DS_Store b/.DS_Store index db68ea8..6de347d 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index 61b83b7..e7bbb62 100644 --- a/.gitignore +++ b/.gitignore @@ -199,3 +199,4 @@ python/babyray/rayclient_pb2_grpc.py-e go/cmd/localobjstore/localobjstore go/cmd/localscheduler/localscheduler gcsobjtable.db +.DS_Store diff --git a/Makefile b/Makefile index 901f6da..3a40457 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: all go py clean build servers docker -PYTHON ?= python +PYTHON ?= python3 all: go py build docker diff --git a/analysis/README.md b/analysis/README.md new file mode 100644 index 0000000..116e292 --- /dev/null +++ b/analysis/README.md @@ -0,0 +1,9 @@ +# analysis +`output_10b_run1.txt`: Results of 10b run1, aka no caching, 2GB memory limit + +* After runs 1-5, realized we've been measuring memory on worker and not the GCS. need to change that!! + +* Run 6 is flush to disk every 1s using AOF and garbage collection +* Run 7 is the new baseline, don't flush to disk. + +run 6 vs run 8 https://chatgpt.com/c/76bd6aef-f711-4067-b626-31446db56773 \ No newline at end of file diff --git a/analysis/fig10b_graphs/attempt2_blurry.png b/analysis/fig10b_graphs/attempt2_blurry.png new file mode 100644 index 0000000..38dd305 Binary files /dev/null and b/analysis/fig10b_graphs/attempt2_blurry.png differ diff --git a/analysis/fig10b_graphs/attempt2_sharp.png b/analysis/fig10b_graphs/attempt2_sharp.png new file mode 100644 index 0000000..2b89c75 Binary files /dev/null and b/analysis/fig10b_graphs/attempt2_sharp.png differ diff --git a/analysis/fig10b_graphs/attempt_Figure_1.png b/analysis/fig10b_graphs/attempt_Figure_1.png new file mode 100644 index 0000000..18dd30f Binary files /dev/null and b/analysis/fig10b_graphs/attempt_Figure_1.png differ diff --git a/analysis/fig10b_graphs/output a(1).png b/analysis/fig10b_graphs/output a(1).png new file mode 100644 index 0000000..a7309eb Binary files /dev/null and b/analysis/fig10b_graphs/output a(1).png differ diff --git a/analysis/fig10b_graphs/outputa(2).png b/analysis/fig10b_graphs/outputa(2).png new file mode 100644 index 0000000..a061c27 Binary files /dev/null and b/analysis/fig10b_graphs/outputa(2).png differ diff --git a/analysis/fig10b_graphs/outputa.png b/analysis/fig10b_graphs/outputa.png new file mode 100644 index 0000000..6e207e6 Binary files /dev/null and b/analysis/fig10b_graphs/outputa.png differ diff --git a/analysis/fig10b_graphs/run_6_vs_run8_1.png b/analysis/fig10b_graphs/run_6_vs_run8_1.png new file mode 100644 index 0000000..32dfb2c Binary files /dev/null and b/analysis/fig10b_graphs/run_6_vs_run8_1.png differ diff --git a/analysis/fig10b_graphs/run_6_vs_run8_2.png b/analysis/fig10b_graphs/run_6_vs_run8_2.png new file mode 100644 index 0000000..31cd3f2 Binary files /dev/null and b/analysis/fig10b_graphs/run_6_vs_run8_2.png differ diff --git a/analysis/fig10b_graphs/run_6_vs_run8_3.png b/analysis/fig10b_graphs/run_6_vs_run8_3.png new file mode 100644 index 0000000..b2b7fbd Binary files /dev/null and b/analysis/fig10b_graphs/run_6_vs_run8_3.png differ diff --git a/analysis/output_10b_run1_cleaned.csv b/analysis/output_10b_run1_cleaned.csv new file mode 100644 index 0000000..eb7fb5c --- /dev/null +++ b/analysis/output_10b_run1_cleaned.csv @@ -0,0 +1,225 @@ +Timestamp,Memory Usage (MB) +2024-06-04 01:44:08.529000,970.90234375 +2024-06-04 01:44:09.530000,977.0 +2024-06-04 01:44:10.533000,986.859375 +2024-06-04 01:44:11.533000,987.625 +2024-06-04 01:44:12.538000,989.30078125 +2024-06-04 01:44:14.551000,991.5859375 +2024-06-04 01:44:15.558000,991.37109375 +2024-06-04 01:44:18.565000,999.69140625 +2024-06-04 01:44:19.571000,999.6328125 +2024-06-04 01:44:21.582000,1001.9453125 +2024-06-04 01:44:23.603000,1002.921875 +2024-06-04 01:44:26.620000,1003.6015625 +2024-06-04 01:44:27.620000,1005.21875 +2024-06-04 01:44:28.620000,1004.91015625 +2024-06-04 01:44:32.635000,1010.0390625 +2024-06-04 01:44:34.655000,1013.94140625 +2024-06-04 01:44:38.734000,1018.734375 +2024-06-04 01:44:39.971000,1018.75390625 +2024-06-04 01:44:40.893000,1018.98828125 +2024-06-04 01:44:41.889000,1019.6015625 +2024-06-04 01:44:42.891000,1023.62109375 +2024-06-04 01:44:43.896000,1023.48046875 +2024-06-04 01:44:44.901000,1023.890625 +2024-06-04 01:44:45.907000,1024.66015625 +2024-06-04 01:44:46.908000,1025.2265625 +2024-06-04 01:44:47.913000,1024.94140625 +2024-06-04 01:44:49.917000,1026.1484375 +2024-06-04 01:44:50.929000,1025.8984375 +2024-06-04 01:44:53.946000,1022.25 +2024-06-04 01:44:54.955000,1021.984375 +2024-06-04 01:44:55.959000,1021.9375 +2024-06-04 01:44:56.961000,1022.0546875 +2024-06-04 01:44:57.967000,1022.19140625 +2024-06-04 01:45:00.974000,1021.69921875 +2024-06-04 01:45:02.982000,1021.86328125 +2024-06-04 01:45:03.990000,1021.4609375 +2024-06-04 01:45:06.003000,1022.73828125 +2024-06-04 01:45:10.023000,1026.3515625 +2024-06-04 01:45:11.028000,1025.765625 +2024-06-04 01:45:12.038000,1025.53125 +2024-06-04 01:45:14.047000,1026.4296875 +2024-06-04 01:45:15.051000,1026.5234375 +2024-06-04 01:45:17.055000,1026.36328125 +2024-06-04 01:45:18.063000,1026.046875 +2024-06-04 01:45:22.081000,1026.33203125 +2024-06-04 01:45:23.084000,1026.7421875 +2024-06-04 01:45:27.102000,1028.72265625 +2024-06-04 01:45:30.134000,1028.61328125 +2024-06-04 01:45:31.138000,1029.0859375 +2024-06-04 01:45:32.145000,1029.4609375 +2024-06-04 01:45:33.153000,1029.68359375 +2024-06-04 01:45:34.154000,1030.1015625 +2024-06-04 01:45:35.163000,1030.99609375 +2024-06-04 01:45:36.171000,1029.703125 +2024-06-04 01:45:37.176000,1029.75 +2024-06-04 01:45:39.180000,1029.24609375 +2024-06-04 01:45:42.189000,1030.2890625 +2024-06-04 01:45:45.206000,1031.359375 +2024-06-04 01:45:46.207000,1031.94921875 +2024-06-04 01:45:47.208000,1032.20703125 +2024-06-04 01:45:48.214000,1033.19921875 +2024-06-04 01:45:51.221000,1032.45703125 +2024-06-04 01:45:53.224000,1033.6171875 +2024-06-04 01:45:54.225000,1033.1796875 +2024-06-04 01:45:57.233000,1034.3515625 +2024-06-04 01:45:59.245000,1035.3203125 +2024-06-04 01:46:00.247000,1035.94140625 +2024-06-04 01:46:01.255000,1037.515625 +2024-06-04 01:46:10.290000,1048.42578125 +2024-06-04 01:46:12.298000,1049.0859375 +2024-06-04 01:46:14.308000,1045.19140625 +2024-06-04 01:46:17.324000,1049.62890625 +2024-06-04 01:46:18.332000,1050.609375 +2024-06-04 01:46:21.344000,1051.9765625 +2024-06-04 01:46:23.352000,1053.4765625 +2024-06-04 01:46:28.372000,1054.71484375 +2024-06-04 01:46:29.373000,1053.77734375 +2024-06-04 01:46:32.387000,1055.52734375 +2024-06-04 01:46:33.396000,1054.6484375 +2024-06-04 01:46:34.397000,1054.453125 +2024-06-04 01:46:37.404000,1056.26171875 +2024-06-04 01:46:38.409000,1056.53125 +2024-06-04 01:46:39.409000,1056.0234375 +2024-06-04 01:46:41.423000,1056.2890625 +2024-06-04 01:46:42.432000,1054.0234375 +2024-06-04 01:46:44.463000,1054.8125 +2024-06-04 01:46:46.468000,1057.4609375 +2024-06-04 01:46:47.471000,1057.5390625 +2024-06-04 01:46:50.486000,1058.125 +2024-06-04 01:46:52.494000,1057.8671875 +2024-06-04 01:46:54.505000,1058.33984375 +2024-06-04 01:46:55.507000,1058.59375 +2024-06-04 01:46:57.516000,1059.72265625 +2024-06-04 01:46:58.538000,1059.53515625 +2024-06-04 01:46:59.542000,1059.1328125 +2024-06-04 01:47:00.544000,1059.484375 +2024-06-04 01:47:03.555000,1059.3359375 +2024-06-04 01:47:05.560000,1059.09375 +2024-06-04 01:47:06.562000,1059.08984375 +2024-06-04 01:47:09.579000,1062.77734375 +2024-06-04 01:47:10.580000,1062.42578125 +2024-06-04 01:47:11.581000,1062.2421875 +2024-06-04 01:47:16.607000,1064.97265625 +2024-06-04 01:47:19.613000,1066.9296875 +2024-06-04 01:47:20.620000,1066.99609375 +2024-06-04 01:47:23.625000,1066.984375 +2024-06-04 01:47:25.638000,1068.578125 +2024-06-04 01:47:26.643000,1068.1796875 +2024-06-04 01:47:27.648000,1068.94921875 +2024-06-04 01:47:32.671000,1069.390625 +2024-06-04 01:47:34.678000,1069.671875 +2024-06-04 01:47:35.685000,1070.22265625 +2024-06-04 01:47:37.694000,1069.1328125 +2024-06-04 01:47:38.700000,1069.8671875 +2024-06-04 01:47:39.702000,1069.93359375 +2024-06-04 01:47:40.704000,1069.94140625 +2024-06-04 01:47:42.711000,1071.3828125 +2024-06-04 01:47:43.715000,1071.140625 +2024-06-04 01:47:44.721000,1071.86328125 +2024-06-04 01:47:47.732000,1077.11328125 +2024-06-04 01:47:49.743000,1078.8984375 +2024-06-04 01:47:51.755000,1083.46484375 +2024-06-04 01:47:52.759000,1083.05078125 +2024-06-04 01:47:53.762000,1084.11328125 +2024-06-04 01:47:54.767000,1084.171875 +2024-06-04 01:47:57.784000,1088.45703125 +2024-06-04 01:47:58.790000,1089.6484375 +2024-06-04 01:47:59.797000,1090.2265625 +2024-06-04 01:48:00.801000,1090.1796875 +2024-06-04 01:48:02.816000,1087.9765625 +2024-06-04 01:48:03.824000,1087.44921875 +2024-06-04 01:48:05.835000,1092.90625 +2024-06-04 01:48:06.837000,1091.5703125 +2024-06-04 01:48:10.853000,1094.234375 +2024-06-04 01:48:14.870000,1094.109375 +2024-06-04 01:48:18.895000,1095.3984375 +2024-06-04 01:48:19.895000,1095.86328125 +2024-06-04 01:48:21.905000,1096.32421875 +2024-06-04 01:48:22.909000,1096.24609375 +2024-06-04 01:48:23.915000,1096.22265625 +2024-06-04 01:48:24.923000,1096.70703125 +2024-06-04 01:48:25.930000,1096.33203125 +2024-06-04 01:48:26.931000,1097.38671875 +2024-06-04 01:48:28.934000,1097.703125 +2024-06-04 01:48:32.952000,1098.62890625 +2024-06-04 01:48:34.958000,1098.76953125 +2024-06-04 01:48:35.964000,1098.85546875 +2024-06-04 01:48:36.969000,1099.4921875 +2024-06-04 01:48:37.973000,1099.6171875 +2024-06-04 01:48:40.982000,1098.63671875 +2024-06-04 01:48:45.997000,1100.421875 +2024-06-04 01:48:48.006000,1101.375 +2024-06-04 01:48:52.017000,1102.9296875 +2024-06-04 01:48:54.028000,1103.86328125 +2024-06-04 01:48:55.033000,1103.33984375 +2024-06-04 01:48:58.043000,1101.92578125 +2024-06-04 01:49:01.050000,1103.71484375 +2024-06-04 01:49:03.060000,1110.6015625 +2024-06-04 01:49:04.067000,1110.89453125 +2024-06-04 01:50:38.096000,1130.8515625 +2024-06-04 01:51:08.099000,1138.8671875 +2024-06-04 01:51:38.103000,1166.4609375 +2024-06-04 01:52:08.104000,1178.38671875 +2024-06-04 01:52:38.107000,1180.4375 +2024-06-04 01:53:08.107000,1186.28125 +2024-06-04 01:54:38.123000,1221.234375 +2024-06-04 01:56:08.139000,1239.5703125 +2024-06-04 01:56:38.149000,1245.96875 +2024-06-04 01:57:08.156000,1254.7734375 +2024-06-04 02:00:08.183000,1348.3046875 +2024-06-04 02:00:38.186000,1345.56640625 +2024-06-04 02:01:08.189000,1353.29296875 +2024-06-04 02:03:38.214000,1388.85546875 +2024-06-04 02:04:08.223000,1432.5390625 +2024-06-04 02:06:08.241000,1430.30859375 +2024-06-04 02:06:38.246000,1432.46875 +2024-06-04 02:07:08.251000,1441.54296875 +2024-06-04 02:08:08.260000,1461.6875 +2024-06-04 02:09:08.269000,1540.4609375 +2024-06-04 02:10:38.280000,1518.2578125 +2024-06-04 02:11:08.283000,1508.953125 +2024-06-04 02:12:08.290000,1563.53515625 +2024-06-04 02:12:38.302000,1554.7265625 +2024-06-04 02:13:08.310000,1594.203125 +2024-06-04 02:14:38.328000,1715.3515625 +2024-06-04 02:15:08.334000,1722.95703125 +2024-06-04 02:15:38.336000,1720.2890625 +2024-06-04 02:17:08.358000,1663.44140625 +2024-06-04 02:17:38.359000,1677.34765625 +2024-06-04 02:18:08.360000,1678.1796875 +2024-06-04 02:19:08.373000,1754.8046875 +2024-06-04 02:19:38.376000,1681.375 +2024-06-04 02:20:08.378000,1684.84375 +2024-06-04 02:20:38.381000,1696.22265625 +2024-06-04 02:22:08.395000,1719.63671875 +2024-06-04 02:22:38.395000,1711.984375 +2024-06-04 02:23:08.396000,1719.796875 +2024-06-04 02:23:38.397000,1730.06640625 +2024-06-04 02:24:08.400000,1766.78125 +2024-06-04 02:24:38.405000,1775.64453125 +2024-06-04 02:25:38.411000,1788.42578125 +2024-06-04 02:26:08.413000,1801.94140625 +2024-06-04 02:26:38.417000,1809.4609375 +2024-06-04 02:27:08.419000,1816.6640625 +2024-06-04 02:28:08.422000,1795.0859375 +2024-06-04 02:28:38.427000,1802.6015625 +2024-06-04 02:29:38.432000,1847.20703125 +2024-06-04 02:30:38.438000,1859.59375 +2024-06-04 02:31:38.448000,1868.171875 +2024-06-04 02:33:08.458000,1830.6640625 +2024-06-04 02:34:08.465000,1883.7109375 +2024-06-04 02:34:38.467000,1893.63671875 +2024-06-04 02:35:08.474000,1901.78515625 +2024-06-04 02:35:38.479000,1912.9296875 +2024-06-04 02:36:08.481000,1932.51171875 +2024-06-04 02:36:38.487000,1939.3203125 +2024-06-04 02:37:08.491000,1898.3125 +2024-06-04 02:37:38.494000,1906.39453125 +2024-06-04 02:38:08.498000,1913.25 +2024-06-04 02:38:38.498000,1922.63671875 +2024-06-04 02:39:38.503000,1979.796875 +2024-06-04 02:40:08.505000,1974.71484375 +2024-06-04 02:41:08.514000,1992.9921875 +2024-06-04 02:41:38.516000,2002.17578125 diff --git a/analysis/output_10b_run2_300s_flush_2b_memory_limit.txt b/analysis/output_10b_run2_300s_flush_2b_memory_limit.txt new file mode 100644 index 0000000..eec01ff --- /dev/null +++ b/analysis/output_10b_run2_300s_flush_2b_memory_limit.txt @@ -0,0 +1,440 @@ +worker1-1 | 2024/06/04 03:33:54.719 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 03:33:54.734 FIG10b: Starting tasks submission... +worker1-1 | 2024/06/04 03:33:55.734 FIG10b: Memory usage without flush after 122 tasks: 1012.97265625 MB +worker1-1 | 2024/06/04 03:33:56.736 FIG10b: Memory usage without flush after 243 tasks: 1013.26171875 MB +worker1-1 | [worker] stored...[worker]2024/06/04 03:33:57.744 FIG10b: Memory usage without flush after 367 tasks: 1014.125 MB +worker1-1 | 2024/06/04 03:33:58.747 FIG10b: Memory usage without flush after 473 tasks: 1017.6484375 MB +worker1-1 | 2024/06/04 03:33:59.759 FIG10b: Memory usage without flush after 579 tasks: 1021.41796875 MB +worker1-1 | 2024/06/04 03:34:00.767 2024/06/04 03:34:00.767 FIG10b: Memory usage without flush after 691 tasks: 1022.6875 MB +worker1-1 | Executed!2024/06/04 03:34:01.777 FIG10b: Memory usage without flush after 807 tasks: 1022.875 MB +worker1-1 | 2024/06/04 03:34:02.777 FIG10b: Memory usage without flush after 924 tasks: 1022.78125 MB +worker1-1 | 2024/06/04 03:34:03.785 FIG10b: Memory usage without flush after 1037 tasks: 1026.54296875 MB +worker1-1 | 2024/06/04 03:34:04.793 FIG10b: Memory usage without flush after 1137 tasks: 1025.8046875 MB +worker1-1 | [worker]2024/06/04 03:34:05.794 FIG10b: Memory usage without flush after 1237 tasks: 1025.7734375 MB +worker1-1 | 2024/06/04 03:34:06.800 FIG10b: Memory usage without flush after 1338 tasks: 1026.49609375 MB +worker1-1 | 2024/06/04 03:34:07.803[worker] FIG10b: Memory usage without flush after 1441 tasks: 1025.46484375 MB +worker1-1 | [worker] [worker]2024/06/04 03:34:08.809 FIG10b: Memory usage without flush after 1548 tasks: 1025.01171875 MB +worker1-1 | 2024/06/04 03:34:09.812 FIG10b: Memory usage without flush after 1645 tasks: 1025.4765625 MB +worker1-1 | 2024/06/04 03:34:10.816 FIG10b: Memory usage without flush after 1747 tasks: 1025.19140625 MB +worker1-1 | 2024/06/04 03:34:11.822 2024/06/04 03:34:11.823 FIG10b: Memory usage without flush after 1845 tasks: 1026.1015625 MB +worker1-1 | 2024/06/04 03:34:12.826 FIG10b: Memory usage without flush after 1948 tasks: 1025.671875 MB +worker1-1 | 2024/06/04 03:34:13.827 FIG10b: Memory usage without flush after 2043 tasks: 1025.4296875 MB +worker1-1 | 2024/06/04 03:34:14.846 FIG10b: Memory usage without flush after 2140 tasks: 1025.375 MB +worker1-1 | 2024/06/04 03:34:15.847 2024/06/04 03:34:15.847 FIG10b: Memory usage without flush after 2243 tasks: 1025.87890625 MB +worker1-1 | 2024/06/04 03:34:16.855 FIG10b: Memory usage without flush after 2333 tasks: 1026.3203125 MB +worker1-1 | 2024/06/04 03:34:17.865 2024/06/04 03:34:17.866 FIG10b: Memory usage without flush after 2420 tasks: 1025.1640625 MB +worker1-1 | 2024/06/04 03:34:18.872 FIG10b: Memory usage without flush after 2515 tasks: 1030.8359375 MB +worker1-1 | 2024/06/04 03:34:19.881 FIG10b: Memory usage without flush after 2598 tasks: 1035.09765625 MB +worker1-1 | 2024/06/04 03:34:20.8932024/06/04 03:34:20.893 FIG10b: Memory usage without flush after 2693 tasks: 1034.47265625 MB +worker1-1 | 2024/06/04 03:34:21.899 FIG10b: Memory usage without flush after 2791 tasks: 1035.06640625 MB +worker1-1 | 2024/06/04 03:34:22.904 FIG10b: Memory usage without flush after 2911 tasks: 1034.64453125 MB +worker1-1 | 2024/06/04 03:34:23.911 [worker] 2024/06/04 03:34:23.913 FIG10b: Memory usage without flush after 3016 tasks: 1035.40625 MB +worker1-1 | 2024/06/04 03:34:24.915 FIG10b: Memory usage without flush after 3133 tasks: 1034.56640625 MB +worker1-1 | FIG10b: Memory usage without flush after 3237 tasks: 1035.44140625 MB +worker1-1 | 2024/06/04 03:34:26.925 FIG10b: Memory usage without flush after 3335 tasks: 1037.55859375 MB +worker1-1 | 2024/06/04 03:34:27.934 FIG10b: Memory usage without flush after 3453 tasks: 1038.2890625 MB +worker1-1 | 2024/06/04 03:34:28.955 FIG10b: Memory usage without flush after 3562 tasks: 1039.37109375 MB +worker1-1 | FIG10b: Memory usage without flush after 3653 tasks: 1039.29296875 MB +worker1-1 | 2024/06/04 03:34:30.965 FIG10b: Memory usage without flush after 3764 tasks: 1039.109375 MB +worker1-1 | [worker]2024/06/04 03:34:31.978 [worker]2024/06/04 03:34:31.979 FIG10b: Memory usage without flush after 3879 tasks: 1039.41796875 MB +worker1-1 | 2024/06/04 03:34:32.9932024/06/04 03:34:32.993 FIG10b: Memory usage without flush after 3978 tasks: 1039.7734375 MB +worker1-1 | 2024/06/04 03:34:33.9982024/06/04 03:34:33.998 FIG10b: Memory usage without flush after 4071 tasks: 1039.69921875 MB +worker1-1 | 2024/06/04 03:34:35.0012024/06/04 03:34:35.001 FIG10b: Memory usage without flush after 4171 tasks: 1039.90234375 MB +worker1-1 | 2024/06/04 03:34:36.0022024/06/04 03:34:36.003 FIG10b: Memory usage without flush after 4294 tasks: 1039.9453125 MB +worker1-1 | 2024/06/04 03:34:37.010 FIG10b: Memory usage without flush after 4403 tasks: 1046.2890625 MB +worker1-1 | 2024/06/04 03:34:38.0342024/06/04 03:34:38.035 FIG10b: Memory usage without flush after 4505 tasks: 1047.60546875 MB +worker1-1 | took this many seconds: 0.03362024/06/04 03:34:39.036 FIG10b: Memory usage without flush after 4620 tasks: 1047.2734375 MB +worker1-1 | 2024/06/04 03:34:40.0362024/06/04 03:34:40.0362024/06/04 03:34:40.036 FIG10b: Memory usage without flush after 4737 tasks: 1047.328125 MB +worker1-1 | 2024/06/04 03:34:41.042 2024/06/04 03:34:41.042 FIG10b: Memory usage without flush after 4845 tasks: 1048.1328125 MB +worker1-1 | 2024/06/04 03:34:42.052 FIG10b: Memory usage without flush after 4965 tasks: 1049.14453125 MB +worker1-1 | 2024/06/04 03:34:43.052 FIG10b: Memory usage without flush after 5063 tasks: 1050.30078125 MB +worker1-1 | 2024/06/04 03:34:44.061 FIG10b: Memory usage without flush after 5153 tasks: 1050.24609375 MB +worker1-1 | [worker] in Run() rn2024/06/04 03:34:45.063 FIG10b: Memory usage without flush after 5249 tasks: 1052.046875 MB +worker1-1 | 2024/06/04 03:34:46.073 FIG10b: Memory usage without flush after 5330 tasks: 1051.2734375 MB +worker1-1 | 2024/06/04 03:34:47.079 FIG10b: Memory usage without flush after 5443 tasks: 1051.11328125 MB +worker1-1 | 2024/06/04 03:34:48.081 FIG10b: Memory usage without flush after 5553 tasks: 1057.07421875 MB +worker1-1 | 2024/06/04 03:34:49.0892024/06/04 03:34:49.090 FIG10b: Memory usage without flush after 5681 tasks: 1056.59375 MB +worker1-1 | 2024/06/04 03:34:50.0942024/06/04 03:34:50.094 FIG10b: Memory usage without flush after 5804 tasks: 1063.7734375 MB +worker1-1 | 2024/06/04 03:34:51.100 FIG10b: Memory usage without flush after 5918 tasks: 1063.61328125 MB +worker1-1 | [worker]2024/06/04 03:34:52.103 FIG10b: Memory usage without flush after 6036 tasks: 1064.015625 MB +worker1-1 | 2024/06/04 03:34:53.106 FIG10b: Memory usage without flush after 6152 tasks: 1065.19140625 MB +worker1-1 | 2024/06/04 03:34:54.110 FIG10b: Memory usage without flush after 6267 tasks: 1064.7109375 MB +worker1-1 | gonna store now2024/06/04 03:34:55.113 FIG10b: Memory usage without flush after 6383 tasks: 1065.71484375 MB +worker1-1 | 2024/06/04 03:34:56.118 FIG10b: Memory usage without flush after 6510 tasks: 1066.42578125 MB +worker1-1 | [worker] 2024/06/04 03:34:57.1232024/06/04 03:34:57.123 FIG10b: Memory usage without flush after 6629 tasks: 1067.671875 MB +worker1-1 | 2024/06/04 03:34:58.134 FIG10b: Memory usage without flush after 6748 tasks: 1068.85546875 MB +worker1-1 | 2024/06/04 03:34:59.137 FIG10b: Memory usage without flush after 6873 tasks: 1068.48046875 MB +worker1-1 | 2024/06/04 03:35:00.142 FIG10b: Memory usage without flush after 7012 tasks: 1069.43359375 MB +worker1-1 | Executed!2024/06/04 03:35:01.157 FIG10b: Memory usage without flush after 7149 tasks: 1069.0859375 MB +worker1-1 | 2024/06/04 03:35:02.158 FIG10b: Memory usage without flush after 7271 tasks: 1070.07421875 MB +worker1-1 | 2024/06/04 03:35:03.166 FIG10b: Memory usage without flush after 7382 tasks: 1069.4140625 MB +worker1-1 | 2024/06/04 03:35:04.167 FIG10b: Memory usage without flush after 7503 tasks: 1068.91015625 MB +worker1-1 | 2024/06/04 03:35:05.171 FIG10b: Memory usage without flush after 7622 tasks: 1070.0625 MB +worker1-1 | [worker]2024/06/04 03:35:06.174 FIG10b: Memory usage without flush after 7739 tasks: 1070.83984375 MB +worker1-1 | 2024/06/04 03:35:07.180 FIG10b: Memory usage without flush after 7846 tasks: 1070.69921875 MB +worker1-1 | 2024/06/04 03:35:08.184 FIG10b: Memory usage without flush after 7966 tasks: 1071.61328125 MB +worker1-1 | [worker] stored... Executed!2024/06/04 03:35:09.188 FIG10b: Memory usage without flush after 8078 tasks: 1072.328125 MB +worker1-1 | 2024/06/04 03:35:10.192 2024/06/04 03:35:10.1922024/06/04 03:35:10.1932024/06/04 03:35:10.193 FIG10b: Memory usage without flush after 8204 tasks: 1072.8359375 MB +worker1-1 | 2024/06/04 03:35:11.193 FIG10b: Memory usage without flush after 8333 tasks: 1071.91796875 MB +worker1-1 | 2024/06/04 03:35:12.193 2024/06/04 03:35:12.193 FIG10b: Memory usage without flush after 8443 tasks: 1072.87890625 MB +worker1-1 | 2024/06/04 03:35:13.203 FIG10b: Memory usage without flush after 8550 tasks: 1073.85546875 MB +worker1-1 | 2024/06/04 03:35:14.205 2024/06/04 03:35:14.205 FIG10b: Memory usage without flush after 8673 tasks: 1073.84375 MB +worker1-1 | 2024/06/04 03:35:15.211 FIG10b: Memory usage without flush after 8794 tasks: 1075.859375 MB +worker1-1 | 2024/06/04 03:35:16.214 [worker]2024/06/04 03:35:16.215 FIG10b: Memory usage without flush after 8921 tasks: 1075.84375 MB +worker1-1 | 2024/06/04 03:35:17.217 FIG10b: Memory usage without flush after 9042 tasks: 1076.56640625 MB +worker1-1 | 2024/06/04 03:35:18.2262024/06/04 03:35:18.227 FIG10b: Memory usage without flush after 9157 tasks: 1076.85546875 MB +worker1-1 | FIG10b: Memory usage without flush after 9258 tasks: 1077.8671875 MB +worker1-1 | 2024/06/04 03:35:20.239 FIG10b: Memory usage without flush after 9390 tasks: 1069.203125 MB +worker1-1 | 2024/06/04 03:35:21.248 [worker]2024/06/04 03:35:21.249 FIG10b: Memory usage without flush after 9516 tasks: 1069.6484375 MB +worker1-1 | 2024/06/04 03:35:22.254 FIG10b: Memory usage without flush after 9632 tasks: 1069.48828125 MB +worker1-1 | 2024/06/04 03:35:23.254 FIG10b: Memory usage without flush after 9746 tasks: 1069.06640625 MB +worker1-1 | 2024/06/04 03:35:24.260 FIG10b: Memory usage without flush after 9862 tasks: 1069.4765625 MB +worker1-1 | Executed!2024/06/04 03:35:25.265 FIG10b: Memory usage without flush after 9970 tasks: 1069.65625 MB +worker1-1 | 2024/06/04 03:35:26.2652024/06/04 03:35:26.265 FIG10b: Memory usage without flush after 10084 tasks: 1069.08203125 MB +worker1-1 | 2024/06/04 03:35:27.272 FIG10b: Memory usage without flush after 10192 tasks: 1069.46875 MB +worker1-1 | 2024/06/04 03:35:28.2752024/06/04 03:35:28.276[worker] FIG10b: Memory usage without flush after 10301 tasks: 1069.15234375 MB +worker1-1 | 2024/06/04 03:35:29.2762024/06/04 03:35:29.276 FIG10b: Memory usage without flush after 10408 tasks: 1069.77734375 MB +worker1-1 | 2024/06/04 03:35:30.280 FIG10b: Memory usage without flush after 10524 tasks: 1070.6875 MB +worker1-1 | 2024/06/04 03:35:31.285 in Run() rn2024/06/04 03:35:31.285 FIG10b: Memory usage without flush after 10629 tasks: 1070.36328125 MB +worker1-1 | 2024/06/04 03:35:32.288 FIG10b: Memory usage without flush after 10750 tasks: 1070.51171875 MB +worker1-1 | 2024/06/04 03:35:33.2882024/06/04 03:35:33.289 FIG10b: Memory usage without flush after 10874 tasks: 1070.49609375 MB +worker1-1 | Executed!2024/06/04 03:35:34.295 FIG10b: Memory usage without flush after 11015 tasks: 1069.796875 MB +worker1-1 | 2024/06/04 03:35:35.2952024/06/04 03:35:35.2952024/06/04 03:35:35.296 FIG10b: Memory usage without flush after 11148 tasks: 1070.359375 MB +worker1-1 | 2024/06/04 03:35:36.298 FIG10b: Memory usage without flush after 11279 tasks: 1070.12109375 MB +worker1-1 | 2024/06/04 03:35:37.3002024/06/04 03:35:37.300 FIG10b: Memory usage without flush after 11408 tasks: 1070.00390625 MB +worker1-1 | 2024/06/04 03:35:38.301 FIG10b: Memory usage without flush after 11548 tasks: 1070.30078125 MB +worker1-1 | 2024/06/04 03:35:39.304 FIG10b: Memory usage without flush after 11683 tasks: 1070.375 MB +worker1-1 | [worker] Executed!2024/06/04 03:35:40.309 FIG10b: Memory usage without flush after 11822 tasks: 1070.44140625 MB +worker1-1 | 2024/06/04 03:35:41.312 2024/06/04 03:35:41.313 FIG10b: Memory usage without flush after 11953 tasks: 1075.01171875 MB +worker1-1 | 2024/06/04 03:35:42.316 FIG10b: Memory usage without flush after 12080 tasks: 1075.08984375 MB +worker1-1 | 2024/06/04 03:35:43.3172024/06/04 03:35:43.317 FIG10b: Memory usage without flush after 12211 tasks: 1075.15234375 MB +worker1-1 | 2024/06/04 03:35:44.319 FIG10b: Memory usage without flush after 12351 tasks: 1076.2265625 MB +worker1-1 | 2024/06/04 03:35:45.319 FIG10b: Memory usage without flush after 12490 tasks: 1076.9453125 MB +worker1-1 | 2024/06/04 03:35:46.327 FIG10b: Memory usage without flush after 12612 tasks: 1077.54296875 MB +worker1-1 | gonna store now2024/06/04 03:35:47.345 FIG10b: Memory usage without flush after 12717 tasks: 1078.91796875 MB +worker1-1 | Executed!2024/06/04 03:35:48.3502024/06/04 03:35:48.350 FIG10b: Memory usage without flush after 12829 tasks: 1078.58203125 MB +worker1-1 | 2024/06/04 03:35:49.367 FIG10b: Memory usage without flush after 12960 tasks: 1095.51171875 MB +worker1-1 | 2024/06/04 03:35:50.371 FIG10b: Memory usage without flush after 13074 tasks: 1096.14453125 MB +worker1-1 | 2024/06/04 03:35:51.373 2024/06/04 03:35:51.374 FIG10b: Memory usage without flush after 13198 tasks: 1095.47265625 MB[worker] +worker1-1 | [worker]2024/06/04 03:35:52.375 FIG10b: Memory usage without flush after 13318 tasks: 1091.81640625 MB +worker1-1 | 2024/06/04 03:35:53.378 FIG10b: Memory usage without flush after 13452 tasks: 1091.921875 MB +worker1-1 | 2024/06/04 03:35:54.382 FIG10b: Memory usage without flush after 13579 tasks: 1093.2421875 MB +worker1-1 | 2024/06/04 03:35:55.387 FIG10b: Memory usage without flush after 13703 tasks: 1094.23828125 MB +worker1-1 | 2024/06/04 03:35:56.389 FIG10b: Memory usage without flush after 13831 tasks: 1094.46484375 MB +worker1-1 | 2024/06/04 03:35:57.396 FIG10b: Memory usage without flush after 13941 tasks: 1095.83984375 MB +worker1-1 | 2024/06/04 03:35:58.398 FIG10b: Memory usage without flush after 14054 tasks: 1096.02734375 MB +worker1-1 | 2024/06/04 03:35:59.406 FIG10b: Memory usage without flush after 14161 tasks: 1095.27734375 MB +worker1-1 | FIG10b: Memory usage without flush after 14266 tasks: 1095.265625 MB +worker1-1 | 2024/06/04 03:36:01.410 FIG10b: Memory usage without flush after 14382 tasks: 1095.453125 MB +worker1-1 | 2024/06/04 03:36:02.417 FIG10b: Memory usage without flush after 14487 tasks: 1096.1484375 MB +worker1-1 | 2024/06/04 03:36:03.4222024/06/04 03:36:03.423 FIG10b: Memory usage without flush after 14604 tasks: 1096.5625 MB +worker1-1 | 2024/06/04 03:36:04.434 FIG10b: Memory usage without flush after 14700 tasks: 1096.8203125 MB[worker] took this many seconds: 0.0298 +worker1-1 | 2024/06/04 03:36:05.444 FIG10b: Memory usage without flush after 14785 tasks: 1097.08203125 MB +worker1-1 | 2024/06/04 03:36:06.502 FIG10b: Memory usage without flush after 14899 tasks: 1097.53125 MB +worker1-1 | 2024/06/04 03:36:07.511 FIG10b: Memory usage without flush after 15011 tasks: 1097.96875 MB +worker1-1 | 2024/06/04 03:36:08.515 FIG10b: Memory usage without flush after 15101 tasks: 1097.9609375 MB +worker1-1 | 2024/06/04 03:36:09.5262024/06/04 03:36:09.526 FIG10b: Memory usage without flush after 15202 tasks: 1098.64453125 MB +worker1-1 | 2024/06/04 03:36:10.529 FIG10b: Memory usage without flush after 15305 tasks: 1098.609375 MB +worker1-1 | 2024/06/04 03:36:11.532 FIG10b: Memory usage without flush after 15392 tasks: 1098.86328125 MB +worker1-1 | 2024/06/04 03:36:12.546 FIG10b: Memory usage without flush after 15464 tasks: 1098.69140625 MB +worker1-1 | 2024/06/04 03:36:13.551 FIG10b: Memory usage without flush after 15554 tasks: 1099.46875 MB +worker1-1 | 2024/06/04 03:36:14.557 FIG10b: Memory usage without flush after 15644 tasks: 1099.46484375 MB +worker1-1 | 2024/06/04 03:36:15.5642024/06/04 03:36:15.565 FIG10b: Memory usage without flush after 15748 tasks: 1099.40234375 MB +worker1-1 | [worker] stored...2024/06/04 03:36:16.571 FIG10b: Memory usage without flush after 15848 tasks: 1097.1796875 MB +worker1-1 | [worker] stored...2024/06/04 03:36:17.5822024/06/04 03:36:17.582 FIG10b: Memory usage without flush after 15939 tasks: 1098.3671875 MB +worker1-1 | 2024/06/04 03:36:18.5842024/06/04 03:36:18.584 FIG10b: Memory usage without flush after 16026 tasks: 1097.3671875 MB +worker1-1 | 2024/06/04 03:36:19.5962024/06/04 03:36:19.596 2024/06/04 03:36:19.597 FIG10b: Memory usage without flush after 16113 tasks: 1097.875 MB +worker1-1 | 2024/06/04 03:36:20.600 FIG10b: Memory usage without flush after 16219 tasks: 1098.4921875 MB +worker1-1 | 2024/06/04 03:36:21.601 2024/06/04 03:36:21.602 FIG10b: Memory usage without flush after 16330 tasks: 1097.99609375 MB +worker1-1 | 2024/06/04 03:36:22.607[worker] FIG10b: Memory usage without flush after 16451 tasks: 1096.95703125 MB +worker1-1 | 2024/06/04 03:36:23.6132024/06/04 03:36:23.613 FIG10b: Memory usage without flush after 16577 tasks: 1097.08203125 MB +worker1-1 | 2024/06/04 03:36:24.615 FIG10b: Memory usage without flush after 16692 tasks: 1098.44921875 MB +worker1-1 | 2024/06/04 03:36:25.619 FIG10b: Memory usage without flush after 16810 tasks: 1098.3203125 MB +worker1-1 | 2024/06/04 03:36:26.6232024/06/04 03:36:26.623 FIG10b: Memory usage without flush after 16929 tasks: 1098.296875 MB +worker1-1 | [worker] [worker] took this many seconds: 0.00839 2024/06/04 03:36:27.629 FIG10b: Memory usage without flush after 17054 tasks: 1098.3046875 MB +worker1-1 | 2024/06/04 03:36:28.639 2024/06/04 03:36:28.639FIG10b: Memory usage without flush after 17176 tasks: 1098.6953125 MB +worker1-1 | 2024/06/04 03:36:29.644 FIG10b: Memory usage without flush after 17299 tasks: 1098.828125 MB +worker1-1 | 2024/06/04 03:36:30.649 FIG10b: Memory usage without flush after 17423 tasks: 1099.31640625 MB +worker1-1 | 2024/06/04 03:36:31.657 FIG10b: Memory usage without flush after 17544 tasks: 1099.48046875 MB +worker1-1 | in Run() rn2024/06/04 03:36:32.659 FIG10b: Memory usage without flush after 17673 tasks: 1099.81640625 MB +worker1-1 | 2024/06/04 03:36:33.6642024/06/04 03:36:33.665 FIG10b: Memory usage without flush after 17787 tasks: 1099.8125 MB +worker1-1 | 2024/06/04 03:36:34.665 FIG10b: Memory usage without flush after 17902 tasks: 1099.734375 MB +worker1-1 | 2024/06/04 03:36:35.6672024/06/04 03:36:35.667 FIG10b: Memory usage without flush after 18025 tasks: 1100.22265625 MB +worker1-1 | [worker] 2024/06/04 03:36:36.673 FIG10b: Memory usage without flush after 18159 tasks: 1100.91015625 MB +worker1-1 | FIG10b: Memory usage without flush after 18273 tasks: 1101.87109375 MB +worker1-1 | 2024/06/04 03:36:38.681 FIG10b: Memory usage without flush after 18390 tasks: 1101.14453125 MB +worker1-1 | 2024/06/04 03:36:39.684 FIG10b: Memory usage without flush after 18504 tasks: 1101.84765625 MB +worker1-1 | 2024/06/04 03:36:40.6882024/06/04 03:36:40.688 FIG10b: Memory usage without flush after 18616 tasks: 1102.7890625 MB +worker1-1 | 2024/06/04 03:36:41.696 FIG10b: Memory usage without flush after 18741 tasks: 1102.9375 MB +worker1-1 | FIG10b: Memory usage without flush after 18871 tasks: 1103.6171875 MB +worker1-1 | 2024/06/04 03:36:43.712 FIG10b: Memory usage without flush after 18998 tasks: 1104.55859375 MB +worker1-1 | [worker] in Run() rn2024/06/04 03:36:44.712 FIG10b: Memory usage without flush after 19127 tasks: 1105.1796875 MB +worker1-1 | 2024/06/04 03:36:45.713 FIG10b: Memory usage without flush after 19259 tasks: 1105.37890625 MB +worker1-1 | 2024/06/04 03:36:46.721 FIG10b: Memory usage without flush after 19372 tasks: 1105.16015625 MB +worker1-1 | 2024/06/04 03:36:47.735 FIG10b: Memory usage without flush after 19424 tasks: 1105.28125 MB +worker1-1 | 2024/06/04 03:36:48.7402024/06/04 03:36:48.740 FIG10b: Memory usage without flush after 19522 tasks: 1105.9453125 MB +worker1-1 | 2024/06/04 03:36:49.753 FIG10b: Memory usage without flush after 19635 tasks: 1115.32421875 MB +worker1-1 | 2024/06/04 03:36:50.765 FIG10b: Memory usage without flush after 19716 tasks: 1116.33984375 MB +worker1-1 | stored...2024/06/04 03:36:51.773 FIG10b: Memory usage without flush after 19820 tasks: 1116.3515625 MB +worker1-1 | 2024/06/04 03:36:52.782 FIG10b: Memory usage without flush after 19906 tasks: 1116.1484375 MB +worker1-1 | in Run() rn2024/06/04 03:36:53.788 FIG10b: Memory usage without flush after 20029 tasks: 1117.4765625 MB +worker1-1 | 2024/06/04 03:36:54.8012024/06/04 03:36:54.802 FIG10b: Memory usage without flush after 20145 tasks: 1117.5625 MB +worker1-1 | 2024/06/04 03:36:55.8042024/06/04 03:36:55.805 FIG10b: Memory usage without flush after 20265 tasks: 1118.0 MB +worker1-1 | 2024/06/04 03:36:56.804 [worker]2024/06/04 03:36:56.804 FIG10b: Memory usage without flush after 20381 tasks: 1118.46875 MB +worker1-1 | 2024/06/04 03:36:57.807 FIG10b: Memory usage without flush after 20496 tasks: 1118.36328125 MB +worker1-1 | 2024/06/04 03:36:58.810 [worker]2024/06/04 03:36:58.811 FIG10b: Memory usage without flush after 20611 tasks: 1119.22265625 MB +worker1-1 | 2024/06/04 03:36:59.815 FIG10b: Memory usage without flush after 20730 tasks: 1119.8828125 MB +worker1-1 | 2024/06/04 03:37:00.827 FIG10b: Memory usage without flush after 20852 tasks: 1120.515625 MB +worker1-1 | 2024/06/04 03:37:01.830 FIG10b: Memory usage without flush after 20969 tasks: 1120.85546875 MB +worker1-1 | 2024/06/04 03:37:02.836 [worker] 2024/06/04 03:37:02.837 FIG10b: Memory usage without flush after 21093 tasks: 1120.75390625 MB +worker1-1 | 2024/06/04 03:37:03.844 FIG10b: Memory usage without flush after 21214 tasks: 1121.13671875 MB +worker1-1 | 2024/06/04 03:37:04.845 FIG10b: Memory usage without flush after 21338 tasks: 1121.2421875 MB +worker1-1 | 2024/06/04 03:37:05.845 FIG10b: Memory usage without flush after 21456 tasks: 1121.77734375 MB +worker1-1 | 2024/06/04 03:37:06.851 FIG10b: Memory usage without flush after 21582 tasks: 1121.4296875 MB +worker1-1 | 2024/06/04 03:37:07.855 FIG10b: Memory usage without flush after 21698 tasks: 1121.3203125 MB +worker1-1 | 2024/06/04 03:37:08.855 FIG10b: Memory usage without flush after 21819 tasks: 1122.23046875 MB +worker1-1 | 2024/06/04 03:37:09.8592024/06/04 03:37:09.860 FIG10b: Memory usage without flush after 21938 tasks: 1121.94140625 MB +worker1-1 | 2024/06/04 03:37:10.865 FIG10b: Memory usage without flush after 22061 tasks: 1122.046875 MB +worker1-1 | 2024/06/04 03:37:11.867 2024/06/04 03:37:11.867[worker] Executed!FIG10b: Memory usage without flush after 22185 tasks: 1122.3828125 MB +worker1-1 | 2024/06/04 03:37:12.868 FIG10b: Memory usage without flush after 22307 tasks: 1122.72265625 MB +worker1-1 | 2024/06/04 03:37:13.879 FIG10b: Memory usage without flush after 22425 tasks: 1122.13671875 MB +worker1-1 | 2024/06/04 03:37:14.888 FIG10b: Memory usage without flush after 22551 tasks: 1123.8359375 MB +worker1-1 | 2024/06/04 03:37:15.890 FIG10b: Memory usage without flush after 22671 tasks: 1123.9375 MB +worker1-1 | [worker] [worker] 2024/06/04 03:37:16.893 FIG10b: Memory usage without flush after 22789 tasks: 1123.1796875 MB +worker1-1 | 2024/06/04 03:37:17.903 FIG10b: Memory usage without flush after 22897 tasks: 1123.37109375 MB +worker1-1 | 2024/06/04 03:37:18.911 FIG10b: Memory usage without flush after 23011 tasks: 1123.01953125 MB +worker1-1 | FIG10b: Memory usage without flush after 23127 tasks: 1123.34765625 MB +worker1-1 | 2024/06/04 03:37:20.918 FIG10b: Memory usage without flush after 23234 tasks: 1123.0546875 MB +worker1-1 | 2024/06/04 03:37:21.922 FIG10b: Memory usage without flush after 23352 tasks: 1122.96875 MB +worker1-1 | 2024/06/04 03:37:22.927 FIG10b: Memory usage without flush after 23475 tasks: 1124.24609375 MB +worker1-1 | 2024/06/04 03:37:23.937 2024/06/04 03:37:23.938 FIG10b: Memory usage without flush after 23599 tasks: 1124.05859375 MB +worker1-1 | 2024/06/04 03:37:24.941 FIG10b: Memory usage without flush after 23723 tasks: 1124.3671875 MB +worker1-1 | FIG10b: Memory usage without flush after 23850 tasks: 1124.79296875 MB +worker1-1 | 2024/06/04 03:37:26.951 FIG10b: Memory usage without flush after 23975 tasks: 1124.68359375 MB +worker1-1 | 2024/06/04 03:37:27.958 [worker] 2024/06/04 03:37:27.959 FIG10b: Memory usage without flush after 24085 tasks: 1124.91015625 MB +worker1-1 | 2024/06/04 03:37:28.959 FIG10b: Memory usage without flush after 24202 tasks: 1124.66015625 MB +worker1-1 | 2024/06/04 03:37:29.968 FIG10b: Memory usage without flush after 24319 tasks: 1124.80078125 MB +worker1-1 | 2024/06/04 03:37:30.968 FIG10b: Memory usage without flush after 24438 tasks: 1123.5859375 MB +worker1-1 | 2024/06/04 03:37:31.970 FIG10b: Memory usage without flush after 24565 tasks: 1125.1875 MB +worker1-1 | 2024/06/04 03:37:32.978 FIG10b: Memory usage without flush after 24694 tasks: 1125.23828125 MB +worker1-1 | Executed!2024/06/04 03:37:33.990 FIG10b: Memory usage without flush after 24818 tasks: 1124.94140625 MB +worker1-1 | 2024/06/04 03:37:34.9982024/06/04 03:37:34.998 FIG10b: Memory usage without flush after 24936 tasks: 1125.39453125 MB +worker1-1 | 2024/06/04 03:37:36.003 FIG10b: Memory usage without flush after 25061 tasks: 1125.3203125 MB +worker1-1 | 2024/06/04 03:37:37.010 FIG10b: Memory usage without flush after 25176 tasks: 1126.4296875 MB +worker1-1 | 2024/06/04 03:37:38.010 2024/06/04 03:37:38.010 2024/06/04 03:37:38.011 FIG10b: Memory usage without flush after 25291 tasks: 1126.6875 MB +worker1-1 | 2024/06/04 03:37:39.020 FIG10b: Memory usage without flush after 25403 tasks: 1126.13671875 MB +worker1-1 | 2024/06/04 03:37:40.029 FIG10b: Memory usage without flush after 25521 tasks: 1126.3515625 MB +worker1-1 | 2024/06/04 03:37:41.029 FIG10b: Memory usage without flush after 25635 tasks: 1126.3203125 MB +worker1-1 | 2024/06/04 03:37:42.078 FIG10b: Memory usage without flush after 25706 tasks: 1126.84375 MB +worker1-1 | 2024/06/04 03:37:43.086 FIG10b: Memory usage without flush after 25765 tasks: 1127.20703125 MB +worker1-1 | 2024/06/04 03:37:44.089 FIG10b: Memory usage without flush after 25870 tasks: 1127.4453125 MB +worker1-1 | 2024/06/04 03:37:45.093 FIG10b: Memory usage without flush after 25993 tasks: 1127.94921875 MB +worker1-1 | [worker] gonna store now2024/06/04 03:37:46.097 FIG10b: Memory usage without flush after 26123 tasks: 1128.65234375 MB +worker1-1 | 2024/06/04 03:37:47.100 [worker]2024/06/04 03:37:47.100 FIG10b: Memory usage without flush after 26246 tasks: 1130.03125 MB +worker1-1 | 2024/06/04 03:37:48.102 FIG10b: Memory usage without flush after 26364 tasks: 1130.19921875 MB +worker1-1 | 2024/06/04 03:37:49.106 FIG10b: Memory usage without flush after 26484 tasks: 1130.30859375 MB +worker1-1 | 2024/06/04 03:37:50.111 FIG10b: Memory usage without flush after 26617 tasks: 1135.0 MB +worker1-1 | 2024/06/04 03:37:51.118 FIG10b: Memory usage without flush after 26744 tasks: 1140.1171875 MB +worker1-1 | 2024/06/04 03:37:52.123 FIG10b: Memory usage without flush after 26860 tasks: 1140.51953125 MB +worker1-1 | FIG10b: Memory usage without flush after 26984 tasks: 1140.375 MB +worker1-1 | 2024/06/04 03:37:54.135 FIG10b: Memory usage without flush after 27100 tasks: 1140.375 MB +worker1-1 | 2024/06/04 03:37:55.138 FIG10b: Memory usage without flush after 27218 tasks: 1141.25390625 MB +worker1-1 | 2024/06/04 03:37:56.143 FIG10b: Memory usage without flush after 27332 tasks: 1139.09375 MB +worker1-1 | 2024/06/04 03:37:57.149 FIG10b: Memory usage without flush after 27451 tasks: 1140.0078125 MB +worker1-1 | 2024/06/04 03:37:58.152 FIG10b: Memory usage without flush after 27578 tasks: 1144.21875 MB +worker1-1 | 2024/06/04 03:37:59.155 FIG10b: Memory usage without flush after 27698 tasks: 1144.8671875 MB +worker1-1 | 2024/06/04 03:38:00.164 FIG10b: Memory usage without flush after 27830 tasks: 1145.25390625 MB +worker1-1 | 2024/06/04 03:38:01.168 FIG10b: Memory usage without flush after 27952 tasks: 1145.1640625 MB +worker1-1 | [worker] 2024/06/04 03:38:02.177stored... FIG10b: Memory usage without flush after 28075 tasks: 1146.30859375 MB +worker1-1 | 2024/06/04 03:38:03.1812024/06/04 03:38:03.181 FIG10b: Memory usage without flush after 28195 tasks: 1146.046875 MB +worker1-1 | 2024/06/04 03:38:04.191 FIG10b: Memory usage without flush after 28321 tasks: 1146.23828125 MB +worker1-1 | 2024/06/04 03:38:05.198 FIG10b: Memory usage without flush after 28443 tasks: 1146.875 MB +worker1-1 | 2024/06/04 03:38:06.1992024/06/04 03:38:06.199 FIG10b: Memory usage without flush after 28567 tasks: 1147.00390625 MB +worker1-1 | 2024/06/04 03:38:07.202 FIG10b: Memory usage without flush after 28687 tasks: 1147.734375 MB +worker1-1 | stored...2024/06/04 03:38:08.2082024/06/04 03:38:08.208 FIG10b: Memory usage without flush after 28812 tasks: 1147.4140625 MB +worker1-1 | 2024/06/04 03:38:09.212 FIG10b: Memory usage without flush after 28931 tasks: 1147.59765625 MB +worker1-1 | 2024/06/04 03:38:10.216 FIG10b: Memory usage without flush after 29010 tasks: 1147.1953125 MB +worker1-1 | [worker]2024/06/04 03:38:11.225 FIG10b: Memory usage without flush after 29110 tasks: 1146.921875 MB +worker1-1 | FIG10b: Memory usage without flush after 29236 tasks: 1148.05078125 MB +worker1-1 | 2024/06/04 03:38:13.2322024/06/04 03:38:13.232 [worker]FIG10b: Memory usage without flush after 29359 tasks: 1148.52734375 MB +worker1-1 | 2024/06/04 03:38:14.2402024/06/04 03:38:14.240 FIG10b: Memory usage without flush after 29478 tasks: 1151.6875 MB +worker1-1 | 2024/06/04 03:38:15.2432024/06/04 03:38:15.244 [worker]FIG10b: Memory usage without flush after 29600 tasks: 1150.91796875 MB +worker1-1 | 2024/06/04 03:38:16.255 FIG10b: Memory usage without flush after 29716 tasks: 1150.5625 MB +worker1-1 | 2024/06/04 03:38:17.260 FIG10b: Memory usage without flush after 29842 tasks: 1148.6796875 MB +worker1-1 | [worker] stored...2024/06/04 03:38:18.275 FIG10b: Memory usage without flush after 29974 tasks: 1147.09765625 MB +worker1-1 | 2024/06/04 03:38:19.285 FIG10b: Memory usage without flush after 30098 tasks: 1146.625 MB +worker1-1 | 2024/06/04 03:38:20.289 FIG10b: Memory usage without flush after 30192 tasks: 1146.55078125 MB +worker1-1 | 2024/06/04 03:38:21.2932024/06/04 03:38:21.294 FIG10b: Memory usage without flush after 30301 tasks: 1146.21875 MB +worker1-1 | 2024/06/04 03:38:22.301 [worker]2024/06/04 03:38:22.3022024/06/04 03:38:22.303 FIG10b: Memory usage without flush after 30418 tasks: 1146.8828125 MB +worker1-1 | Executed![worker]2024/06/04 03:38:23.307 FIG10b: Memory usage without flush after 30543 tasks: 1146.13671875 MB +worker1-1 | 2024/06/04 03:38:24.309 FIG10b: Memory usage without flush after 30668 tasks: 1147.875 MB +worker1-1 | 2024/06/04 03:38:25.311 FIG10b: Memory usage without flush after 30793 tasks: 1147.71875 MB +worker1-1 | 2024/06/04 03:38:26.317 FIG10b: Memory usage without flush after 30915 tasks: 1147.58984375 MB +worker1-1 | 2024/06/04 03:38:27.323 2024/06/04 03:38:27.323 2024/06/04 03:38:27.324 FIG10b: Memory usage without flush after 31047 tasks: 1147.44140625 MB +worker1-1 | 2024/06/04 03:38:28.3372024/06/04 03:38:28.338 FIG10b: Memory usage without flush after 31175 tasks: 1147.54296875 MB +worker1-1 | 2024/06/04 03:38:29.341 FIG10b: Memory usage without flush after 31313 tasks: 1147.875 MB +worker1-1 | [worker] 2024/06/04 03:38:30.345 FIG10b: Memory usage without flush after 31437 tasks: 1147.96875 MB +worker1-1 | [worker][worker]2024/06/04 03:38:31.352 FIG10b: Memory usage without flush after 31560 tasks: 1148.609375 MB +worker1-1 | 2024/06/04 03:38:32.352 FIG10b: Memory usage without flush after 31685 tasks: 1148.92578125 MB +worker1-1 | 2024/06/04 03:38:33.3572024/06/04 03:38:33.357 FIG10b: Memory usage without flush after 31809 tasks: 1149.67578125 MB +worker1-1 | 2024/06/04 03:38:34.360 FIG10b: Memory usage without flush after 31941 tasks: 1153.609375 MB +worker1-1 | 2024/06/04 03:38:35.364 2024/06/04 03:38:35.364 FIG10b: Memory usage without flush after 32062 tasks: 1153.7421875 MB +worker1-1 | 2024/06/04 03:38:36.369 FIG10b: Memory usage without flush after 32193 tasks: 1153.8046875 MB +worker1-1 | 2024/06/04 03:38:37.3752024/06/04 03:38:37.376 FIG10b: Memory usage without flush after 32320 tasks: 1153.8984375 MB +worker1-1 | 2024/06/04 03:38:38.383 FIG10b: Memory usage without flush after 32445 tasks: 1153.578125 MB +worker1-1 | 2024/06/04 03:38:39.389 FIG10b: Memory usage without flush after 32571 tasks: 1154.625 MB +worker1-1 | 2024/06/04 03:38:40.3912024/06/04 03:38:40.392 FIG10b: Memory usage without flush after 32701 tasks: 1154.89453125 MB +worker1-1 | 2024/06/04 03:38:41.394 FIG10b: Memory usage without flush after 32826 tasks: 1154.28125 MB +worker1-1 | FIG10b: Memory usage without flush after 32949 tasks: 1155.33984375 MB +worker1-1 | 2024/06/04 03:38:43.4042024/06/04 03:38:43.404 2024/06/04 03:38:43.404 FIG10b: Memory usage without flush after 33069 tasks: 1155.69921875 MB +worker1-1 | in Run() rn2024/06/04 03:38:44.410 FIG10b: Memory usage without flush after 33196 tasks: 1156.5 MB +worker1-1 | FIG10b: Memory usage without flush after 33315 tasks: 1156.82421875 MB +worker1-1 | 2024/06/04 03:38:46.416 FIG10b: Memory usage without flush after 33444 tasks: 1156.66015625 MB +worker1-1 | 2024/06/04 03:38:47.418 2024/06/04 03:38:47.419 FIG10b: Memory usage without flush after 33569 tasks: 1157.25 MB +worker1-1 | stored...2024/06/04 03:38:48.422 2024/06/04 03:38:48.423 FIG10b: Memory usage without flush after 33699 tasks: 1157.5703125 MB +worker1-1 | 2024/06/04 03:38:49.429 FIG10b: Memory usage without flush after 33843 tasks: 1166.3828125 MB +worker1-1 | FIG10b: Memory usage without flush after 33977 tasks: 1166.5703125 MB +worker1-1 | 2024/06/04 03:38:51.439 FIG10b: Memory usage without flush after 34102 tasks: 1166.8671875 MB +worker1-1 | 2024/06/04 03:38:52.438 FIG10b: Memory usage without flush after 34224 tasks: 1167.05859375 MB +worker1-1 | 2024/06/04 03:38:53.446 FIG10b: Memory usage without flush after 34349 tasks: 1160.99609375 MB +worker1-1 | gonna store now2024/06/04 03:38:54.447 2024/06/04 03:38:54.448 FIG10b: Memory usage without flush after 34476 tasks: 1160.08984375 MB +worker1-1 | FIG10b: Memory usage without flush after 34607 tasks: 1160.0078125 MB +worker1-1 | 2024/06/04 03:39:25.455 FIG10b: Memory usage without flush after 38383 tasks: 1168.05078125 MB +worker1-1 | 2024/06/04 03:39:55.459 2024/06/04 03:39:55.459[worker] stored...FIG10b: Memory usage without flush after 42110 tasks: 1179.35546875 MB +worker1-1 | 2024/06/04 03:40:25.4622024/06/04 03:40:25.463 2024/06/04 03:40:25.463 FIG10b: Memory usage without flush after 45621 tasks: 1181.609375 MB +worker1-1 | 2024/06/04 03:40:55.469 FIG10b: Memory usage without flush after 49078 tasks: 1239.30859375 MB +worker1-1 | [worker]Executed! 2024/06/04 03:41:25.473 FIG10b: Memory usage without flush after 52745 tasks: 1236.12890625 MB +worker1-1 | 2024/06/04 03:41:55.479 FIG10b: Memory usage without flush after 56159 tasks: 1259.33203125 MB +worker1-1 | 2024/06/04 03:42:25.482 2024/06/04 03:42:25.483 FIG10b: Memory usage without flush after 59753 tasks: 1264.109375 MB +worker1-1 | 2024/06/04 03:42:55.488 FIG10b: Memory usage without flush after 63191 tasks: 1283.40625 MB +worker1-1 | 2024/06/04 03:43:25.496 FIG10b: Memory usage without flush after 66727 tasks: 1284.46875 MB +worker1-1 | [worker] 2024/06/04 03:43:55.501 FIG10b: Memory usage without flush after 70311 tasks: 1311.26953125 MB +worker1-1 | FIG10b: Memory usage without flush after 74003 tasks: 1314.0703125 MB +worker1-1 | [worker]2024/06/04 03:44:55.505 FIG10b: Memory usage without flush after 77762 tasks: 1350.19140625 MB +worker1-1 | FIG10b: Memory usage without flush after 81519 tasks: 1345.234375 MB +worker1-1 | 2024/06/04 03:45:55.516 2024/06/04 03:45:55.517[worker] FIG10b: Memory usage without flush after 85240 tasks: 1362.51171875 MB +worker1-1 | 2024/06/04 03:46:25.521 FIG10b: Memory usage without flush after 88858 tasks: 1366.875 MB +worker1-1 | 2024/06/04 03:46:55.5212024/06/04 03:46:55.521 FIG10b: Memory usage without flush after 92388 tasks: 1377.2109375 MB +worker1-1 | 2024/06/04 03:47:25.524 FIG10b: Memory usage without flush after 95891 tasks: 1382.34375 MB +worker1-1 | 2024/06/04 03:47:55.534 FIG10b: Memory usage without flush after 99075 tasks: 1372.3203125 MB +worker1-1 | 2024/06/04 03:48:25.535[worker] FIG10b: Memory usage without flush after 102285 tasks: 1379.0234375 MBin Run() rn +worker1-1 | 2024/06/04 03:48:55.535 2024/06/04 03:48:55.536[worker] FIG10b: Memory usage without flush after 105536 tasks: 1406.2265625 MB +worker1-1 | 2024/06/04 03:49:25.541 FIG10b: Memory usage without flush after 108916 tasks: 1444.30078125 MB +worker1-1 | 2024/06/04 03:49:55.5422024/06/04 03:49:55.542 FIG10b: Memory usage without flush after 112426 tasks: 1444.98828125 MB +worker1-1 | 2024/06/04 03:50:25.546 FIG10b: Memory usage without flush after 115719 tasks: 1449.98828125 MB +worker1-1 | 2024/06/04 03:50:55.551 FIG10b: Memory usage without flush after 118837 tasks: 1466.87890625 MB +worker1-1 | 2024/06/04 03:51:25.5532024/06/04 03:51:25.553 FIG10b: Memory usage without flush after 122080 tasks: 1470.30859375 MB +worker1-1 | 2024/06/04 03:51:55.557 FIG10b: Memory usage without flush after 125602 tasks: 1483.0234375 MB +worker1-1 | 2024/06/04 03:52:25.559 [worker]2024/06/04 03:52:25.560 FIG10b: Memory usage without flush after 129043 tasks: 1481.91796875 MB +worker1-1 | 2024/06/04 03:52:55.563 FIG10b: Memory usage without flush after 132450 tasks: 1513.37890625 MB +worker1-1 | 2024/06/04 03:53:25.566 FIG10b: Memory usage without flush after 135785 tasks: 1512.1796875 MB +worker1-1 | 2024/06/04 03:53:55.569 2024/06/04 03:53:55.570 FIG10b: Memory usage without flush after 139343 tasks: 1516.23828125 MB +worker1-1 | 2024/06/04 03:54:25.576 FIG10b: Memory usage without flush after 142807 tasks: 1515.0546875 MB +worker1-1 | 2024/06/04 03:54:55.578 FIG10b: Memory usage without flush after 146258 tasks: 1530.1796875 MB +worker1-1 | 2024/06/04 03:55:25.5922024/06/04 03:55:25.593 FIG10b: Memory usage without flush after 149944 tasks: 1538.828125 MB +worker1-1 | 2024/06/04 03:55:55.593 FIG10b: Memory usage without flush after 153732 tasks: 1552.4609375 MB +worker1-1 | 2024/06/04 03:56:25.5972024/06/04 03:56:25.597 [worker] gonna store now FIG10b: Memory usage without flush after 157388 tasks: 1559.82421875 MB +worker1-1 | 2024/06/04 03:56:55.602 FIG10b: Memory usage without flush after 160873 tasks: 1554.2890625 MB +worker1-1 | 2024/06/04 03:57:25.603 FIG10b: Memory usage without flush after 164476 tasks: 1551.7109375 MB +worker1-1 | 2024/06/04 03:57:55.605 FIG10b: Memory usage without flush after 167926 tasks: 1572.1171875 MB +worker1-1 | 2024/06/04 03:58:25.605 FIG10b: Memory usage without flush after 171576 tasks: 1562.6640625 MB +worker1-1 | 2024/06/04 03:58:55.606 FIG10b: Memory usage without flush after 174582 tasks: 1589.87109375 MB +worker1-1 | 2024/06/04 03:59:25.609 FIG10b: Memory usage without flush after 177860 tasks: 1593.6640625 MB +worker1-1 | 2024/06/04 03:59:55.6102024/06/04 03:59:55.610 FIG10b: Memory usage without flush after 181241 tasks: 1601.9296875 MB +worker1-1 | 2024/06/04 04:00:25.6152024/06/04 04:00:25.615 [worker] FIG10b: Memory usage without flush after 184613 tasks: 1588.4140625 MBExecuted! +worker1-1 | in Run() rn2024/06/04 04:00:55.616 FIG10b: Memory usage without flush after 187904 tasks: 1619.25 MB +worker1-1 | 2024/06/04 04:01:25.623 FIG10b: Memory usage without flush after 191567 tasks: 1618.2421875 MB +worker1-1 | 2024/06/04 04:01:55.630 FIG10b: Memory usage without flush after 195241 tasks: 1628.71484375 MB +worker1-1 | 2024/06/04 04:02:25.632 FIG10b: Memory usage without flush after 198610 tasks: 1638.32421875 MB +worker1-1 | 2024/06/04 04:02:55.637 FIG10b: Memory usage without flush after 201951 tasks: 1632.52734375 MB +worker1-1 | 2024/06/04 04:03:25.640 FIG10b: Memory usage without flush after 205401 tasks: 1623.203125 MB +worker1-1 | [worker]gonna store now 2024/06/04 04:03:55.641 FIG10b: Memory usage without flush after 208998 tasks: 1651.734375 MB +worker1-1 | FIG10b: Memory usage without flush after 212537 tasks: 1638.828125 MB +worker1-1 | 2024/06/04 04:04:55.648 FIG10b: Memory usage without flush after 216026 tasks: 1695.09375 MB +worker1-1 | 2024/06/04 04:05:25.6512024/06/04 04:05:25.652 FIG10b: Memory usage without flush after 219372 tasks: 1726.16015625 MB +worker1-1 | 2024/06/04 04:05:55.653 FIG10b: Memory usage without flush after 222864 tasks: 1734.11328125 MB +worker1-1 | [worker]2024/06/04 04:06:25.6542024/06/04 04:06:25.654 FIG10b: Memory usage without flush after 226081 tasks: 1731.8203125 MB +worker1-1 | 2024/06/04 04:06:55.658 FIG10b: Memory usage without flush after 229701 tasks: 1756.484375 MB +worker1-1 | 2024/06/04 04:07:25.660 FIG10b: Memory usage without flush after 233367 tasks: 1700.55078125 MB +worker1-1 | 2024/06/04 04:07:55.664 FIG10b: Memory usage without flush after 236870 tasks: 1724.6171875 MB +worker1-1 | 2024/06/04 04:08:25.668 FIG10b: Memory usage without flush after 240530 tasks: 1729.3984375 MB +worker1-1 | 2024/06/04 04:08:55.668 2024/06/04 04:08:55.669 FIG10b: Memory usage without flush after 244174 tasks: 1721.2421875 MB +worker1-1 | 2024/06/04 04:09:25.679 FIG10b: Memory usage without flush after 247469 tasks: 1728.60546875 MB +worker1-1 | 2024/06/04 04:09:55.678 FIG10b: Memory usage without flush after 250949 tasks: 1728.96875 MB +worker1-1 | FIG10b: Memory usage without flush after 254053 tasks: 1734.640625 MB +worker1-1 | 2024/06/04 04:10:55.684 FIG10b: Memory usage without flush after 257225 tasks: 1768.9609375 MB +worker1-1 | 2024/06/04 04:11:25.689 FIG10b: Memory usage without flush after 260317 tasks: 1773.13671875 MB +worker1-1 | 2024/06/04 04:11:55.691 FIG10b: Memory usage without flush after 263734 tasks: 1784.29296875 MB +worker1-1 | 2024/06/04 04:12:25.6922024/06/04 04:12:25.692 FIG10b: Memory usage without flush after 268056 tasks: 1514.75390625 MB +worker1-1 | 2024/06/04 04:12:55.693 FIG10b: Memory usage without flush after 273259 tasks: 1509.0625 MB +worker1-1 | 2024/06/04 04:13:25.701 FIG10b: Memory usage without flush after 278844 tasks: 1508.140625 MB +worker1-1 | 2024/06/04 04:13:55.702 FIG10b: Memory usage without flush after 284273 tasks: 1508.68359375 MB +worker1-1 | 2024/06/04 04:14:25.705 FIG10b: Memory usage without flush after 289699 tasks: 1504.7578125 MB +worker1-1 | 2024/06/04 04:14:55.709 FIG10b: Memory usage without flush after 295409 tasks: 1491.7421875 MB +worker1-1 | 2024/06/04 04:15:25.7092024/06/04 04:15:25.709 FIG10b: Memory usage without flush after 300543 tasks: 1506.33984375 MB +worker1-1 | 2024/06/04 04:15:55.714 FIG10b: Memory usage without flush after 305443 tasks: 1508.61328125 MB +worker1-1 | [worker]2024/06/04 04:16:25.720 FIG10b: Memory usage without flush after 311212 tasks: 1500.80859375 MB +worker1-1 | 2024/06/04 04:16:55.722 FIG10b: Memory usage without flush after 316949 tasks: 1501.8125 MB +worker1-1 | 2024/06/04 04:17:25.727 FIG10b: Memory usage without flush after 322754 tasks: 1497.07421875 MB +worker1-1 | 2024/06/04 04:17:55.729 FIG10b: Memory usage without flush after 328758 tasks: 1498.6484375 MB +worker1-1 | 2024/06/04 04:18:25.732 FIG10b: Memory usage without flush after 334751 tasks: 1496.65234375 MB +worker1-1 | 2024/06/04 04:18:55.737 FIG10b: Memory usage without flush after 340765 tasks: 1499.8125 MB +worker1-1 | 2024/06/04 04:19:25.737 FIG10b: Memory usage without flush after 346433 tasks: 1515.08203125 MB +worker1-1 | 2024/06/04 04:19:55.744 FIG10b: Memory usage without flush after 350800 tasks: 1512.96875 MB +worker1-1 | 2024/06/04 04:20:25.744 FIG10b: Memory usage without flush after 356010 tasks: 1511.6015625 MB +worker1-1 | 2024/06/04 04:20:55.7462024/06/04 04:20:55.746 FIG10b: Memory usage without flush after 361839 tasks: 1513.046875 MB +worker1-1 | 2024/06/04 04:21:25.748 FIG10b: Memory usage without flush after 367734 tasks: 1513.9921875 MB +worker1-1 | 2024/06/04 04:21:55.753 FIG10b: Memory usage without flush after 373440 tasks: 1521.23046875 MB +worker1-1 | 2024/06/04 04:22:25.754 FIG10b: Memory usage without flush after 379209 tasks: 1522.34375 MB +worker1-1 | 2024/06/04 04:22:55.759 [worker] gonna store now2024/06/04 04:22:55.759 FIG10b: Memory usage without flush after 385098 tasks: 1524.828125 MB +worker1-1 | 2024/06/04 04:23:25.763 [worker] 2024/06/04 04:23:25.763 FIG10b: Memory usage without flush after 391063 tasks: 1524.265625 MB +worker1-1 | 2024/06/04 04:23:55.768 2024/06/04 04:23:55.770[worker] FIG10b: Memory usage without flush after 396503 tasks: 1524.72265625 MB +worker1-1 | 2024/06/04 04:24:25.772 FIG10b: Memory usage without flush after 401902 tasks: 1524.765625 MB +worker1-1 | 2024/06/04 04:24:55.773 FIG10b: Memory usage without flush after 406157 tasks: 1514.16796875 MB +worker1-1 | 2024/06/04 04:25:25.776 FIG10b: Memory usage without flush after 411477 tasks: 1508.60546875 MB +worker1-1 | 2024/06/04 04:25:55.780 FIG10b: Memory usage without flush after 417100 tasks: 1505.30859375 MB +worker1-1 | 2024/06/04 04:26:25.782 FIG10b: Memory usage without flush after 422902 tasks: 1510.6328125 MB +worker1-1 | 2024/06/04 04:26:55.787 2024/06/04 04:26:55.787FIG10b: Memory usage without flush after 428513 tasks: 1482.6640625 MB +worker1-1 | 2024/06/04 04:27:25.789 FIG10b: Memory usage without flush after 434246 tasks: 1496.74609375 MB +worker1-1 | 2024/06/04 04:27:55.793 FIG10b: Memory usage without flush after 440186 tasks: 1498.75 MB +worker1-1 | 2024/06/04 04:28:25.794 FIG10b: Memory usage without flush after 446138 tasks: 1508.09375 MB +worker1-1 | FIG10b: Memory usage without flush after 452049 tasks: 1510.578125 MB +worker1-1 | 2024/06/04 04:29:25.797 FIG10b: Memory usage without flush after 457856 tasks: 1517.16015625 MB +worker1-1 | 2024/06/04 04:29:55.803 FIG10b: Memory usage without flush after 463820 tasks: 1510.99609375 MB +worker1-1 | 2024/06/04 04:30:25.804 FIG10b: Memory usage without flush after 469815 tasks: 1522.984375 MB +worker1-1 | 2024/06/04 04:30:55.8062024/06/04 04:30:55.806 FIG10b: Memory usage without flush after 475805 tasks: 1523.7578125 MB +worker1-1 | 2024/06/04 04:31:25.808 FIG10b: Memory usage without flush after 481768 tasks: 1523.3515625 MB +worker1-1 | 2024/06/04 04:31:55.811 FIG10b: Memory usage without flush after 486783 tasks: 1527.33203125 MB +worker1-1 | 2024/06/04 04:32:25.811 FIG10b: Memory usage without flush after 492761 tasks: 1529.6015625 MB +worker1-1 | 2024/06/04 04:32:55.816 2024/06/04 04:32:55.817 FIG10b: Memory usage without flush after 498701 tasks: 1528.40234375 MB +worker1-1 | 2024/06/04 04:33:25.821 FIG10b: Memory usage without flush after 504730 tasks: 1532.078125 MB +worker1-1 | 2024/06/04 04:33:55.824 FIG10b: Memory usage without flush after 510617 tasks: 1531.3671875 MB +worker1-1 | [worker] 2024/06/04 04:34:25.829 FIG10b: Memory usage without flush after 516592 tasks: 1520.625 MB +worker1-1 | 2024/06/04 04:34:55.838 FIG10b: Memory usage without flush after 522535 tasks: 1514.1953125 MB +worker1-1 | 2024/06/04 04:35:25.840 FIG10b: Memory usage without flush after 528541 tasks: 1522.14453125 MB +worker1-1 | FIG10b: Memory usage without flush after 534348 tasks: 1526.203125 MB +worker1-1 | 2024/06/04 04:39:34.984 FIG10b: Memory usage without flush after 538865 tasks: 1527.59375 MB +worker1-1 | 2024/06/04 04:40:04.9862024/06/04 04:40:04.986 FIG10b: Memory usage without flush after 543675 tasks: 1524.3828125 MB +worker1-1 | 2024/06/04 04:40:34.991 FIG10b: Memory usage without flush after 548015 tasks: 1540.796875 MB +worker1-1 | 2024/06/04 04:41:04.992 FIG10b: Memory usage without flush after 552634 tasks: 1538.796875 MB +worker1-1 | 2024/06/04 04:41:34.993 FIG10b: Memory usage without flush after 558653 tasks: 1545.36328125 MB +worker1-1 | 2024/06/04 04:42:05.000 FIG10b: Memory usage without flush after 564666 tasks: 1542.1640625 MB +worker1-1 | 2024/06/04 04:42:35.004 FIG10b: Memory usage without flush after 570630 tasks: 1532.55078125 MB +worker1-1 | 2024/06/04 04:43:05.009 FIG10b: Memory usage without flush after 576229 tasks: 1533.05859375 MB +worker1-1 | 2024/06/04 04:43:35.016 FIG10b: Memory usage without flush after 581646 tasks: 1535.55859375 MB +worker1-1 | 2024/06/04 04:44:05.020 FIG10b: Memory usage without flush after 587138 tasks: 1535.04296875 MB +worker1-1 | 2024/06/04 04:44:35.024 FIG10b: Memory usage without flush after 593129 tasks: 1536.75 MB +worker1-1 | gonna store now2024/06/04 04:45:05.028 FIG10b: Memory usage without flush after 599081 tasks: 1540.94140625 MB +worker1-1 | 2024/06/04 04:45:35.030 FIG10b: Memory usage without flush after 604470 tasks: 1541.0234375 MB +worker1-1 | 2024/06/04 04:46:05.030 FIG10b: Memory usage without flush after 608515 tasks: 1529.6484375 MB +worker1-1 | FIG10b: Memory usage without flush after 612468 tasks: 1543.72265625 MB +worker1-1 | 2024/06/04 04:47:05.047 FIG10b: Memory usage without flush after 617421 tasks: 1543.23828125 MB +worker1-1 | 2024/06/04 04:47:35.050 FIG10b: Memory usage without flush after 621885 tasks: 1544.4140625 MB +worker1-1 | 2024/06/04 04:48:05.052 FIG10b: Memory usage without flush after 627460 tasks: 1534.34375 MB +worker1-1 | 2024/06/04 04:48:35.0542024/06/04 04:48:35.054 FIG10b: Memory usage without flush after 633249 tasks: 1529.2890625 MB +worker1-1 | 2024/06/04 04:49:05.056 FIG10b: Memory usage without flush after 639038 tasks: 1531.3984375 MB +worker1-1 | 2024/06/04 04:49:35.060 FIG10b: Memory usage without flush after 644336 tasks: 1532.1796875 MB +worker1-1 | 2024/06/04 04:50:05.064 FIG10b: Memory usage without flush after 649827 tasks: 1537.39453125 MB +worker1-1 | 2024/06/04 04:50:35.068 FIG10b: Memory usage without flush after 655347 tasks: 1534.8828125 MB +worker1-1 | 2024/06/04 04:51:05.071 FIG10b: Memory usage without flush after 660475 tasks: 1534.60546875 MB +worker1-1 | 2024/06/04 04:51:35.0732024/06/04 04:51:35.074 FIG10b: Memory usage without flush after 665921 tasks: 1532.44921875 MB diff --git a/analysis/output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv b/analysis/output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv new file mode 100644 index 0000000..613afdf --- /dev/null +++ b/analysis/output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv @@ -0,0 +1,266 @@ +Timestamp,Memory Usage (MB) +2024-06-04 03:33:55.734000,1012.97265625 +2024-06-04 03:33:56.736000,1013.26171875 +2024-06-04 03:33:58.747000,1017.6484375 +2024-06-04 03:33:59.759000,1021.41796875 +2024-06-04 03:34:02.777000,1022.78125 +2024-06-04 03:34:03.785000,1026.54296875 +2024-06-04 03:34:04.793000,1025.8046875 +2024-06-04 03:34:06.800000,1026.49609375 +2024-06-04 03:34:09.812000,1025.4765625 +2024-06-04 03:34:10.816000,1025.19140625 +2024-06-04 03:34:12.826000,1025.671875 +2024-06-04 03:34:13.827000,1025.4296875 +2024-06-04 03:34:14.846000,1025.375 +2024-06-04 03:34:16.855000,1026.3203125 +2024-06-04 03:34:18.872000,1030.8359375 +2024-06-04 03:34:19.881000,1035.09765625 +2024-06-04 03:34:21.899000,1035.06640625 +2024-06-04 03:34:22.904000,1034.64453125 +2024-06-04 03:34:24.915000,1034.56640625 +2024-06-04 03:34:26.925000,1037.55859375 +2024-06-04 03:34:27.934000,1038.2890625 +2024-06-04 03:34:28.955000,1039.37109375 +2024-06-04 03:34:30.965000,1039.109375 +2024-06-04 03:34:37.010000,1046.2890625 +2024-06-04 03:34:42.052000,1049.14453125 +2024-06-04 03:34:43.052000,1050.30078125 +2024-06-04 03:34:44.061000,1050.24609375 +2024-06-04 03:34:46.073000,1051.2734375 +2024-06-04 03:34:47.079000,1051.11328125 +2024-06-04 03:34:48.081000,1057.07421875 +2024-06-04 03:34:51.100000,1063.61328125 +2024-06-04 03:34:53.106000,1065.19140625 +2024-06-04 03:34:54.110000,1064.7109375 +2024-06-04 03:34:56.118000,1066.42578125 +2024-06-04 03:34:58.134000,1068.85546875 +2024-06-04 03:34:59.137000,1068.48046875 +2024-06-04 03:35:00.142000,1069.43359375 +2024-06-04 03:35:02.158000,1070.07421875 +2024-06-04 03:35:03.166000,1069.4140625 +2024-06-04 03:35:04.167000,1068.91015625 +2024-06-04 03:35:05.171000,1070.0625 +2024-06-04 03:35:07.180000,1070.69921875 +2024-06-04 03:35:08.184000,1071.61328125 +2024-06-04 03:35:11.193000,1071.91796875 +2024-06-04 03:35:13.203000,1073.85546875 +2024-06-04 03:35:15.211000,1075.859375 +2024-06-04 03:35:17.217000,1076.56640625 +2024-06-04 03:35:20.239000,1069.203125 +2024-06-04 03:35:22.254000,1069.48828125 +2024-06-04 03:35:23.254000,1069.06640625 +2024-06-04 03:35:24.260000,1069.4765625 +2024-06-04 03:35:27.272000,1069.46875 +2024-06-04 03:35:30.280000,1070.6875 +2024-06-04 03:35:32.288000,1070.51171875 +2024-06-04 03:35:36.298000,1070.12109375 +2024-06-04 03:35:38.301000,1070.30078125 +2024-06-04 03:35:39.304000,1070.375 +2024-06-04 03:35:42.316000,1075.08984375 +2024-06-04 03:35:44.319000,1076.2265625 +2024-06-04 03:35:45.319000,1076.9453125 +2024-06-04 03:35:46.327000,1077.54296875 +2024-06-04 03:35:49.367000,1095.51171875 +2024-06-04 03:35:50.371000,1096.14453125 +2024-06-04 03:35:53.378000,1091.921875 +2024-06-04 03:35:54.382000,1093.2421875 +2024-06-04 03:35:55.387000,1094.23828125 +2024-06-04 03:35:56.389000,1094.46484375 +2024-06-04 03:35:57.396000,1095.83984375 +2024-06-04 03:35:58.398000,1096.02734375 +2024-06-04 03:35:59.406000,1095.27734375 +2024-06-04 03:36:01.410000,1095.453125 +2024-06-04 03:36:02.417000,1096.1484375 +2024-06-04 03:36:04.434000,1096.8203125 +2024-06-04 03:36:05.444000,1097.08203125 +2024-06-04 03:36:06.502000,1097.53125 +2024-06-04 03:36:07.511000,1097.96875 +2024-06-04 03:36:08.515000,1097.9609375 +2024-06-04 03:36:10.529000,1098.609375 +2024-06-04 03:36:11.532000,1098.86328125 +2024-06-04 03:36:12.546000,1098.69140625 +2024-06-04 03:36:13.551000,1099.46875 +2024-06-04 03:36:14.557000,1099.46484375 +2024-06-04 03:36:20.600000,1098.4921875 +2024-06-04 03:36:24.615000,1098.44921875 +2024-06-04 03:36:25.619000,1098.3203125 +2024-06-04 03:36:29.644000,1098.828125 +2024-06-04 03:36:30.649000,1099.31640625 +2024-06-04 03:36:31.657000,1099.48046875 +2024-06-04 03:36:34.665000,1099.734375 +2024-06-04 03:36:38.681000,1101.14453125 +2024-06-04 03:36:39.684000,1101.84765625 +2024-06-04 03:36:41.696000,1102.9375 +2024-06-04 03:36:43.712000,1104.55859375 +2024-06-04 03:36:45.713000,1105.37890625 +2024-06-04 03:36:46.721000,1105.16015625 +2024-06-04 03:36:47.735000,1105.28125 +2024-06-04 03:36:49.753000,1115.32421875 +2024-06-04 03:36:50.765000,1116.33984375 +2024-06-04 03:36:52.782000,1116.1484375 +2024-06-04 03:36:57.807000,1118.36328125 +2024-06-04 03:36:59.815000,1119.8828125 +2024-06-04 03:37:00.827000,1120.515625 +2024-06-04 03:37:01.830000,1120.85546875 +2024-06-04 03:37:03.844000,1121.13671875 +2024-06-04 03:37:04.845000,1121.2421875 +2024-06-04 03:37:05.845000,1121.77734375 +2024-06-04 03:37:06.851000,1121.4296875 +2024-06-04 03:37:07.855000,1121.3203125 +2024-06-04 03:37:08.855000,1122.23046875 +2024-06-04 03:37:10.865000,1122.046875 +2024-06-04 03:37:12.868000,1122.72265625 +2024-06-04 03:37:13.879000,1122.13671875 +2024-06-04 03:37:14.888000,1123.8359375 +2024-06-04 03:37:15.890000,1123.9375 +2024-06-04 03:37:17.903000,1123.37109375 +2024-06-04 03:37:18.911000,1123.01953125 +2024-06-04 03:37:20.918000,1123.0546875 +2024-06-04 03:37:21.922000,1122.96875 +2024-06-04 03:37:22.927000,1124.24609375 +2024-06-04 03:37:24.941000,1124.3671875 +2024-06-04 03:37:26.951000,1124.68359375 +2024-06-04 03:37:28.959000,1124.66015625 +2024-06-04 03:37:29.968000,1124.80078125 +2024-06-04 03:37:30.968000,1123.5859375 +2024-06-04 03:37:31.970000,1125.1875 +2024-06-04 03:37:32.978000,1125.23828125 +2024-06-04 03:37:36.003000,1125.3203125 +2024-06-04 03:37:37.010000,1126.4296875 +2024-06-04 03:37:39.020000,1126.13671875 +2024-06-04 03:37:40.029000,1126.3515625 +2024-06-04 03:37:41.029000,1126.3203125 +2024-06-04 03:37:42.078000,1126.84375 +2024-06-04 03:37:43.086000,1127.20703125 +2024-06-04 03:37:44.089000,1127.4453125 +2024-06-04 03:37:45.093000,1127.94921875 +2024-06-04 03:37:48.102000,1130.19921875 +2024-06-04 03:37:49.106000,1130.30859375 +2024-06-04 03:37:50.111000,1135.0 +2024-06-04 03:37:51.118000,1140.1171875 +2024-06-04 03:37:52.123000,1140.51953125 +2024-06-04 03:37:54.135000,1140.375 +2024-06-04 03:37:55.138000,1141.25390625 +2024-06-04 03:37:56.143000,1139.09375 +2024-06-04 03:37:57.149000,1140.0078125 +2024-06-04 03:37:58.152000,1144.21875 +2024-06-04 03:37:59.155000,1144.8671875 +2024-06-04 03:38:00.164000,1145.25390625 +2024-06-04 03:38:01.168000,1145.1640625 +2024-06-04 03:38:04.191000,1146.23828125 +2024-06-04 03:38:05.198000,1146.875 +2024-06-04 03:38:07.202000,1147.734375 +2024-06-04 03:38:09.212000,1147.59765625 +2024-06-04 03:38:10.216000,1147.1953125 +2024-06-04 03:38:16.255000,1150.5625 +2024-06-04 03:38:17.260000,1148.6796875 +2024-06-04 03:38:19.285000,1146.625 +2024-06-04 03:38:20.289000,1146.55078125 +2024-06-04 03:38:24.309000,1147.875 +2024-06-04 03:38:25.311000,1147.71875 +2024-06-04 03:38:26.317000,1147.58984375 +2024-06-04 03:38:29.341000,1147.875 +2024-06-04 03:38:32.352000,1148.92578125 +2024-06-04 03:38:34.360000,1153.609375 +2024-06-04 03:38:36.369000,1153.8046875 +2024-06-04 03:38:38.383000,1153.578125 +2024-06-04 03:38:39.389000,1154.625 +2024-06-04 03:38:41.394000,1154.28125 +2024-06-04 03:38:46.416000,1156.66015625 +2024-06-04 03:38:49.429000,1166.3828125 +2024-06-04 03:38:51.439000,1166.8671875 +2024-06-04 03:38:52.438000,1167.05859375 +2024-06-04 03:38:53.446000,1160.99609375 +2024-06-04 03:39:25.455000,1168.05078125 +2024-06-04 03:40:55.469000,1239.30859375 +2024-06-04 03:41:55.479000,1259.33203125 +2024-06-04 03:42:55.488000,1283.40625 +2024-06-04 03:43:25.496000,1284.46875 +2024-06-04 03:46:25.521000,1366.875 +2024-06-04 03:47:25.524000,1382.34375 +2024-06-04 03:47:55.534000,1372.3203125 +2024-06-04 03:49:25.541000,1444.30078125 +2024-06-04 03:50:25.546000,1449.98828125 +2024-06-04 03:50:55.551000,1466.87890625 +2024-06-04 03:51:55.557000,1483.0234375 +2024-06-04 03:52:55.563000,1513.37890625 +2024-06-04 03:53:25.566000,1512.1796875 +2024-06-04 03:54:25.576000,1515.0546875 +2024-06-04 03:54:55.578000,1530.1796875 +2024-06-04 03:55:55.593000,1552.4609375 +2024-06-04 03:56:55.602000,1554.2890625 +2024-06-04 03:57:25.603000,1551.7109375 +2024-06-04 03:57:55.605000,1572.1171875 +2024-06-04 03:58:25.605000,1562.6640625 +2024-06-04 03:58:55.606000,1589.87109375 +2024-06-04 03:59:25.609000,1593.6640625 +2024-06-04 04:01:25.623000,1618.2421875 +2024-06-04 04:01:55.630000,1628.71484375 +2024-06-04 04:02:25.632000,1638.32421875 +2024-06-04 04:02:55.637000,1632.52734375 +2024-06-04 04:03:25.640000,1623.203125 +2024-06-04 04:04:55.648000,1695.09375 +2024-06-04 04:05:55.653000,1734.11328125 +2024-06-04 04:06:55.658000,1756.484375 +2024-06-04 04:07:25.660000,1700.55078125 +2024-06-04 04:07:55.664000,1724.6171875 +2024-06-04 04:08:25.668000,1729.3984375 +2024-06-04 04:09:25.679000,1728.60546875 +2024-06-04 04:09:55.678000,1728.96875 +2024-06-04 04:10:55.684000,1768.9609375 +2024-06-04 04:11:25.689000,1773.13671875 +2024-06-04 04:11:55.691000,1784.29296875 +2024-06-04 04:12:55.693000,1509.0625 +2024-06-04 04:13:25.701000,1508.140625 +2024-06-04 04:13:55.702000,1508.68359375 +2024-06-04 04:14:25.705000,1504.7578125 +2024-06-04 04:14:55.709000,1491.7421875 +2024-06-04 04:15:55.714000,1508.61328125 +2024-06-04 04:16:55.722000,1501.8125 +2024-06-04 04:17:25.727000,1497.07421875 +2024-06-04 04:17:55.729000,1498.6484375 +2024-06-04 04:18:25.732000,1496.65234375 +2024-06-04 04:18:55.737000,1499.8125 +2024-06-04 04:19:25.737000,1515.08203125 +2024-06-04 04:19:55.744000,1512.96875 +2024-06-04 04:20:25.744000,1511.6015625 +2024-06-04 04:21:25.748000,1513.9921875 +2024-06-04 04:21:55.753000,1521.23046875 +2024-06-04 04:22:25.754000,1522.34375 +2024-06-04 04:24:25.772000,1524.765625 +2024-06-04 04:24:55.773000,1514.16796875 +2024-06-04 04:25:25.776000,1508.60546875 +2024-06-04 04:25:55.780000,1505.30859375 +2024-06-04 04:26:25.782000,1510.6328125 +2024-06-04 04:27:25.789000,1496.74609375 +2024-06-04 04:27:55.793000,1498.75 +2024-06-04 04:28:25.794000,1508.09375 +2024-06-04 04:29:25.797000,1517.16015625 +2024-06-04 04:29:55.803000,1510.99609375 +2024-06-04 04:30:25.804000,1522.984375 +2024-06-04 04:31:25.808000,1523.3515625 +2024-06-04 04:31:55.811000,1527.33203125 +2024-06-04 04:32:25.811000,1529.6015625 +2024-06-04 04:33:25.821000,1532.078125 +2024-06-04 04:33:55.824000,1531.3671875 +2024-06-04 04:34:55.838000,1514.1953125 +2024-06-04 04:35:25.840000,1522.14453125 +2024-06-04 04:39:34.984000,1527.59375 +2024-06-04 04:40:34.991000,1540.796875 +2024-06-04 04:41:04.992000,1538.796875 +2024-06-04 04:41:34.993000,1545.36328125 +2024-06-04 04:42:35.004000,1532.55078125 +2024-06-04 04:43:05.009000,1533.05859375 +2024-06-04 04:43:35.016000,1535.55859375 +2024-06-04 04:44:05.020000,1535.04296875 +2024-06-04 04:44:35.024000,1536.75 +2024-06-04 04:45:35.030000,1541.0234375 +2024-06-04 04:46:05.030000,1529.6484375 +2024-06-04 04:47:05.047000,1543.23828125 +2024-06-04 04:47:35.050000,1544.4140625 +2024-06-04 04:48:05.052000,1534.34375 +2024-06-04 04:49:05.056000,1531.3984375 +2024-06-04 04:49:35.060000,1532.1796875 +2024-06-04 04:50:05.064000,1537.39453125 +2024-06-04 04:50:35.068000,1534.8828125 +2024-06-04 04:51:05.071000,1534.60546875 diff --git a/analysis/output_10b_run3_40s_flush_2gb_memory_limit.txt b/analysis/output_10b_run3_40s_flush_2gb_memory_limit.txt new file mode 100644 index 0000000..cd8d1a3 --- /dev/null +++ b/analysis/output_10b_run3_40s_flush_2gb_memory_limit.txt @@ -0,0 +1,409 @@ +worker1-1 | 2024/06/04 05:02:07.015 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 05:02:07.040 FIG10b: Starting tasks submission... +worker1-1 | 2024/06/04 05:02:08.045 FIG10b: Memory usage without flush after 99 tasks: 1046.09765625 MB +worker1-1 | 2024/06/04 05:02:09.053 FIG10b: Memory usage without flush after 190 tasks: 1047.3203125 MB +worker1-1 | 2024/06/04 05:02:10.057 FIG10b: Memory usage without flush after 293 tasks: 1038.96484375 MB +worker1-1 | 2024/06/04 05:02:11.060 FIG10b: Memory usage without flush after 409 tasks: 1044.33203125 MB +worker1-1 | 2024/06/04 05:02:12.069 FIG10b: Memory usage without flush after 526 tasks: 1049.59375 MB +worker1-1 | 2024/06/04 05:02:13.069 FIG10b: Memory usage without flush after 639 tasks: 1055.1796875 MB +worker1-1 | 2024/06/04 05:02:14.072 2024/06/04 05:02:14.074[worker] FIG10b: Memory usage without flush after 753 tasks: 1056.06640625 MB +worker1-1 | 2024/06/04 05:02:15.084 FIG10b: Memory usage without flush after 865 tasks: 1055.5078125 MB +worker1-1 | 2024/06/04 05:02:16.088 FIG10b: Memory usage without flush after 990 tasks: 1055.58203125 MB +worker1-1 | 2024/06/04 05:02:17.094[worker] FIG10b: Memory usage without flush after 1077 tasks: 1056.84375 MB +worker1-1 | 2024/06/04 05:02:18.099 FIG10b: Memory usage without flush after 1170 tasks: 1057.578125 MB +worker1-1 | 2024/06/04 05:02:19.109 FIG10b: Memory usage without flush after 1279 tasks: 1057.734375 MB +worker1-1 | 2024/06/04 05:02:20.128 FIG10b: Memory usage without flush after 1372 tasks: 1059.52734375 MB +worker1-1 | 2024/06/04 05:02:21.134 FIG10b: Memory usage without flush after 1459 tasks: 1060.06640625 MB +worker1-1 | [worker] stored... [worker]2024/06/04 05:02:22.146 FIG10b: Memory usage without flush after 1537 tasks: 1060.64453125 MB +worker1-1 | 2024/06/04 05:02:23.153 FIG10b: Memory usage without flush after 1630 tasks: 1061.59375 MB +worker1-1 | 2024/06/04 05:02:24.1552024/06/04 05:02:24.155 FIG10b: Memory usage without flush after 1750 tasks: 1064.69140625 MB +worker1-1 | 2024/06/04 05:02:25.161 FIG10b: Memory usage without flush after 1826 tasks: 1065.05078125 MB +worker1-1 | 2024/06/04 05:02:26.171 FIG10b: Memory usage without flush after 1875 tasks: 1065.30078125 MB +worker1-1 | 2024/06/04 05:02:27.474 FIG10b: Memory usage without flush after 1898 tasks: 1065.41015625 MB +worker1-1 | 2024/06/04 05:02:28.481 FIG10b: Memory usage without flush after 1927 tasks: 1069.33984375 MB +worker1-1 | 2024/06/04 05:02:29.515 FIG10b: Memory usage without flush after 1980 tasks: 1069.31640625 MB +worker1-1 | 2024/06/04 05:02:30.520 FIG10b: Memory usage without flush after 2041 tasks: 1073.29296875 MB +worker1-1 | 2024/06/04 05:02:31.5212024/06/04 05:02:31.521 FIG10b: Memory usage without flush after 2110 tasks: 1073.7578125 MB +worker1-1 | 2024/06/04 05:02:32.522 FIG10b: Memory usage without flush after 2198 tasks: 1074.93359375 MB +worker1-1 | 2024/06/04 05:02:33.5282024/06/04 05:02:33.528 FIG10b: Memory usage without flush after 2305 tasks: 1075.1875 MB +worker1-1 | 2024/06/04 05:02:34.5412024/06/04 05:02:34.541 FIG10b: Memory usage without flush after 2407 tasks: 1075.49609375 MB +worker1-1 | 2024/06/04 05:02:35.543 FIG10b: Memory usage without flush after 2488 tasks: 1074.82421875 MB +worker1-1 | 2024/06/04 05:02:36.5432024/06/04 05:02:36.543 FIG10b: Memory usage without flush after 2588 tasks: 1076.00390625 MB +worker1-1 | 2024/06/04 05:02:37.547 [worker]2024/06/04 05:02:37.548 FIG10b: Memory usage without flush after 2697 tasks: 1076.19140625 MB +worker1-1 | 2024/06/04 05:02:38.554 FIG10b: Memory usage without flush after 2803 tasks: 1076.67578125 MB +worker1-1 | 2024/06/04 05:02:39.556 FIG10b: Memory usage without flush after 2894 tasks: 1076.46875 MB +worker1-1 | 2024/06/04 05:02:40.638 FIG10b: Memory usage without flush after 2995 tasks: 1076.50390625 MB +worker1-1 | 2024/06/04 05:02:41.666 FIG10b: Memory usage without flush after 3080 tasks: 1086.84765625 MB +worker1-1 | 2024/06/04 05:02:42.665 [worker]2024/06/04 05:02:42.666 FIG10b: Memory usage without flush after 3178 tasks: 1085.69921875 MB +worker1-1 | 2024/06/04 05:02:43.666 FIG10b: Memory usage without flush after 3285 tasks: 1086.4140625 MB +worker1-1 | 2024/06/04 05:02:44.674 FIG10b: Memory usage without flush after 3403 tasks: 1088.109375 MB +worker1-1 | FIG10b: Memory usage without flush after 3527 tasks: 1089.0546875 MB +worker1-1 | 2024/06/04 05:02:46.683 2024/06/04 05:02:46.6842024/06/04 05:02:46.685 FIG10b: Memory usage without flush after 3649 tasks: 1089.08984375 MB +worker1-1 | [worker]stored...2024/06/04 05:02:47.688 FIG10b: Memory usage without flush after 3754 tasks: 1089.1015625 MB +worker1-1 | 2024/06/04 05:02:48.696 FIG10b: Memory usage without flush after 3872 tasks: 1088.625 MB +worker1-1 | 2024/06/04 05:02:49.701 [worker] 2024/06/04 05:02:49.703in Run() rn FIG10b: Memory usage without flush after 3978 tasks: 1089.31640625 MB +worker1-1 | 2024/06/04 05:02:50.7062024/06/04 05:02:50.706 FIG10b: Memory usage without flush after 4084 tasks: 1091.57421875 MB +worker1-1 | 2024/06/04 05:02:51.7132024/06/04 05:02:51.714 2024/06/04 05:02:51.714 FIG10b: Memory usage without flush after 4204 tasks: 1091.6328125 MB +worker1-1 | 2024/06/04 05:02:52.716 FIG10b: Memory usage without flush after 4326 tasks: 1091.8828125 MB +worker1-1 | 2024/06/04 05:02:53.7242024/06/04 05:02:53.725 FIG10b: Memory usage without flush after 4431 tasks: 1093.078125 MB +worker1-1 | 2024/06/04 05:02:54.736 FIG10b: Memory usage without flush after 4542 tasks: 1093.75 MB +worker1-1 | 2024/06/04 05:02:55.7402024/06/04 05:02:55.740 FIG10b: Memory usage without flush after 4644 tasks: 1093.98046875 MB +worker1-1 | [worker][worker]2024/06/04 05:02:56.748 FIG10b: Memory usage without flush after 4748 tasks: 1093.734375 MB +worker1-1 | 2024/06/04 05:02:57.758 2024/06/04 05:02:57.759 FIG10b: Memory usage without flush after 4864 tasks: 1093.1484375 MB +worker1-1 | 2024/06/04 05:02:58.760 FIG10b: Memory usage without flush after 4953 tasks: 1093.91015625 MB +worker1-1 | 2024/06/04 05:02:59.763 FIG10b: Memory usage without flush after 5056 tasks: 1093.84765625 MB +worker1-1 | [worker] 2024/06/04 05:03:00.771 FIG10b: Memory usage without flush after 5156 tasks: 1094.3125 MB +worker1-1 | 2024/06/04 05:03:01.777 FIG10b: Memory usage without flush after 5251 tasks: 1094.30859375 MB +worker1-1 | FIG10b: Memory usage without flush after 5359 tasks: 1094.29296875 MB +worker1-1 | 2024/06/04 05:03:03.784 FIG10b: Memory usage without flush after 5474 tasks: 1094.41015625 MB +worker1-1 | FIG10b: Memory usage without flush after 5562 tasks: 1094.703125 MB +worker1-1 | FIG10b: Memory usage without flush after 5656 tasks: 1094.7265625 MB +worker1-1 | 2024/06/04 05:03:06.818 FIG10b: Memory usage without flush after 5745 tasks: 1094.76171875 MB +worker1-1 | 2024/06/04 05:03:07.8252024/06/04 05:03:07.825 FIG10b: Memory usage without flush after 5837 tasks: 1095.03515625 MB +worker1-1 | 2024/06/04 05:03:08.8392024/06/04 05:03:08.840 FIG10b: Memory usage without flush after 5936 tasks: 1095.2578125 MB +worker1-1 | 2024/06/04 05:03:09.845 FIG10b: Memory usage without flush after 6051 tasks: 1094.91796875 MB +worker1-1 | 2024/06/04 05:03:10.847[worker]2024/06/04 05:03:10.847 2024/06/04 05:03:10.848 FIG10b: Memory usage without flush after 6170 tasks: 1095.015625 MB +worker1-1 | 2024/06/04 05:03:11.848 FIG10b: Memory usage without flush after 6276 tasks: 1095.92578125 MB +worker1-1 | 2024/06/04 05:03:12.860 FIG10b: Memory usage without flush after 6383 tasks: 1095.65234375 MB +worker1-1 | FIG10b: Memory usage without flush after 6506 tasks: 1096.24609375 MB +worker1-1 | 2024/06/04 05:03:14.872 FIG10b: Memory usage without flush after 6633 tasks: 1097.30078125 MB +worker1-1 | 2024/06/04 05:03:16.0052024/06/04 05:03:16.004 FIG10b: Memory usage without flush after 6730 tasks: 1097.03515625 MB[worker] +worker1-1 | 2024/06/04 05:03:17.012 FIG10b: Memory usage without flush after 6837 tasks: 1096.73046875 MB +worker1-1 | 2024/06/04 05:03:18.0152024/06/04 05:03:18.015 FIG10b: Memory usage without flush after 6946 tasks: 1097.6484375 MB +worker1-1 | 2024/06/04 05:03:19.022 FIG10b: Memory usage without flush after 7071 tasks: 1097.453125 MB +worker1-1 | 2024/06/04 05:03:20.023 FIG10b: Memory usage without flush after 7183 tasks: 1097.36328125 MB +worker1-1 | 2024/06/04 05:03:21.0282024/06/04 05:03:21.028 FIG10b: Memory usage without flush after 7279 tasks: 1098.08203125 MB +worker1-1 | [worker]2024/06/04 05:03:22.032 FIG10b: Memory usage without flush after 7411 tasks: 1117.26171875 MB +worker1-1 | 2024/06/04 05:03:23.0332024/06/04 05:03:23.033 FIG10b: Memory usage without flush after 7514 tasks: 1117.23046875 MB +worker1-1 | 2024/06/04 05:03:24.034 FIG10b: Memory usage without flush after 7632 tasks: 1117.375 MB +worker1-1 | 2024/06/04 05:03:25.037 2024/06/04 05:03:25.038 FIG10b: Memory usage without flush after 7750 tasks: 1117.7421875 MB +worker1-1 | 2024/06/04 05:03:26.043[worker]2024/06/04 05:03:26.044 FIG10b: Memory usage without flush after 7862 tasks: 1118.1484375 MB +worker1-1 | 2024/06/04 05:03:27.047 2024/06/04 05:03:27.047 FIG10b: Memory usage without flush after 7975 tasks: 1118.29296875 MB +worker1-1 | 2024/06/04 05:03:28.0492024/06/04 05:03:28.049 FIG10b: Memory usage without flush after 8089 tasks: 1119.4453125 MB +worker1-1 | 2024/06/04 05:03:29.054 [worker]2024/06/04 05:03:29.055 FIG10b: Memory usage without flush after 8201 tasks: 1120.78515625 MB +worker1-1 | 2024/06/04 05:03:30.060 FIG10b: Memory usage without flush after 8325 tasks: 1121.453125 MB +worker1-1 | 2024/06/04 05:03:31.064 2024/06/04 05:03:31.065 FIG10b: Memory usage without flush after 8444 tasks: 1122.74609375 MB +worker1-1 | 2024/06/04 05:03:32.067 FIG10b: Memory usage without flush after 8566 tasks: 1122.484375 MB +worker1-1 | 2024/06/04 05:03:33.071 FIG10b: Memory usage without flush after 8689 tasks: 1122.0546875 MB +worker1-1 | 2024/06/04 05:03:34.074 FIG10b: Memory usage without flush after 8801 tasks: 1122.00390625 MB +worker1-1 | 2024/06/04 05:03:35.076 FIG10b: Memory usage without flush after 8906 tasks: 1121.9609375 MB +worker1-1 | 2024/06/04 05:03:36.080 FIG10b: Memory usage without flush after 9026 tasks: 1121.7421875 MB +worker1-1 | 2024/06/04 05:03:37.149 FIG10b: Memory usage without flush after 9151 tasks: 1121.2734375 MB +worker1-1 | 2024/06/04 05:03:38.155 FIG10b: Memory usage without flush after 9280 tasks: 1121.265625 MB +worker1-1 | 2024/06/04 05:03:39.157 FIG10b: Memory usage without flush after 9412 tasks: 1122.70703125 MB +worker1-1 | 2024/06/04 05:03:40.157 FIG10b: Memory usage without flush after 9539 tasks: 1123.43359375 MB +worker1-1 | 2024/06/04 05:03:41.160 FIG10b: Memory usage without flush after 9663 tasks: 1123.70703125 MB +worker1-1 | 2024/06/04 05:03:42.163 [worker]2024/06/04 05:03:42.1642024/06/04 05:03:42.164 FIG10b: Memory usage without flush after 9792 tasks: 1124.19140625 MB +worker1-1 | 2024/06/04 05:03:43.1652024/06/04 05:03:43.165 FIG10b: Memory usage without flush after 9907 tasks: 1124.6171875 MB +worker1-1 | 2024/06/04 05:03:44.166 FIG10b: Memory usage without flush after 10010 tasks: 1124.14453125 MB +worker1-1 | 2024/06/04 05:03:45.178 FIG10b: Memory usage without flush after 10107 tasks: 1124.15234375 MB +worker1-1 | 2024/06/04 05:03:46.259 FIG10b: Memory usage without flush after 10209 tasks: 1123.47265625 MB +worker1-1 | 2024/06/04 05:03:47.2602024/06/04 05:03:47.261 FIG10b: Memory usage without flush after 10313 tasks: 1124.1796875 MB +worker1-1 | 2024/06/04 05:03:48.2692024/06/04 05:03:48.269 FIG10b: Memory usage without flush after 10422 tasks: 1124.65234375 MB +worker1-1 | 2024/06/04 05:03:49.2742024/06/04 05:03:49.275 FIG10b: Memory usage without flush after 10532 tasks: 1124.453125 MB +worker1-1 | 2024/06/04 05:03:50.276 FIG10b: Memory usage without flush after 10644 tasks: 1124.7265625 MB +worker1-1 | [worker]2024/06/04 05:03:51.2762024/06/04 05:03:51.277 FIG10b: Memory usage without flush after 10752 tasks: 1124.53125 MB +worker1-1 | Executed!2024/06/04 05:03:52.283 FIG10b: Memory usage without flush after 10852 tasks: 1124.94140625 MB +worker1-1 | [worker]2024/06/04 05:03:53.285 FIG10b: Memory usage without flush after 10968 tasks: 1125.73046875 MB +worker1-1 | 2024/06/04 05:03:54.2862024/06/04 05:03:54.286 FIG10b: Memory usage without flush after 11067 tasks: 1125.9921875 MB +worker1-1 | 2024/06/04 05:03:55.288 FIG10b: Memory usage without flush after 11169 tasks: 1125.7421875 MB +worker1-1 | 2024/06/04 05:03:56.298 FIG10b: Memory usage without flush after 11253 tasks: 1125.046875 MB +worker1-1 | 2024/06/04 05:03:57.299 FIG10b: Memory usage without flush after 11304 tasks: 1125.49609375 MB +worker1-1 | 2024/06/04 05:03:58.322 [worker]2024/06/04 05:03:58.328 FIG10b: Memory usage without flush after 11363 tasks: 1125.99609375 MB +worker1-1 | FIG10b: Memory usage without flush after 11426 tasks: 1124.56640625 MB +worker1-1 | [worker]2024/06/04 05:04:00.342 FIG10b: Memory usage without flush after 11546 tasks: 1124.44921875 MB +worker1-1 | 2024/06/04 05:04:01.344 FIG10b: Memory usage without flush after 11652 tasks: 1126.3515625 MB +worker1-1 | 2024/06/04 05:04:02.353 FIG10b: Memory usage without flush after 11767 tasks: 1135.16796875 MB +worker1-1 | 2024/06/04 05:04:03.373 FIG10b: Memory usage without flush after 11871 tasks: 1135.34765625 MB +worker1-1 | 2024/06/04 05:04:04.377 FIG10b: Memory usage without flush after 11965 tasks: 1135.10546875 MB +worker1-1 | 2024/06/04 05:04:05.3772024/06/04 05:04:05.377 FIG10b: Memory usage without flush after 12078 tasks: 1135.76171875 MB +worker1-1 | 2024/06/04 05:04:06.3822024/06/04 05:04:06.383 FIG10b: Memory usage without flush after 12183 tasks: 1135.49609375 MB +worker1-1 | [worker] 2024/06/04 05:04:07.390 FIG10b: Memory usage without flush after 12289 tasks: 1135.65234375 MB +worker1-1 | 2024/06/04 05:04:08.396 FIG10b: Memory usage without flush after 12388 tasks: 1135.859375 MB +worker1-1 | 2024/06/04 05:04:09.403 FIG10b: Memory usage without flush after 12471 tasks: 1135.97265625 MB2024/06/04 05:04:09.403 +worker1-1 | 2024/06/04 05:04:10.414 FIG10b: Memory usage without flush after 12574 tasks: 1135.953125 MB +worker1-1 | 2024/06/04 05:04:11.427 FIG10b: Memory usage without flush after 12665 tasks: 1135.75 MB +worker1-1 | 2024/06/04 05:04:12.431 FIG10b: Memory usage without flush after 12770 tasks: 1135.921875 MB +worker1-1 | 2024/06/04 05:04:13.4342024/06/04 05:04:13.434 FIG10b: Memory usage without flush after 12866 tasks: 1136.171875 MB +worker1-1 | 2024/06/04 05:04:14.446 FIG10b: Memory usage without flush after 12952 tasks: 1136.7109375 MB +worker1-1 | stored...[worker]2024/06/04 05:04:15.4562024/06/04 05:04:15.456 FIG10b: Memory usage without flush after 13074 tasks: 1137.08203125 MB +worker1-1 | 2024/06/04 05:04:16.462 FIG10b: Memory usage without flush after 13192 tasks: 1137.453125 MB +worker1-1 | 2024/06/04 05:04:17.468 FIG10b: Memory usage without flush after 13316 tasks: 1138.04296875 MB +worker1-1 | FIG10b: Memory usage without flush after 13439 tasks: 1138.6640625 MB +worker1-1 | 2024/06/04 05:04:19.478 FIG10b: Memory usage without flush after 13561 tasks: 1139.48046875 MB +worker1-1 | 2024/06/04 05:04:20.482 FIG10b: Memory usage without flush after 13680 tasks: 1140.09375 MB +worker1-1 | 2024/06/04 05:04:21.486 FIG10b: Memory usage without flush after 13800 tasks: 1140.7265625 MB +worker1-1 | 2024/06/04 05:04:22.494 FIG10b: Memory usage without flush after 13921 tasks: 1141.3671875 MB +worker1-1 | 2024/06/04 05:04:23.495 FIG10b: Memory usage without flush after 14046 tasks: 1141.96484375 MB +worker1-1 | 2024/06/04 05:04:24.5012024/06/04 05:04:24.502 [worker] 2024/06/04 05:04:24.502 FIG10b: Memory usage without flush after 14166 tasks: 1141.828125 MB +worker1-1 | 2024/06/04 05:04:25.509 FIG10b: Memory usage without flush after 14279 tasks: 1142.01171875 MB +worker1-1 | 2024/06/04 05:04:26.512 2024/06/04 05:04:26.5132024/06/04 05:04:26.513 FIG10b: Memory usage without flush after 14396 tasks: 1141.9140625 MB +worker1-1 | 2024/06/04 05:04:27.515 2024/06/04 05:04:27.515[worker] 2024/06/04 05:04:27.516 FIG10b: Memory usage without flush after 14515 tasks: 1142.26953125 MB +worker1-1 | 2024/06/04 05:04:28.522 FIG10b: Memory usage without flush after 14639 tasks: 1142.375 MB +worker1-1 | took this many seconds: 0.032024/06/04 05:04:29.524 FIG10b: Memory usage without flush after 14744 tasks: 1142.41015625 MB +worker1-1 | 2024/06/04 05:04:30.531 2024/06/04 05:04:30.531 FIG10b: Memory usage without flush after 14851 tasks: 1141.79296875 MB +worker1-1 | 2024/06/04 05:04:31.535 FIG10b: Memory usage without flush after 14954 tasks: 1142.7734375 MB +worker1-1 | 2024/06/04 05:04:32.537 FIG10b: Memory usage without flush after 15067 tasks: 1142.21875 MB +worker1-1 | [worker] 2024/06/04 05:04:33.539 FIG10b: Memory usage without flush after 15183 tasks: 1142.64453125 MB +worker1-1 | 2024/06/04 05:04:34.543 FIG10b: Memory usage without flush after 15294 tasks: 1142.89453125 MB +worker1-1 | 2024/06/04 05:04:35.5472024/06/04 05:04:35.547 FIG10b: Memory usage without flush after 15416 tasks: 1142.06640625 MB +worker1-1 | 2024/06/04 05:04:36.548 FIG10b: Memory usage without flush after 15534 tasks: 1142.44921875 MB +worker1-1 | [worker] 2024/06/04 05:04:37.554 FIG10b: Memory usage without flush after 15646 tasks: 1142.3671875 MB +worker1-1 | 2024/06/04 05:04:38.574 FIG10b: Memory usage without flush after 15755 tasks: 1141.66015625 MB +worker1-1 | 2024/06/04 05:04:39.580 [worker] 2024/06/04 05:04:39.581 FIG10b: Memory usage without flush after 15883 tasks: 1141.34375 MB +worker1-1 | 2024/06/04 05:04:40.5812024/06/04 05:04:40.581 FIG10b: Memory usage without flush after 16006 tasks: 1141.25 MB +worker1-1 | 2024/06/04 05:04:41.589 FIG10b: Memory usage without flush after 16131 tasks: 1149.65234375 MB +worker1-1 | 2024/06/04 05:04:42.597 FIG10b: Memory usage without flush after 16251 tasks: 1161.16796875 MB +worker1-1 | 2024/06/04 05:04:43.598 FIG10b: Memory usage without flush after 16373 tasks: 1156.609375 MB +worker1-1 | 2024/06/04 05:04:44.606 FIG10b: Memory usage without flush after 16496 tasks: 1154.74609375 MB +worker1-1 | 2024/06/04 05:04:45.608in Run() rn FIG10b: Memory usage without flush after 16628 tasks: 1155.5703125 MB +worker1-1 | 2024/06/04 05:04:46.621 FIG10b: Memory usage without flush after 16747 tasks: 1155.28125 MB +worker1-1 | 2024/06/04 05:04:47.6242024/06/04 05:04:47.625 FIG10b: Memory usage without flush after 16862 tasks: 1154.99609375 MB +worker1-1 | 2024/06/04 05:04:48.628 FIG10b: Memory usage without flush after 16973 tasks: 1155.8046875 MB +worker1-1 | [worker]took this many seconds: 0.0156[worker]2024/06/04 05:04:49.635 FIG10b: Memory usage without flush after 17101 tasks: 1155.41796875 MB +worker1-1 | 2024/06/04 05:04:50.636 FIG10b: Memory usage without flush after 17219 tasks: 1155.15234375 MB +worker1-1 | 2024/06/04 05:04:51.639 FIG10b: Memory usage without flush after 17346 tasks: 1155.5078125 MB +worker1-1 | stored...2024/06/04 05:04:52.648 FIG10b: Memory usage without flush after 17458 tasks: 1155.9765625 MB +worker1-1 | 2024/06/04 05:04:53.6532024/06/04 05:04:53.6532024/06/04 05:04:53.653 FIG10b: Memory usage without flush after 17579 tasks: 1157.46484375 MB +worker1-1 | 2024/06/04 05:04:54.656 FIG10b: Memory usage without flush after 17688 tasks: 1157.9921875 MB +worker1-1 | 2024/06/04 05:04:55.656 FIG10b: Memory usage without flush after 17801 tasks: 1158.453125 MB +worker1-1 | 2024/06/04 05:04:56.6582024/06/04 05:04:56.658 FIG10b: Memory usage without flush after 17924 tasks: 1155.96484375 MB +worker1-1 | 2024/06/04 05:04:57.659 FIG10b: Memory usage without flush after 18047 tasks: 1155.86328125 MB +worker1-1 | 2024/06/04 05:04:58.667 FIG10b: Memory usage without flush after 18160 tasks: 1156.83984375 MB +worker1-1 | 2024/06/04 05:04:59.672 2024/06/04 05:04:59.672[worker] FIG10b: Memory usage without flush after 18286 tasks: 1156.71484375 MB +worker1-1 | 2024/06/04 05:05:00.678 FIG10b: Memory usage without flush after 18409 tasks: 1155.984375 MB +worker1-1 | 2024/06/04 05:05:01.680 FIG10b: Memory usage without flush after 18534 tasks: 1156.67578125 MB +worker1-1 | 2024/06/04 05:05:02.6912024/06/04 05:05:02.690 FIG10b: Memory usage without flush after 18660 tasks: 1156.65234375 MB +worker1-1 | 2024/06/04 05:05:03.6962024/06/04 05:05:03.6962024/06/04 05:05:03.696 FIG10b: Memory usage without flush after 18792 tasks: 1156.6484375 MB +worker1-1 | 2024/06/04 05:05:04.6992024/06/04 05:05:04.699 FIG10b: Memory usage without flush after 18909 tasks: 1156.46875 MB +worker1-1 | 2024/06/04 05:05:05.705 FIG10b: Memory usage without flush after 19029 tasks: 1156.47265625 MB +worker1-1 | 2024/06/04 05:05:06.709 FIG10b: Memory usage without flush after 19155 tasks: 1157.63671875 MB +worker1-1 | 2024/06/04 05:05:07.718 FIG10b: Memory usage without flush after 19277 tasks: 1158.078125 MB +worker1-1 | 2024/06/04 05:05:08.724 FIG10b: Memory usage without flush after 19402 tasks: 1157.0 MB +worker1-1 | [worker]2024/06/04 05:05:09.727 FIG10b: Memory usage without flush after 19524 tasks: 1157.7265625 MB +worker1-1 | FIG10b: Memory usage without flush after 19631 tasks: 1157.71875 MB +worker1-1 | 2024/06/04 05:05:11.737 FIG10b: Memory usage without flush after 19751 tasks: 1158.40234375 MB +worker1-1 | 2024/06/04 05:05:12.745 FIG10b: Memory usage without flush after 19849 tasks: 1158.05078125 MB +worker1-1 | 2024/06/04 05:05:13.746 FIG10b: Memory usage without flush after 19952 tasks: 1157.85546875 MB +worker1-1 | 2024/06/04 05:05:14.750 FIG10b: Memory usage without flush after 20055 tasks: 1158.12109375 MB +worker1-1 | 2024/06/04 05:05:15.7542024/06/04 05:05:15.755 FIG10b: Memory usage without flush after 20168 tasks: 1158.3203125 MB +worker1-1 | 2024/06/04 05:05:16.761 FIG10b: Memory usage without flush after 20294 tasks: 1158.09765625 MB +worker1-1 | 2024/06/04 05:05:17.768 FIG10b: Memory usage without flush after 20417 tasks: 1158.53515625 MB +worker1-1 | 2024/06/04 05:05:18.772 FIG10b: Memory usage without flush after 20544 tasks: 1159.5078125 MB +worker1-1 | 2024/06/04 05:05:19.778[worker] took this many seconds: 0.03232024/06/04 05:05:19.779 FIG10b: Memory usage without flush after 20663 tasks: 1159.77734375 MB +worker1-1 | 2024/06/04 05:05:20.787 FIG10b: Memory usage without flush after 20786 tasks: 1159.3125 MB +worker1-1 | 2024/06/04 05:05:21.794 FIG10b: Memory usage without flush after 20935 tasks: 1175.1171875 MB +worker1-1 | 2024/06/04 05:05:22.798 FIG10b: Memory usage without flush after 21040 tasks: 1175.8046875 MB +worker1-1 | 2024/06/04 05:05:23.799 FIG10b: Memory usage without flush after 21169 tasks: 1176.72265625 MB2024/06/04 05:05:23.800 [lobs] WERE GONNA CREATE A /DEV FILE WITH THIS NAME: /dev/shm/5211556889717931035 +worker1-1 | 2024/06/04 05:05:24.8032024/06/04 05:05:24.803 FIG10b: Memory usage without flush after 21294 tasks: 1176.23828125 MB +worker1-1 | 2024/06/04 05:05:25.8052024/06/04 05:05:25.805 FIG10b: Memory usage without flush after 21424 tasks: 1176.46484375 MB +worker1-1 | 2024/06/04 05:05:26.813 FIG10b: Memory usage without flush after 21550 tasks: 1177.3515625 MB +worker1-1 | 2024/06/04 05:05:27.822 FIG10b: Memory usage without flush after 21675 tasks: 1176.828125 MB +worker1-1 | took this many seconds: 0.0182024/06/04 05:05:28.825 FIG10b: Memory usage without flush after 21788 tasks: 1177.84375 MB +worker1-1 | 2024/06/04 05:05:29.8292024/06/04 05:05:29.829 FIG10b: Memory usage without flush after 21922 tasks: 1176.84765625 MB +worker1-1 | 2024/06/04 05:05:30.8312024/06/04 05:05:30.832 FIG10b: Memory usage without flush after 22047 tasks: 1177.5546875 MB +worker1-1 | FIG10b: Memory usage without flush after 22167 tasks: 1178.4609375 MB +worker1-1 | FIG10b: Memory usage without flush after 22286 tasks: 1179.0546875 MB +worker1-1 | 2024/06/04 05:05:33.8442024/06/04 05:05:33.844 FIG10b: Memory usage without flush after 22407 tasks: 1180.11328125 MB +worker1-1 | 2024/06/04 05:05:34.909 FIG10b: Memory usage without flush after 22510 tasks: 1179.80078125 MB +worker1-1 | 2024/06/04 05:05:35.916 FIG10b: Memory usage without flush after 22634 tasks: 1179.89453125 MB +worker1-1 | 2024/06/04 05:05:36.925 FIG10b: Memory usage without flush after 22761 tasks: 1179.71484375 MB +worker1-1 | 2024/06/04 05:05:37.930 FIG10b: Memory usage without flush after 22880 tasks: 1184.296875 MB +worker1-1 | [worker]2024/06/04 05:05:38.931 gonna store nowFIG10b: Memory usage without flush after 22981 tasks: 1184.23828125 MB +worker1-1 | 2024/06/04 05:05:39.9372024/06/04 05:05:39.937 FIG10b: Memory usage without flush after 23091 tasks: 1183.90234375 MB +worker1-1 | 2024/06/04 05:05:40.940 FIG10b: Memory usage without flush after 23213 tasks: 1184.53515625 MB +worker1-1 | 2024/06/04 05:05:41.953 FIG10b: Memory usage without flush after 23340 tasks: 1184.80859375 MB +worker1-1 | 2024/06/04 05:05:42.958 FIG10b: Memory usage without flush after 23449 tasks: 1184.7421875 MB +worker1-1 | 2024/06/04 05:05:43.9612024/06/04 05:05:43.961 FIG10b: Memory usage without flush after 23565 tasks: 1185.1875 MB +worker1-1 | 2024/06/04 05:05:44.960 [worker]2024/06/04 05:05:44.961 FIG10b: Memory usage without flush after 23679 tasks: 1185.80859375 MB +worker1-1 | 2024/06/04 05:05:45.966 FIG10b: Memory usage without flush after 23788 tasks: 1185.24609375 MB +worker1-1 | 2024/06/04 05:05:46.9702024/06/04 05:05:46.971 [worker] took this many seconds: 0.00822FIG10b: Memory usage without flush after 23898 tasks: 1185.58203125 MB +worker1-1 | FIG10b: Memory usage without flush after 23998 tasks: 1185.28125 MB +worker1-1 | 2024/06/04 05:05:48.973 2024/06/04 05:05:48.973 FIG10b: Memory usage without flush after 24109 tasks: 1185.68359375 MB +worker1-1 | 2024/06/04 05:05:49.9782024/06/04 05:05:49.978 FIG10b: Memory usage without flush after 24218 tasks: 1186.0859375 MB +worker1-1 | 2024/06/04 05:05:50.982 FIG10b: Memory usage without flush after 24335 tasks: 1186.41796875 MB +worker1-1 | 2024/06/04 05:05:51.989stored...2024/06/04 05:05:51.989 FIG10b: Memory usage without flush after 24459 tasks: 1186.80859375 MB +worker1-1 | 2024/06/04 05:05:52.993 FIG10b: Memory usage without flush after 24567 tasks: 1187.203125 MB +worker1-1 | 2024/06/04 05:05:53.996 FIG10b: Memory usage without flush after 24692 tasks: 1187.26953125 MB +worker1-1 | [worker] 2024/06/04 05:05:55.002 FIG10b: Memory usage without flush after 24804 tasks: 1187.609375 MB +worker1-1 | 2024/06/04 05:05:56.005 FIG10b: Memory usage without flush after 24912 tasks: 1187.33984375 MB +worker1-1 | 2024/06/04 05:05:57.009 FIG10b: Memory usage without flush after 25042 tasks: 1187.41015625 MB +worker1-1 | 2024/06/04 05:05:58.0162024/06/04 05:05:58.017 FIG10b: Memory usage without flush after 25134 tasks: 1188.08203125 MB +worker1-1 | FIG10b: Memory usage without flush after 25256 tasks: 1187.7578125 MB +worker1-1 | 2024/06/04 05:06:00.023 FIG10b: Memory usage without flush after 25373 tasks: 1187.66015625 MB +worker1-1 | 2024/06/04 05:06:01.034 FIG10b: Memory usage without flush after 25480 tasks: 1188.265625 MB +worker1-1 | 2024/06/04 05:06:02.053 FIG10b: Memory usage without flush after 25595 tasks: 1195.21484375 MB +worker1-1 | 2024/06/04 05:06:03.057 2024/06/04 05:06:03.058 FIG10b: Memory usage without flush after 25707 tasks: 1195.9140625 MB +worker1-1 | Executed![worker]2024/06/04 05:06:04.0622024/06/04 05:06:04.062 FIG10b: Memory usage without flush after 25821 tasks: 1196.6640625 MB +worker1-1 | 2024/06/04 05:06:05.068 FIG10b: Memory usage without flush after 25942 tasks: 1196.41796875 MB +worker1-1 | 2024/06/04 05:06:06.078 FIG10b: Memory usage without flush after 26060 tasks: 1196.69921875 MB +worker1-1 | 2024/06/04 05:06:07.090 FIG10b: Memory usage without flush after 26179 tasks: 1195.98046875 MB +worker1-1 | 2024/06/04 05:06:08.095 FIG10b: Memory usage without flush after 26306 tasks: 1195.9140625 MB +worker1-1 | 2024/06/04 05:06:09.103 FIG10b: Memory usage without flush after 26426 tasks: 1194.25 MB +worker1-1 | 2024/06/04 05:06:10.109 FIG10b: Memory usage without flush after 26532 tasks: 1194.55078125 MB +worker1-1 | 2024/06/04 05:06:11.111 FIG10b: Memory usage without flush after 26653 tasks: 1195.765625 MB +worker1-1 | 2024/06/04 05:06:12.137 [worker]2024/06/04 05:06:12.138 FIG10b: Memory usage without flush after 26779 tasks: 1195.46484375 MB +worker1-1 | [worker][worker] 2024/06/04 05:06:13.138 FIG10b: Memory usage without flush after 26904 tasks: 1196.9296875 MB +worker1-1 | [worker]2024/06/04 05:06:14.139[worker] FIG10b: Memory usage without flush after 27032 tasks: 1197.1171875 MB +worker1-1 | 2024/06/04 05:06:15.146 FIG10b: Memory usage without flush after 27151 tasks: 1196.859375 MB +worker1-1 | 2024/06/04 05:06:16.155 FIG10b: Memory usage without flush after 27280 tasks: 1197.3515625 MB +worker1-1 | 2024/06/04 05:06:17.158 FIG10b: Memory usage without flush after 27409 tasks: 1198.2734375 MB +worker1-1 | 2024/06/04 05:06:18.163 FIG10b: Memory usage without flush after 27537 tasks: 1198.72265625 MB +worker1-1 | 2024/06/04 05:06:19.1722024/06/04 05:06:19.172 FIG10b: Memory usage without flush after 27662 tasks: 1199.4375 MB +worker1-1 | 2024/06/04 05:06:20.177 FIG10b: Memory usage without flush after 27781 tasks: 1206.4609375 MB +worker1-1 | 2024/06/04 05:06:21.1822024/06/04 05:06:21.182 FIG10b: Memory usage without flush after 27910 tasks: 1206.453125 MB +worker1-1 | 2024/06/04 05:06:22.185 FIG10b: Memory usage without flush after 28036 tasks: 1206.6171875 MB +worker1-1 | [worker]stored...in Run() rn2024/06/04 05:06:23.189 FIG10b: Memory usage without flush after 28154 tasks: 1207.31640625 MB +worker1-1 | [worker]2024/06/04 05:06:24.192[worker] 2024/06/04 05:06:24.192 FIG10b: Memory usage without flush after 28269 tasks: 1207.59765625 MB +worker1-1 | 2024/06/04 05:06:25.196 FIG10b: Memory usage without flush after 28387 tasks: 1207.38671875 MB +worker1-1 | 2024/06/04 05:06:26.201 FIG10b: Memory usage without flush after 28509 tasks: 1207.40234375 MB +worker1-1 | 2024/06/04 05:06:27.204 [worker] 2024/06/04 05:06:27.205 FIG10b: Memory usage without flush after 28633 tasks: 1208.078125 MB +worker1-1 | 2024/06/04 05:06:28.2062024/06/04 05:06:28.206 FIG10b: Memory usage without flush after 28764 tasks: 1209.0 MB +worker1-1 | Executed!2024/06/04 05:06:29.211 FIG10b: Memory usage without flush after 28885 tasks: 1209.265625 MB +worker1-1 | 2024/06/04 05:06:30.218 FIG10b: Memory usage without flush after 29017 tasks: 1209.421875 MB +worker1-1 | 2024/06/04 05:06:31.223 FIG10b: Memory usage without flush after 29135 tasks: 1209.6796875 MB +worker1-1 | 2024/06/04 05:06:32.234 2024/06/04 05:06:32.234 FIG10b: Memory usage without flush after 29252 tasks: 1209.71875 MB +worker1-1 | 2024/06/04 05:06:33.235 FIG10b: Memory usage without flush after 29374 tasks: 1210.04296875 MB +worker1-1 | 2024/06/04 05:06:34.246 FIG10b: Memory usage without flush after 29496 tasks: 1209.84375 MB +worker1-1 | 2024/06/04 05:06:35.251 FIG10b: Memory usage without flush after 29624 tasks: 1210.43359375 MB +worker1-1 | [worker]2024/06/04 05:06:36.252 FIG10b: Memory usage without flush after 29745 tasks: 1209.67578125 MB +worker1-1 | 2024/06/04 05:06:37.270 [worker] 2024/06/04 05:06:37.271 FIG10b: Memory usage without flush after 29839 tasks: 1209.05859375 MB +worker1-1 | 2024/06/04 05:06:38.275 FIG10b: Memory usage without flush after 29945 tasks: 1209.65625 MB +worker1-1 | 2024/06/04 05:06:39.2792024/06/04 05:06:39.279 FIG10b: Memory usage without flush after 30072 tasks: 1209.984375 MB +worker1-1 | 2024/06/04 05:06:40.282 FIG10b: Memory usage without flush after 30204 tasks: 1208.9375 MB +worker1-1 | 2024/06/04 05:06:41.2822024/06/04 05:06:41.282 FIG10b: Memory usage without flush after 30336 tasks: 1209.0546875 MB +worker1-1 | 2024/06/04 05:06:42.286 FIG10b: Memory usage without flush after 30460 tasks: 1227.84765625 MB +worker1-1 | 2024/06/04 05:06:43.2852024/06/04 05:06:43.286 [worker]FIG10b: Memory usage without flush after 30584 tasks: 1228.1796875 MB +worker1-1 | 2024/06/04 05:06:44.297 FIG10b: Memory usage without flush after 30708 tasks: 1229.43359375 MB +worker1-1 | gonna store now2024/06/04 05:06:45.305 FIG10b: Memory usage without flush after 30838 tasks: 1229.54296875 MB +worker1-1 | 2024/06/04 05:06:46.311 FIG10b: Memory usage without flush after 30970 tasks: 1229.41796875 MB +worker1-1 | [worker][worker]2024/06/04 05:06:47.313 FIG10b: Memory usage without flush after 31094 tasks: 1229.97265625 MB +worker1-1 | FIG10b: Memory usage without flush after 31229 tasks: 1229.83984375 MB +worker1-1 | 2024/06/04 05:06:49.321 2024/06/04 05:06:49.322 FIG10b: Memory usage without flush after 31354 tasks: 1230.15625 MB +worker1-1 | [worker] 2024/06/04 05:06:50.326 FIG10b: Memory usage without flush after 31485 tasks: 1230.7109375 MB +worker1-1 | 2024/06/04 05:06:51.331 FIG10b: Memory usage without flush after 31609 tasks: 1230.125 MB +worker1-1 | 2024/06/04 05:06:52.337 FIG10b: Memory usage without flush after 31719 tasks: 1229.9921875 MB +worker1-1 | 2024/06/04 05:06:53.339 [worker]2024/06/04 05:06:53.339 FIG10b: Memory usage without flush after 31835 tasks: 1231.0703125 MB +worker1-1 | Executed!2024/06/04 05:06:54.346 FIG10b: Memory usage without flush after 31940 tasks: 1231.6640625 MB +worker1-1 | 2024/06/04 05:06:55.351 [worker] [worker] in Run() rn2024/06/04 05:06:55.352 2024/06/04 05:06:55.352 FIG10b: Memory usage without flush after 32057 tasks: 1231.9921875 MB +worker1-1 | 2024/06/04 05:06:56.357 FIG10b: Memory usage without flush after 32183 tasks: 1231.83984375 MB +worker1-1 | 2024/06/04 05:06:57.359 FIG10b: Memory usage without flush after 32309 tasks: 1231.92578125 MB +worker1-1 | 2024/06/04 05:06:58.363 FIG10b: Memory usage without flush after 32443 tasks: 1232.46484375 MB +worker1-1 | 2024/06/04 05:06:59.366 FIG10b: Memory usage without flush after 32568 tasks: 1231.86328125 MB +worker1-1 | 2024/06/04 05:07:00.373 [worker]2024/06/04 05:07:00.373 FIG10b: Memory usage without flush after 32687 tasks: 1232.2734375 MB +worker1-1 | 2024/06/04 05:07:01.3952024/06/04 05:07:01.395 FIG10b: Memory usage without flush after 32815 tasks: 1231.91796875 MB +worker1-1 | FIG10b: Memory usage without flush after 32915 tasks: 1232.0390625 MB +worker1-1 | 2024/06/04 05:07:03.407 FIG10b: Memory usage without flush after 33013 tasks: 1232.234375 MB +worker1-1 | 2024/06/04 05:07:04.408 FIG10b: Memory usage without flush after 33137 tasks: 1232.72265625 MB +worker1-1 | 2024/06/04 05:07:05.408 FIG10b: Memory usage without flush after 33259 tasks: 1232.98046875 MB +worker1-1 | 2024/06/04 05:07:06.4152024/06/04 05:07:06.415 FIG10b: Memory usage without flush after 33383 tasks: 1232.5546875 MB +worker1-1 | 2024/06/04 05:07:07.420 FIG10b: Memory usage without flush after 33509 tasks: 1233.26171875 MB +worker1-1 | [worker][worker]2024/06/04 05:07:37.421 FIG10b: Memory usage without flush after 36983 tasks: 1247.23828125 MB +worker1-1 | 2024/06/04 05:08:07.422 FIG10b: Memory usage without flush after 40479 tasks: 1292.04296875 MB +worker1-1 | 2024/06/04 05:08:37.433 FIG10b: Memory usage without flush after 43649 tasks: 1284.76171875 MB +worker1-1 | 2024/06/04 05:09:07.435 FIG10b: Memory usage without flush after 47102 tasks: 1307.30859375 MB +worker1-1 | 2024/06/04 05:09:37.437 FIG10b: Memory usage without flush after 50458 tasks: 1310.375 MB +worker1-1 | 2024/06/04 05:10:07.443 FIG10b: Memory usage without flush after 54043 tasks: 1344.3984375 MB +worker1-1 | 2024/06/04 05:10:37.444 FIG10b: Memory usage without flush after 57421 tasks: 1365.17578125 MB +worker1-1 | 2024/06/04 05:11:07.450 FIG10b: Memory usage without flush after 60590 tasks: 1376.80078125 MB +worker1-1 | 2024/06/04 05:11:37.457 FIG10b: Memory usage without flush after 63784 tasks: 1370.8515625 MB +worker1-1 | 2024/06/04 05:12:07.463 FIG10b: Memory usage without flush after 66952 tasks: 1390.1875 MB +worker1-1 | 2024/06/04 05:12:37.467 FIG10b: Memory usage without flush after 70098 tasks: 1391.84765625 MB +worker1-1 | 2024/06/04 05:13:07.468 2024/06/04 05:13:07.469 FIG10b: Memory usage without flush after 73440 tasks: 1402.96484375 MB +worker1-1 | 2024/06/04 05:13:37.4822024/06/04 05:13:37.483 FIG10b: Memory usage without flush after 76749 tasks: 1402.9140625 MB +worker1-1 | 2024/06/04 05:14:07.493[worker] FIG10b: Memory usage without flush after 79857 tasks: 1417.44140625 MB +worker1-1 | 2024/06/04 05:14:37.497 FIG10b: Memory usage without flush after 83054 tasks: 1417.73828125 MB +worker1-1 | 2024/06/04 05:15:07.498 [worker] in Run() rn2024/06/04 05:15:07.499 FIG10b: Memory usage without flush after 86111 tasks: 1424.1328125 MB +worker1-1 | 2024/06/04 05:15:37.499 FIG10b: Memory usage without flush after 89183 tasks: 1435.15625 MB +worker1-1 | 2024/06/04 05:16:07.509 FIG10b: Memory usage without flush after 92171 tasks: 1450.89453125 MB +worker1-1 | 2024/06/04 05:16:37.5142024/06/04 05:16:37.514 FIG10b: Memory usage without flush after 95399 tasks: 1445.234375 MB +worker1-1 | 2024/06/04 05:17:07.516 FIG10b: Memory usage without flush after 98729 tasks: 1459.6484375 MB +worker1-1 | [worker] 2024/06/04 05:17:37.518 FIG10b: Memory usage without flush after 101889 tasks: 1468.859375 MB +worker1-1 | 2024/06/04 05:18:07.520 FIG10b: Memory usage without flush after 105104 tasks: 1477.3515625 MB +worker1-1 | 2024/06/04 05:18:37.523 FIG10b: Memory usage without flush after 108452 tasks: 1489.90625 MB +worker1-1 | 2024/06/04 05:19:07.524 FIG10b: Memory usage without flush after 111871 tasks: 1498.66796875 MB +worker1-1 | 2024/06/04 05:19:37.5292024/06/04 05:19:37.530 FIG10b: Memory usage without flush after 115452 tasks: 1559.05859375 MB +worker1-1 | 2024/06/04 05:20:07.533 2024/06/04 05:20:07.533 FIG10b: Memory usage without flush after 118325 tasks: 1581.3359375 MB +worker1-1 | 2024/06/04 05:20:37.541 FIG10b: Memory usage without flush after 121643 tasks: 1576.234375 MB +worker1-1 | 2024/06/04 05:21:07.544 2024/06/04 05:21:07.544 FIG10b: Memory usage without flush after 125200 tasks: 1609.65234375 MB +worker1-1 | [worker]2024/06/04 05:21:37.563 FIG10b: Memory usage without flush after 128689 tasks: 1587.80859375 MB +worker1-1 | 2024/06/04 05:22:07.570 FIG10b: Memory usage without flush after 132013 tasks: 1588.0546875 MB +worker1-1 | 2024/06/04 05:22:37.579 FIG10b: Memory usage without flush after 135527 tasks: 1569.06640625 MB +worker1-1 | 2024/06/04 05:23:07.580 FIG10b: Memory usage without flush after 139395 tasks: 1677.1171875 MB +worker1-1 | 2024/06/04 05:23:37.5842024/06/04 05:23:37.585 FIG10b: Memory usage without flush after 142900 tasks: 1680.1640625 MB +worker1-1 | 2024/06/04 05:24:07.586 FIG10b: Memory usage without flush after 146401 tasks: 1626.828125 MB +worker1-1 | 2024/06/04 05:24:37.589 FIG10b: Memory usage without flush after 149920 tasks: 1621.16015625 MB +worker1-1 | 2024/06/04 05:25:07.593 [worker] 2024/06/04 05:25:07.593 FIG10b: Memory usage without flush after 153002 tasks: 1613.37890625 MB +worker1-1 | [worker]2024/06/04 05:25:37.594 FIG10b: Memory usage without flush after 156473 tasks: 1619.60546875 MB +worker1-1 | [worker][worker][worker]2024/06/04 05:26:07.599 FIG10b: Memory usage without flush after 159999 tasks: 1633.79296875 MB +worker1-1 | 2024/06/04 05:26:37.606[worker]2024/06/04 05:26:37.606 Executed! FIG10b: Memory usage without flush after 163639 tasks: 1622.60546875 MB +worker1-1 | 2024/06/04 05:27:07.613 FIG10b: Memory usage without flush after 167187 tasks: 1626.93359375 MB +worker1-1 | 2024/06/04 05:27:37.6202024/06/04 05:27:37.620 FIG10b: Memory usage without flush after 170695 tasks: 1628.02734375 MB +worker1-1 | 2024/06/04 05:28:07.6252024/06/04 05:28:07.625 FIG10b: Memory usage without flush after 174250 tasks: 1638.90625 MB +worker1-1 | 2024/06/04 05:28:37.633 in Run() rn2024/06/04 05:28:37.633 FIG10b: Memory usage without flush after 177854 tasks: 1645.7734375 MB +worker1-1 | 2024/06/04 05:29:07.639 [worker]2024/06/04 05:29:07.6402024/06/04 05:29:07.640 FIG10b: Memory usage without flush after 181212 tasks: 1632.65234375 MB +worker1-1 | [worker]2024/06/04 05:29:37.648 FIG10b: Memory usage without flush after 184665 tasks: 1645.6171875 MB +worker1-1 | 2024/06/04 05:30:07.650 FIG10b: Memory usage without flush after 188221 tasks: 1648.38671875 MB +worker1-1 | 2024/06/04 05:30:37.6562024/06/04 05:30:37.656 FIG10b: Memory usage without flush after 191943 tasks: 1637.078125 MB +worker1-1 | 2024/06/04 05:31:07.661 [worker]2024/06/04 05:31:07.661 2024/06/04 05:31:07.661 FIG10b: Memory usage without flush after 195503 tasks: 1674.83203125 MB +worker1-1 | 2024/06/04 05:31:37.6622024/06/04 05:31:37.662 FIG10b: Memory usage without flush after 198972 tasks: 1683.1796875 MB +worker1-1 | 2024/06/04 05:32:07.663 FIG10b: Memory usage without flush after 202546 tasks: 1681.87890625 MB +worker1-1 | 2024/06/04 05:32:37.666 FIG10b: Memory usage without flush after 205837 tasks: 1683.41796875 MB +worker1-1 | 2024/06/04 05:33:07.669 FIG10b: Memory usage without flush after 208914 tasks: 1697.05859375 MB +worker1-1 | 2024/06/04 05:33:37.678 FIG10b: Memory usage without flush after 211956 tasks: 1697.8671875 MB +worker1-1 | 2024/06/04 05:34:07.685 FIG10b: Memory usage without flush after 215103 tasks: 1711.38671875 MB +worker1-1 | 2024/06/04 05:34:37.6932024/06/04 05:34:37.693 FIG10b: Memory usage without flush after 218373 tasks: 1704.4921875 MB +worker1-1 | 2024/06/04 05:35:07.695 2024/06/04 05:35:07.6962024/06/04 05:35:07.696 FIG10b: Memory usage without flush after 221631 tasks: 1735.703125 MB +worker1-1 | 2024/06/04 05:35:37.697 FIG10b: Memory usage without flush after 225768 tasks: 1933.51171875 MB +worker1-1 | 2024/06/04 05:36:07.713 FIG10b: Memory usage without flush after 229021 tasks: 1952.546875 MB +worker1-1 | 2024/06/04 05:36:37.721 FIG10b: Memory usage without flush after 232420 tasks: 1962.08203125 MB +worker1-1 | 2024/06/04 05:37:07.722 FIG10b: Memory usage without flush after 235733 tasks: 1857.82421875 MB +worker1-1 | 2024/06/04 05:37:37.726 FIG10b: Memory usage without flush after 239177 tasks: 1834.546875 MB +worker1-1 | 2024/06/04 05:38:07.728 FIG10b: Memory usage without flush after 242716 tasks: 1841.734375 MB +worker1-1 | in Run() rn2024/06/04 05:38:37.731 FIG10b: Memory usage without flush after 246342 tasks: 1846.953125 MB +worker1-1 | 2024/06/04 05:39:07.733 [worker] 2024/06/04 05:39:07.733 FIG10b: Memory usage without flush after 249650 tasks: 1819.546875 MB +worker1-1 | 2024/06/04 05:39:37.755 FIG10b: Memory usage without flush after 252670 tasks: 1806.94140625 MB +worker1-1 | 2024/06/04 05:40:07.759 FIG10b: Memory usage without flush after 255800 tasks: 1800.9921875 MB +worker1-1 | 2024/06/04 05:40:37.765 2024/06/04 05:40:37.765 FIG10b: Memory usage without flush after 259206 tasks: 1800.98046875 MB +worker1-1 | 2024/06/04 05:41:07.768 FIG10b: Memory usage without flush after 262385 tasks: 1814.44921875 MB +worker1-1 | 2024/06/04 05:41:37.787 2024/06/04 05:41:37.787 FIG10b: Memory usage without flush after 265365 tasks: 1825.640625 MB +worker1-1 | 2024/06/04 05:42:07.7862024/06/04 05:42:07.787 FIG10b: Memory usage without flush after 268546 tasks: 1849.9140625 MB +worker1-1 | 2024/06/04 05:42:37.797 FIG10b: Memory usage without flush after 271943 tasks: 1718.9375 MB +worker1-1 | 2024/06/04 05:43:07.808 FIG10b: Memory usage without flush after 276123 tasks: 1734.16796875 MB +worker1-1 | 2024/06/04 05:43:37.815 FIG10b: Memory usage without flush after 279980 tasks: 1741.33984375 MB +worker1-1 | 2024/06/04 05:44:07.857 FIG10b: Memory usage without flush after 284536 tasks: 1747.734375 MB +worker1-1 | 2024/06/04 05:44:37.8582024/06/04 05:44:37.859 FIG10b: Memory usage without flush after 289278 tasks: 1754.3046875 MB +worker1-1 | 2024/06/04 05:45:07.896 2024/06/04 05:45:07.896 FIG10b: Memory usage without flush after 293687 tasks: 1757.74609375 MB +worker1-1 | 2024/06/04 05:45:37.901 FIG10b: Memory usage without flush after 297528 tasks: 1764.03125 MB +worker1-1 | 2024/06/04 05:46:07.904 FIG10b: Memory usage without flush after 301888 tasks: 1772.7265625 MB +worker1-1 | 2024/06/04 05:46:37.907 FIG10b: Memory usage without flush after 306560 tasks: 1785.47265625 MB +worker1-1 | 2024/06/04 05:47:07.912 FIG10b: Memory usage without flush after 310804 tasks: 1786.0703125 MB +worker1-1 | 2024/06/04 05:47:37.914 FIG10b: Memory usage without flush after 314913 tasks: 1790.37890625 MB +worker1-1 | 2024/06/04 05:48:07.917 FIG10b: Memory usage without flush after 319430 tasks: 1790.66796875 MB +worker1-1 | 2024/06/04 05:48:37.926 FIG10b: Memory usage without flush after 323794 tasks: 1799.15234375 MB +worker1-1 | 2024/06/04 05:49:07.926 FIG10b: Memory usage without flush after 328756 tasks: 1816.2578125 MB +worker1-1 | 2024/06/04 05:49:37.929 FIG10b: Memory usage without flush after 333277 tasks: 1824.21875 MB +worker1-1 | 2024/06/04 05:50:07.934 FIG10b: Memory usage without flush after 337899 tasks: 1832.9375 MB +worker1-1 | 2024/06/04 05:50:37.937 FIG10b: Memory usage without flush after 342472 tasks: 1840.79296875 MB +worker1-1 | [worker] 2024/06/04 05:51:07.941 FIG10b: Memory usage without flush after 346984 tasks: 1849.5546875 MB +worker1-1 | 2024/06/04 05:51:37.942 FIG10b: Memory usage without flush after 351769 tasks: 1852.1875 MB +worker1-1 | FIG10b: Memory usage without flush after 356876 tasks: 1860.515625 MB +worker1-1 | 2024/06/04 05:52:37.996 2024/06/04 05:52:37.997 FIG10b: Memory usage without flush after 361299 tasks: 1863.34765625 MB +worker1-1 | 2024/06/04 05:53:08.005 FIG10b: Memory usage without flush after 365767 tasks: 1875.44140625 MB +worker1-1 | 2024/06/04 05:53:38.005 FIG10b: Memory usage without flush after 370574 tasks: 1877.84765625 MB +worker1-1 | 2024/06/04 05:54:08.009 FIG10b: Memory usage without flush after 375416 tasks: 1888.6796875 MB +worker1-1 | [worker]2024/06/04 05:54:38.010 FIG10b: Memory usage without flush after 379714 tasks: 1895.55078125 MB +worker1-1 | 2024/06/04 05:55:08.011 FIG10b: Memory usage without flush after 384529 tasks: 1865.66015625 MB +worker1-1 | 2024/06/04 05:55:38.011 FIG10b: Memory usage without flush after 389515 tasks: 1887.671875 MB +worker1-1 | 2024/06/04 05:56:08.016 FIG10b: Memory usage without flush after 394431 tasks: 1888.40234375 MB +worker1-1 | 2024/06/04 05:56:38.015 FIG10b: Memory usage without flush after 399252 tasks: 1893.87890625 MB +worker1-1 | 2024/06/04 05:57:08.020 FIG10b: Memory usage without flush after 404301 tasks: 1903.44140625 MB +worker1-1 | 2024/06/04 05:57:38.021 FIG10b: Memory usage without flush after 409302 tasks: 1913.1875 MB +worker1-1 | 2024/06/04 05:58:08.0222024/06/04 05:58:08.022 FIG10b: Memory usage without flush after 414394 tasks: 1935.921875 MB[worker] +worker1-1 | 2024/06/04 05:58:38.028 FIG10b: Memory usage without flush after 419527 tasks: 1930.99609375 MB +worker1-1 | 2024/06/04 05:59:08.0322024/06/04 05:59:08.032 FIG10b: Memory usage without flush after 424439 tasks: 1943.5625 MB +worker1-1 | 2024/06/04 05:59:38.039 FIG10b: Memory usage without flush after 429165 tasks: 1948.5234375 MB +worker1-1 | 2024/06/04 06:00:08.0392024/06/04 06:00:08.040 FIG10b: Memory usage without flush after 434211 tasks: 1955.40625 MB +worker1-1 | 2024/06/04 06:00:38.043 FIG10b: Memory usage without flush after 438970 tasks: 1962.734375 MB +worker1-1 | 2024/06/04 06:01:08.046 FIG10b: Memory usage without flush after 443753 tasks: 1968.8828125 MB +worker1-1 | 2024/06/04 06:01:38.049 FIG10b: Memory usage without flush after 448729 tasks: 2005.7421875 MB diff --git a/analysis/output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv b/analysis/output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv new file mode 100644 index 0000000..fb900b3 --- /dev/null +++ b/analysis/output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv @@ -0,0 +1,237 @@ +Timestamp,Memory Usage (MB) +2024-06-04 05:02:08.045000,1046.09765625 +2024-06-04 05:02:09.053000,1047.3203125 +2024-06-04 05:02:10.057000,1038.96484375 +2024-06-04 05:02:11.060000,1044.33203125 +2024-06-04 05:02:12.069000,1049.59375 +2024-06-04 05:02:13.069000,1055.1796875 +2024-06-04 05:02:15.084000,1055.5078125 +2024-06-04 05:02:16.088000,1055.58203125 +2024-06-04 05:02:18.099000,1057.578125 +2024-06-04 05:02:19.109000,1057.734375 +2024-06-04 05:02:20.128000,1059.52734375 +2024-06-04 05:02:21.134000,1060.06640625 +2024-06-04 05:02:23.153000,1061.59375 +2024-06-04 05:02:25.161000,1065.05078125 +2024-06-04 05:02:26.171000,1065.30078125 +2024-06-04 05:02:27.474000,1065.41015625 +2024-06-04 05:02:28.481000,1069.33984375 +2024-06-04 05:02:29.515000,1069.31640625 +2024-06-04 05:02:30.520000,1073.29296875 +2024-06-04 05:02:32.522000,1074.93359375 +2024-06-04 05:02:35.543000,1074.82421875 +2024-06-04 05:02:38.554000,1076.67578125 +2024-06-04 05:02:39.556000,1076.46875 +2024-06-04 05:02:40.638000,1076.50390625 +2024-06-04 05:02:41.666000,1086.84765625 +2024-06-04 05:02:43.666000,1086.4140625 +2024-06-04 05:02:44.674000,1088.109375 +2024-06-04 05:02:48.696000,1088.625 +2024-06-04 05:02:52.716000,1091.8828125 +2024-06-04 05:02:54.736000,1093.75 +2024-06-04 05:02:58.760000,1093.91015625 +2024-06-04 05:02:59.763000,1093.84765625 +2024-06-04 05:03:01.777000,1094.30859375 +2024-06-04 05:03:03.784000,1094.41015625 +2024-06-04 05:03:06.818000,1094.76171875 +2024-06-04 05:03:09.845000,1094.91796875 +2024-06-04 05:03:11.848000,1095.92578125 +2024-06-04 05:03:12.860000,1095.65234375 +2024-06-04 05:03:14.872000,1097.30078125 +2024-06-04 05:03:17.012000,1096.73046875 +2024-06-04 05:03:19.022000,1097.453125 +2024-06-04 05:03:20.023000,1097.36328125 +2024-06-04 05:03:24.034000,1117.375 +2024-06-04 05:03:30.060000,1121.453125 +2024-06-04 05:03:32.067000,1122.484375 +2024-06-04 05:03:33.071000,1122.0546875 +2024-06-04 05:03:34.074000,1122.00390625 +2024-06-04 05:03:35.076000,1121.9609375 +2024-06-04 05:03:36.080000,1121.7421875 +2024-06-04 05:03:37.149000,1121.2734375 +2024-06-04 05:03:38.155000,1121.265625 +2024-06-04 05:03:39.157000,1122.70703125 +2024-06-04 05:03:40.157000,1123.43359375 +2024-06-04 05:03:41.160000,1123.70703125 +2024-06-04 05:03:44.166000,1124.14453125 +2024-06-04 05:03:45.178000,1124.15234375 +2024-06-04 05:03:46.259000,1123.47265625 +2024-06-04 05:03:50.276000,1124.7265625 +2024-06-04 05:03:55.288000,1125.7421875 +2024-06-04 05:03:56.298000,1125.046875 +2024-06-04 05:03:57.299000,1125.49609375 +2024-06-04 05:04:01.344000,1126.3515625 +2024-06-04 05:04:02.353000,1135.16796875 +2024-06-04 05:04:03.373000,1135.34765625 +2024-06-04 05:04:04.377000,1135.10546875 +2024-06-04 05:04:08.396000,1135.859375 +2024-06-04 05:04:09.403000,1135.97265625 +2024-06-04 05:04:10.414000,1135.953125 +2024-06-04 05:04:11.427000,1135.75 +2024-06-04 05:04:12.431000,1135.921875 +2024-06-04 05:04:14.446000,1136.7109375 +2024-06-04 05:04:16.462000,1137.453125 +2024-06-04 05:04:17.468000,1138.04296875 +2024-06-04 05:04:19.478000,1139.48046875 +2024-06-04 05:04:20.482000,1140.09375 +2024-06-04 05:04:21.486000,1140.7265625 +2024-06-04 05:04:22.494000,1141.3671875 +2024-06-04 05:04:23.495000,1141.96484375 +2024-06-04 05:04:25.509000,1142.01171875 +2024-06-04 05:04:28.522000,1142.375 +2024-06-04 05:04:31.535000,1142.7734375 +2024-06-04 05:04:32.537000,1142.21875 +2024-06-04 05:04:34.543000,1142.89453125 +2024-06-04 05:04:36.548000,1142.44921875 +2024-06-04 05:04:38.574000,1141.66015625 +2024-06-04 05:04:41.589000,1149.65234375 +2024-06-04 05:04:42.597000,1161.16796875 +2024-06-04 05:04:43.598000,1156.609375 +2024-06-04 05:04:44.606000,1154.74609375 +2024-06-04 05:04:46.621000,1155.28125 +2024-06-04 05:04:48.628000,1155.8046875 +2024-06-04 05:04:50.636000,1155.15234375 +2024-06-04 05:04:51.639000,1155.5078125 +2024-06-04 05:04:54.656000,1157.9921875 +2024-06-04 05:04:55.656000,1158.453125 +2024-06-04 05:04:57.659000,1155.86328125 +2024-06-04 05:04:58.667000,1156.83984375 +2024-06-04 05:05:00.678000,1155.984375 +2024-06-04 05:05:01.680000,1156.67578125 +2024-06-04 05:05:05.705000,1156.47265625 +2024-06-04 05:05:06.709000,1157.63671875 +2024-06-04 05:05:07.718000,1158.078125 +2024-06-04 05:05:08.724000,1157.0 +2024-06-04 05:05:11.737000,1158.40234375 +2024-06-04 05:05:12.745000,1158.05078125 +2024-06-04 05:05:13.746000,1157.85546875 +2024-06-04 05:05:14.750000,1158.12109375 +2024-06-04 05:05:16.761000,1158.09765625 +2024-06-04 05:05:17.768000,1158.53515625 +2024-06-04 05:05:18.772000,1159.5078125 +2024-06-04 05:05:20.787000,1159.3125 +2024-06-04 05:05:21.794000,1175.1171875 +2024-06-04 05:05:22.798000,1175.8046875 +2024-06-04 05:05:23.799000,1176.72265625 +2024-06-04 05:05:26.813000,1177.3515625 +2024-06-04 05:05:27.822000,1176.828125 +2024-06-04 05:05:34.909000,1179.80078125 +2024-06-04 05:05:35.916000,1179.89453125 +2024-06-04 05:05:36.925000,1179.71484375 +2024-06-04 05:05:37.930000,1184.296875 +2024-06-04 05:05:40.940000,1184.53515625 +2024-06-04 05:05:41.953000,1184.80859375 +2024-06-04 05:05:42.958000,1184.7421875 +2024-06-04 05:05:45.966000,1185.24609375 +2024-06-04 05:05:50.982000,1186.41796875 +2024-06-04 05:05:52.993000,1187.203125 +2024-06-04 05:05:53.996000,1187.26953125 +2024-06-04 05:05:56.005000,1187.33984375 +2024-06-04 05:05:57.009000,1187.41015625 +2024-06-04 05:06:00.023000,1187.66015625 +2024-06-04 05:06:01.034000,1188.265625 +2024-06-04 05:06:02.053000,1195.21484375 +2024-06-04 05:06:05.068000,1196.41796875 +2024-06-04 05:06:06.078000,1196.69921875 +2024-06-04 05:06:07.090000,1195.98046875 +2024-06-04 05:06:08.095000,1195.9140625 +2024-06-04 05:06:09.103000,1194.25 +2024-06-04 05:06:10.109000,1194.55078125 +2024-06-04 05:06:11.111000,1195.765625 +2024-06-04 05:06:15.146000,1196.859375 +2024-06-04 05:06:16.155000,1197.3515625 +2024-06-04 05:06:17.158000,1198.2734375 +2024-06-04 05:06:18.163000,1198.72265625 +2024-06-04 05:06:20.177000,1206.4609375 +2024-06-04 05:06:22.185000,1206.6171875 +2024-06-04 05:06:25.196000,1207.38671875 +2024-06-04 05:06:26.201000,1207.40234375 +2024-06-04 05:06:30.218000,1209.421875 +2024-06-04 05:06:31.223000,1209.6796875 +2024-06-04 05:06:33.235000,1210.04296875 +2024-06-04 05:06:34.246000,1209.84375 +2024-06-04 05:06:35.251000,1210.43359375 +2024-06-04 05:06:38.275000,1209.65625 +2024-06-04 05:06:40.282000,1208.9375 +2024-06-04 05:06:42.286000,1227.84765625 +2024-06-04 05:06:44.297000,1229.43359375 +2024-06-04 05:06:46.311000,1229.41796875 +2024-06-04 05:06:51.331000,1230.125 +2024-06-04 05:06:52.337000,1229.9921875 +2024-06-04 05:06:56.357000,1231.83984375 +2024-06-04 05:06:57.359000,1231.92578125 +2024-06-04 05:06:58.363000,1232.46484375 +2024-06-04 05:06:59.366000,1231.86328125 +2024-06-04 05:07:03.407000,1232.234375 +2024-06-04 05:07:04.408000,1232.72265625 +2024-06-04 05:07:05.408000,1232.98046875 +2024-06-04 05:07:07.420000,1233.26171875 +2024-06-04 05:08:07.422000,1292.04296875 +2024-06-04 05:08:37.433000,1284.76171875 +2024-06-04 05:09:07.435000,1307.30859375 +2024-06-04 05:09:37.437000,1310.375 +2024-06-04 05:10:07.443000,1344.3984375 +2024-06-04 05:10:37.444000,1365.17578125 +2024-06-04 05:11:07.450000,1376.80078125 +2024-06-04 05:11:37.457000,1370.8515625 +2024-06-04 05:12:07.463000,1390.1875 +2024-06-04 05:12:37.467000,1391.84765625 +2024-06-04 05:14:37.497000,1417.73828125 +2024-06-04 05:15:37.499000,1435.15625 +2024-06-04 05:16:07.509000,1450.89453125 +2024-06-04 05:17:07.516000,1459.6484375 +2024-06-04 05:18:07.520000,1477.3515625 +2024-06-04 05:18:37.523000,1489.90625 +2024-06-04 05:19:07.524000,1498.66796875 +2024-06-04 05:20:37.541000,1576.234375 +2024-06-04 05:22:07.570000,1588.0546875 +2024-06-04 05:22:37.579000,1569.06640625 +2024-06-04 05:23:07.580000,1677.1171875 +2024-06-04 05:24:07.586000,1626.828125 +2024-06-04 05:24:37.589000,1621.16015625 +2024-06-04 05:27:07.613000,1626.93359375 +2024-06-04 05:30:07.650000,1648.38671875 +2024-06-04 05:32:07.663000,1681.87890625 +2024-06-04 05:32:37.666000,1683.41796875 +2024-06-04 05:33:07.669000,1697.05859375 +2024-06-04 05:33:37.678000,1697.8671875 +2024-06-04 05:34:07.685000,1711.38671875 +2024-06-04 05:35:37.697000,1933.51171875 +2024-06-04 05:36:07.713000,1952.546875 +2024-06-04 05:36:37.721000,1962.08203125 +2024-06-04 05:37:07.722000,1857.82421875 +2024-06-04 05:37:37.726000,1834.546875 +2024-06-04 05:38:07.728000,1841.734375 +2024-06-04 05:39:37.755000,1806.94140625 +2024-06-04 05:40:07.759000,1800.9921875 +2024-06-04 05:41:07.768000,1814.44921875 +2024-06-04 05:42:37.797000,1718.9375 +2024-06-04 05:43:07.808000,1734.16796875 +2024-06-04 05:43:37.815000,1741.33984375 +2024-06-04 05:44:07.857000,1747.734375 +2024-06-04 05:45:37.901000,1764.03125 +2024-06-04 05:46:07.904000,1772.7265625 +2024-06-04 05:46:37.907000,1785.47265625 +2024-06-04 05:47:07.912000,1786.0703125 +2024-06-04 05:47:37.914000,1790.37890625 +2024-06-04 05:48:07.917000,1790.66796875 +2024-06-04 05:48:37.926000,1799.15234375 +2024-06-04 05:49:07.926000,1816.2578125 +2024-06-04 05:49:37.929000,1824.21875 +2024-06-04 05:50:07.934000,1832.9375 +2024-06-04 05:50:37.937000,1840.79296875 +2024-06-04 05:51:37.942000,1852.1875 +2024-06-04 05:53:08.005000,1875.44140625 +2024-06-04 05:53:38.005000,1877.84765625 +2024-06-04 05:54:08.009000,1888.6796875 +2024-06-04 05:55:08.011000,1865.66015625 +2024-06-04 05:55:38.011000,1887.671875 +2024-06-04 05:56:08.016000,1888.40234375 +2024-06-04 05:56:38.015000,1893.87890625 +2024-06-04 05:57:08.020000,1903.44140625 +2024-06-04 05:57:38.021000,1913.1875 +2024-06-04 05:58:38.028000,1930.99609375 +2024-06-04 05:59:38.039000,1948.5234375 +2024-06-04 06:00:38.043000,1962.734375 +2024-06-04 06:01:08.046000,1968.8828125 +2024-06-04 06:01:38.049000,2005.7421875 diff --git a/analysis/output_10b_run4_1s_flush_no_manual_garbage_yet.txt b/analysis/output_10b_run4_1s_flush_no_manual_garbage_yet.txt new file mode 100644 index 0000000..68996ca --- /dev/null +++ b/analysis/output_10b_run4_1s_flush_no_manual_garbage_yet.txt @@ -0,0 +1,361 @@ +worker1-1 | 2024/06/04 17:11:16.070 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 17:11:16.080 FIG10b: Starting tasks submission... +worker1-1 | 2024/06/04 17:11:17.088 FIG10b: Memory usage without flush after 126 tasks: 771.67578125 MB +worker1-1 | 2024/06/04 17:11:18.095 FIG10b: Memory usage without flush after 249 tasks: 772.46484375 MB +worker1-1 | 2024/06/04 17:11:19.103 FIG10b: Memory usage without flush after 369 tasks: 777.73046875 MB +worker1-1 | 2024/06/04 17:11:20.1102024/06/04 17:11:20.110 FIG10b: Memory usage without flush after 501 tasks: 778.734375 MB +worker1-1 | 2024/06/04 17:11:21.113 FIG10b: Memory usage without flush after 628 tasks: 780.4921875 MB +worker1-1 | 2024/06/04 17:11:22.118 2024/06/04 17:11:22.119 FIG10b: Memory usage without flush after 761 tasks: 781.50390625 MB +worker1-1 | 2024/06/04 17:11:23.125 2024/06/04 17:11:23.126 FIG10b: Memory usage without flush after 894 tasks: 783.9140625 MB +worker1-1 | 2024/06/04 17:11:24.133 FIG10b: Memory usage without flush after 1013 tasks: 784.97265625 MB +worker1-1 | 2024/06/04 17:11:25.1402024/06/04 17:11:25.140 FIG10b: Memory usage without flush after 1145 tasks: 785.98828125 MB +worker1-1 | 2024/06/04 17:11:26.149 2024/06/04 17:11:26.149 FIG10b: Memory usage without flush after 1264 tasks: 786.765625 MB +worker1-1 | 2024/06/04 17:11:27.150 FIG10b: Memory usage without flush after 1397 tasks: 791.703125 MB2024/06/04 17:11:27.151 [lobs] WERE GONNA CREATE A /DEV FILE WITH THIS NAME: /dev/shm/5157570045114249705 +worker1-1 | 2024/06/04 17:11:28.164 FIG10b: Memory usage without flush after 1522 tasks: 792.44921875 MB +worker1-1 | 2024/06/04 17:11:29.165 FIG10b: Memory usage without flush after 1640 tasks: 792.2421875 MB +worker1-1 | 2024/06/04 17:11:30.167[worker] FIG10b: Memory usage without flush after 1766 tasks: 798.19921875 MB +worker1-1 | 2024/06/04 17:11:31.176 FIG10b: Memory usage without flush after 1892 tasks: 798.71484375 MB +worker1-1 | 2024/06/04 17:11:32.178 took this many seconds: 0.00527FIG10b: Memory usage without flush after 2022 tasks: 799.47265625 MB +worker1-1 | 2024/06/04 17:11:33.1862024/06/04 17:11:33.186took this many seconds: 0.0114in Run() rn FIG10b: Memory usage without flush after 2151 tasks: 799.859375 MB +worker1-1 | 2024/06/04 17:11:34.188 FIG10b: Memory usage without flush after 2283 tasks: 799.8671875 MB +worker1-1 | 2024/06/04 17:11:35.192 FIG10b: Memory usage without flush after 2415 tasks: 804.99609375 MB +worker1-1 | [worker] Executed!2024/06/04 17:11:36.199 FIG10b: Memory usage without flush after 2542 tasks: 804.78515625 MB +worker1-1 | 2024/06/04 17:11:37.200 FIG10b: Memory usage without flush after 2669 tasks: 805.52734375 MB +worker1-1 | 2024/06/04 17:11:38.205 FIG10b: Memory usage without flush after 2803 tasks: 806.2421875 MB +worker1-1 | 2024/06/04 17:11:39.211 2024/06/04 17:11:39.211 [worker]2024/06/04 17:11:39.2122024/06/04 17:11:39.212 FIG10b: Memory usage without flush after 2929 tasks: 811.859375 MB +worker1-1 | 2024/06/04 17:11:40.2132024/06/04 17:11:40.213 FIG10b: Memory usage without flush after 3056 tasks: 813.1015625 MB +worker1-1 | 2024/06/04 17:11:41.218 FIG10b: Memory usage without flush after 3185 tasks: 818.4453125 MB +worker1-1 | 2024/06/04 17:11:42.225 FIG10b: Memory usage without flush after 3321 tasks: 819.41796875 MB +worker1-1 | 2024/06/04 17:11:43.230 FIG10b: Memory usage without flush after 3449 tasks: 820.62890625 MB +worker1-1 | 2024/06/04 17:11:44.234 FIG10b: Memory usage without flush after 3581 tasks: 820.8046875 MB +worker1-1 | 2024/06/04 17:11:45.235 FIG10b: Memory usage without flush after 3710 tasks: 821.234375 MB +worker1-1 | [worker][worker] 2024/06/04 17:11:46.238 FIG10b: Memory usage without flush after 3830 tasks: 821.2890625 MB +worker1-1 | 2024/06/04 17:11:47.243 FIG10b: Memory usage without flush after 3953 tasks: 821.52734375 MB +worker1-1 | 2024/06/04 17:11:48.249[worker] 2024/06/04 17:11:48.250 FIG10b: Memory usage without flush after 4081 tasks: 822.03125 MB +worker1-1 | 2024/06/04 17:11:49.2552024/06/04 17:11:49.256 FIG10b: Memory usage without flush after 4204 tasks: 822.7578125 MB +worker1-1 | 2024/06/04 17:11:50.262 FIG10b: Memory usage without flush after 4329 tasks: 823.5234375 MB +worker1-1 | 2024/06/04 17:11:51.266 FIG10b: Memory usage without flush after 4446 tasks: 823.375 MB +worker1-1 | 2024/06/04 17:11:52.278 2024/06/04 17:11:52.278 FIG10b: Memory usage without flush after 4574 tasks: 823.78125 MB +worker1-1 | 2024/06/04 17:11:53.287 FIG10b: Memory usage without flush after 4708 tasks: 824.54296875 MB +worker1-1 | 2024/06/04 17:11:54.287 FIG10b: Memory usage without flush after 4834 tasks: 824.58984375 MB +worker1-1 | 2024/06/04 17:11:55.293 FIG10b: Memory usage without flush after 4964 tasks: 824.53515625 MB +worker1-1 | 2024/06/04 17:11:56.300 FIG10b: Memory usage without flush after 5104 tasks: 833.94140625 MB +worker1-1 | 2024/06/04 17:11:57.309 FIG10b: Memory usage without flush after 5233 tasks: 833.9296875 MB +worker1-1 | 2024/06/04 17:11:58.309 FIG10b: Memory usage without flush after 5366 tasks: 837.75 MB +worker1-1 | 2024/06/04 17:11:59.312 FIG10b: Memory usage without flush after 5482 tasks: 833.55859375 MB +worker1-1 | 2024/06/04 17:12:00.3132024/06/04 17:12:00.314 FIG10b: Memory usage without flush after 5612 tasks: 838.21875 MB +worker1-1 | 2024/06/04 17:12:01.319 FIG10b: Memory usage without flush after 5740 tasks: 838.6328125 MB +worker1-1 | 2024/06/04 17:12:02.3242024/06/04 17:12:02.325 FIG10b: Memory usage without flush after 5872 tasks: 839.59375 MB +worker1-1 | 2024/06/04 17:12:03.325 FIG10b: Memory usage without flush after 6004 tasks: 840.375 MB +worker1-1 | 2024/06/04 17:12:04.331 2024/06/04 17:12:04.332 FIG10b: Memory usage without flush after 6134 tasks: 840.078125 MB +worker1-1 | 2024/06/04 17:12:05.335 FIG10b: Memory usage without flush after 6256 tasks: 841.01953125 MB +worker1-1 | 2024/06/04 17:12:06.3352024/06/04 17:12:06.335 FIG10b: Memory usage without flush after 6386 tasks: 841.3125 MB +worker1-1 | 2024/06/04 17:12:07.338 2024/06/04 17:12:07.338 FIG10b: Memory usage without flush after 6513 tasks: 841.9609375 MB +worker1-1 | 2024/06/04 17:12:08.3442024/06/04 17:12:08.344 FIG10b: Memory usage without flush after 6642 tasks: 841.26171875 MB +worker1-1 | 2024/06/04 17:12:09.350 2024/06/04 17:12:09.351 FIG10b: Memory usage without flush after 6774 tasks: 841.546875 MB +worker1-1 | 2024/06/04 17:12:10.356 FIG10b: Memory usage without flush after 6904 tasks: 842.140625 MB +worker1-1 | 2024/06/04 17:12:11.361 FIG10b: Memory usage without flush after 7030 tasks: 842.35546875 MB +worker1-1 | 2024/06/04 17:12:12.366 FIG10b: Memory usage without flush after 7164 tasks: 842.37890625 MB +worker1-1 | 2024/06/04 17:12:13.3722024/06/04 17:12:13.373 FIG10b: Memory usage without flush after 7302 tasks: 842.95703125 MB +worker1-1 | [worker] 2024/06/04 17:12:14.374 FIG10b: Memory usage without flush after 7427 tasks: 843.984375 MB +worker1-1 | 2024/06/04 17:12:15.383 FIG10b: Memory usage without flush after 7558 tasks: 843.76953125 MB +worker1-1 | 2024/06/04 17:12:16.390 FIG10b: Memory usage without flush after 7683 tasks: 845.1796875 MB +worker1-1 | 2024/06/04 17:12:17.394 FIG10b: Memory usage without flush after 7812 tasks: 844.82421875 MB +worker1-1 | 2024/06/04 17:12:18.398 FIG10b: Memory usage without flush after 7949 tasks: 845.65625 MB +worker1-1 | 2024/06/04 17:12:19.410 FIG10b: Memory usage without flush after 8088 tasks: 845.83203125 MB +worker1-1 | 2024/06/04 17:12:20.4232024/06/04 17:12:20.423 FIG10b: Memory usage without flush after 8218 tasks: 845.875 MB +worker1-1 | 2024/06/04 17:12:21.4262024/06/04 17:12:21.426 FIG10b: Memory usage without flush after 8348 tasks: 848.12109375 MB +worker1-1 | 2024/06/04 17:12:22.429 FIG10b: Memory usage without flush after 8474 tasks: 848.828125 MB +worker1-1 | took this many seconds: 0.007982024/06/04 17:12:23.4332024/06/04 17:12:23.433 FIG10b: Memory usage without flush after 8600 tasks: 849.41015625 MB +worker1-1 | 2024/06/04 17:12:24.434 FIG10b: Memory usage without flush after 8725 tasks: 850.30078125 MB +worker1-1 | 2024/06/04 17:12:25.438 FIG10b: Memory usage without flush after 8841 tasks: 851.3046875 MB +worker1-1 | 2024/06/04 17:12:26.443 FIG10b: Memory usage without flush after 8959 tasks: 851.3515625 MB +worker1-1 | 2024/06/04 17:12:27.445 FIG10b: Memory usage without flush after 9095 tasks: 852.7578125 MB +worker1-1 | 2024/06/04 17:12:28.457 FIG10b: Memory usage without flush after 9226 tasks: 853.6328125 MB +worker1-1 | 2024/06/04 17:12:29.458 FIG10b: Memory usage without flush after 9353 tasks: 854.24609375 MB +worker1-1 | FIG10b: Memory usage without flush after 9487 tasks: 854.7890625 MB +worker1-1 | 2024/06/04 17:12:31.473 FIG10b: Memory usage without flush after 9616 tasks: 855.4375 MB +worker1-1 | 2024/06/04 17:12:32.480 FIG10b: Memory usage without flush after 9756 tasks: 855.609375 MB +worker1-1 | 2024/06/04 17:12:33.491 FIG10b: Memory usage without flush after 9894 tasks: 855.484375 MB +worker1-1 | FIG10b: Memory usage without flush after 10030 tasks: 856.63671875 MB +worker1-1 | 2024/06/04 17:12:35.495 FIG10b: Memory usage without flush after 10169 tasks: 857.609375 MB +worker1-1 | 2024/06/04 17:12:36.496 FIG10b: Memory usage without flush after 10298 tasks: 858.7734375 MB +worker1-1 | 2024/06/04 17:12:37.499 FIG10b: Memory usage without flush after 10432 tasks: 859.19140625 MB +worker1-1 | 2024/06/04 17:12:38.504 FIG10b: Memory usage without flush after 10563 tasks: 859.2421875 MB +worker1-1 | [worker] 2024/06/04 17:12:39.508 FIG10b: Memory usage without flush after 10700 tasks: 860.328125 MB +worker1-1 | [worker]2024/06/04 17:12:40.515 FIG10b: Memory usage without flush after 10829 tasks: 860.34765625 MB +worker1-1 | 2024/06/04 17:12:41.517 FIG10b: Memory usage without flush after 10960 tasks: 867.90234375 MB +worker1-1 | 2024/06/04 17:12:42.522 2024/06/04 17:12:42.522FIG10b: Memory usage without flush after 11093 tasks: 868.18359375 MB +worker1-1 | 2024/06/04 17:12:43.5222024/06/04 17:12:43.523 FIG10b: Memory usage without flush after 11225 tasks: 868.25 MB[worker] +worker1-1 | 2024/06/04 17:12:44.530 2024/06/04 17:12:44.531 FIG10b: Memory usage without flush after 11360 tasks: 868.62890625 MB +worker1-1 | 2024/06/04 17:12:45.536 FIG10b: Memory usage without flush after 11490 tasks: 869.44921875 MB +worker1-1 | FIG10b: Memory usage without flush after 11618 tasks: 871.24609375 MB +worker1-1 | gonna store now2024/06/04 17:12:47.548 2024/06/04 17:12:47.548 FIG10b: Memory usage without flush after 11756 tasks: 871.8515625 MB +worker1-1 | 2024/06/04 17:12:48.554Executed!2024/06/04 17:12:48.555 FIG10b: Memory usage without flush after 11897 tasks: 876.53125 MB +worker1-1 | 2024/06/04 17:12:49.556 FIG10b: Memory usage without flush after 12034 tasks: 878.81640625 MB +worker1-1 | [worker] gonna store now2024/06/04 17:12:50.557 FIG10b: Memory usage without flush after 12161 tasks: 878.62109375 MB +worker1-1 | 2024/06/04 17:12:51.562 [worker]2024/06/04 17:12:51.563 FIG10b: Memory usage without flush after 12297 tasks: 879.7421875 MB +worker1-1 | 2024/06/04 17:12:52.568 FIG10b: Memory usage without flush after 12428 tasks: 880.1171875 MB +worker1-1 | 2024/06/04 17:12:53.576 FIG10b: Memory usage without flush after 12565 tasks: 880.9609375 MB +worker1-1 | 2024/06/04 17:12:54.588 [worker]2024/06/04 17:12:54.588 FIG10b: Memory usage without flush after 12703 tasks: 880.77734375 MB +worker1-1 | 2024/06/04 17:12:55.5912024/06/04 17:12:55.591 FIG10b: Memory usage without flush after 12835 tasks: 880.2109375 MB +worker1-1 | 2024/06/04 17:12:56.598 2024/06/04 17:12:56.598 FIG10b: Memory usage without flush after 12971 tasks: 882.234375 MB +worker1-1 | 2024/06/04 17:12:57.600 2024/06/04 17:12:57.600 FIG10b: Memory usage without flush after 13118 tasks: 881.69921875 MB +worker1-1 | 2024/06/04 17:12:58.605 FIG10b: Memory usage without flush after 13250 tasks: 882.234375 MB +worker1-1 | 2024/06/04 17:12:59.605 FIG10b: Memory usage without flush after 13387 tasks: 882.9921875 MB +worker1-1 | 2024/06/04 17:13:00.605 2024/06/04 17:13:00.606 FIG10b: Memory usage without flush after 13517 tasks: 883.83984375 MB +worker1-1 | 2024/06/04 17:13:01.608 [worker] in Run() rn2024/06/04 17:13:01.608 FIG10b: Memory usage without flush after 13647 tasks: 884.6875 MB +worker1-1 | 2024/06/04 17:13:02.612 2024/06/04 17:13:02.612 FIG10b: Memory usage without flush after 13784 tasks: 886.5 MB +worker1-1 | 2024/06/04 17:13:03.6182024/06/04 17:13:03.618 FIG10b: Memory usage without flush after 13909 tasks: 887.3125 MB +worker1-1 | 2024/06/04 17:13:04.6192024/06/04 17:13:04.620 FIG10b: Memory usage without flush after 14044 tasks: 887.86328125 MB +worker1-1 | 2024/06/04 17:13:05.623 FIG10b: Memory usage without flush after 14174 tasks: 888.8828125 MB +worker1-1 | 2024/06/04 17:13:06.627 FIG10b: Memory usage without flush after 14303 tasks: 889.046875 MB +worker1-1 | 2024/06/04 17:13:07.631 FIG10b: Memory usage without flush after 14427 tasks: 886.82421875 MB +worker1-1 | 2024/06/04 17:13:08.634 FIG10b: Memory usage without flush after 14576 tasks: 890.421875 MB +worker1-1 | 2024/06/04 17:13:09.635 FIG10b: Memory usage without flush after 14715 tasks: 891.55859375 MB +worker1-1 | 2024/06/04 17:13:10.639 FIG10b: Memory usage without flush after 14862 tasks: 892.66796875 MB +worker1-1 | 2024/06/04 17:13:11.639 FIG10b: Memory usage without flush after 14992 tasks: 893.765625 MB +worker1-1 | 2024/06/04 17:13:12.640 FIG10b: Memory usage without flush after 15124 tasks: 898.5859375 MB +worker1-1 | 2024/06/04 17:13:13.643 FIG10b: Memory usage without flush after 15256 tasks: 898.77734375 MB +worker1-1 | 2024/06/04 17:13:14.649 FIG10b: Memory usage without flush after 15394 tasks: 901.1171875 MB +worker1-1 | 2024/06/04 17:13:15.652 FIG10b: Memory usage without flush after 15512 tasks: 902.05078125 MB +worker1-1 | 2024/06/04 17:13:16.655 FIG10b: Memory usage without flush after 15648 tasks: 903.03125 MB +worker1-1 | 2024/06/04 17:13:17.659 FIG10b: Memory usage without flush after 15791 tasks: 903.9140625 MB +worker1-1 | 2024/06/04 17:13:18.662 FIG10b: Memory usage without flush after 15927 tasks: 904.69140625 MB +worker1-1 | 2024/06/04 17:13:19.666 FIG10b: Memory usage without flush after 16055 tasks: 904.06640625 MB +worker1-1 | 2024/06/04 17:13:20.667 FIG10b: Memory usage without flush after 16176 tasks: 905.9140625 MB +worker1-1 | 2024/06/04 17:13:21.668 FIG10b: Memory usage without flush after 16312 tasks: 906.046875 MB +worker1-1 | 2024/06/04 17:13:22.672 FIG10b: Memory usage without flush after 16445 tasks: 906.39453125 MB +worker1-1 | 2024/06/04 17:13:23.673 FIG10b: Memory usage without flush after 16585 tasks: 908.36328125 MB +worker1-1 | 2024/06/04 17:13:24.674 FIG10b: Memory usage without flush after 16717 tasks: 907.32421875 MB +worker1-1 | 2024/06/04 17:13:25.677 FIG10b: Memory usage without flush after 16850 tasks: 908.99609375 MB +worker1-1 | 2024/06/04 17:13:26.680 FIG10b: Memory usage without flush after 16983 tasks: 909.5703125 MB +worker1-1 | 2024/06/04 17:13:27.684 FIG10b: Memory usage without flush after 17125 tasks: 911.25 MB +worker1-1 | 2024/06/04 17:13:28.685 FIG10b: Memory usage without flush after 17254 tasks: 912.15234375 MB +worker1-1 | 2024/06/04 17:13:29.689 FIG10b: Memory usage without flush after 17387 tasks: 912.078125 MB +worker1-1 | 2024/06/04 17:13:30.691 FIG10b: Memory usage without flush after 17530 tasks: 916.59765625 MB +worker1-1 | 2024/06/04 17:13:31.693 FIG10b: Memory usage without flush after 17671 tasks: 913.24609375 MB +worker1-1 | 2024/06/04 17:13:32.695 FIG10b: Memory usage without flush after 17809 tasks: 914.359375 MB +worker1-1 | 2024/06/04 17:13:33.695 FIG10b: Memory usage without flush after 17953 tasks: 915.4296875 MB +worker1-1 | 2024/06/04 17:13:34.696 FIG10b: Memory usage without flush after 18093 tasks: 916.84765625 MB +worker1-1 | 2024/06/04 17:13:35.697 FIG10b: Memory usage without flush after 18229 tasks: 916.2265625 MB +worker1-1 | 2024/06/04 17:13:36.697 FIG10b: Memory usage without flush after 18362 tasks: 917.09375 MB +worker1-1 | 2024/06/04 17:13:37.701 FIG10b: Memory usage without flush after 18488 tasks: 917.9921875 MB +worker1-1 | 2024/06/04 17:13:38.701 FIG10b: Memory usage without flush after 18629 tasks: 918.15625 MB +worker1-1 | 2024/06/04 17:13:39.705 FIG10b: Memory usage without flush after 18775 tasks: 919.9140625 MB +worker1-1 | 2024/06/04 17:13:40.711 FIG10b: Memory usage without flush after 18917 tasks: 920.96484375 MB +worker1-1 | 2024/06/04 17:13:41.710 FIG10b: Memory usage without flush after 19046 tasks: 921.4609375 MB +worker1-1 | 2024/06/04 17:13:42.713 FIG10b: Memory usage without flush after 19186 tasks: 925.78515625 MB +worker1-1 | 2024/06/04 17:13:43.717 FIG10b: Memory usage without flush after 19324 tasks: 927.91015625 MB +worker1-1 | 2024/06/04 17:13:44.717 FIG10b: Memory usage without flush after 19458 tasks: 925.828125 MB +worker1-1 | 2024/06/04 17:13:45.721 FIG10b: Memory usage without flush after 19583 tasks: 926.21484375 MB +worker1-1 | 2024/06/04 17:13:46.739 FIG10b: Memory usage without flush after 19721 tasks: 933.5078125 MB +worker1-1 | 2024/06/04 17:13:47.741 FIG10b: Memory usage without flush after 19870 tasks: 934.7421875 MB +worker1-1 | 2024/06/04 17:13:48.743 FIG10b: Memory usage without flush after 20006 tasks: 935.6875 MB +worker1-1 | 2024/06/04 17:13:49.7452024/06/04 17:13:49.746 FIG10b: Memory usage without flush after 20152 tasks: 937.8984375 MB +worker1-1 | 2024/06/04 17:13:50.748 FIG10b: Memory usage without flush after 20291 tasks: 937.63671875 MB +worker1-1 | 2024/06/04 17:13:51.753 FIG10b: Memory usage without flush after 20420 tasks: 938.9921875 MB +worker1-1 | 2024/06/04 17:13:52.756 FIG10b: Memory usage without flush after 20558 tasks: 939.62890625 MB +worker1-1 | 2024/06/04 17:13:53.757 FIG10b: Memory usage without flush after 20695 tasks: 939.671875 MB +worker1-1 | 2024/06/04 17:13:54.786 2024/06/04 17:13:54.787 FIG10b: Memory usage without flush after 20825 tasks: 938.67578125 MB +worker1-1 | 2024/06/04 17:13:55.797 FIG10b: Memory usage without flush after 20958 tasks: 938.77734375 MB +worker1-1 | 2024/06/04 17:13:56.799 FIG10b: Memory usage without flush after 21109 tasks: 939.7734375 MB +worker1-1 | 2024/06/04 17:13:57.808 [worker] 2024/06/04 17:13:57.808 FIG10b: Memory usage without flush after 21232 tasks: 940.71875 MB +worker1-1 | 2024/06/04 17:13:58.811 FIG10b: Memory usage without flush after 21366 tasks: 940.22265625 MB +worker1-1 | 2024/06/04 17:13:59.813 FIG10b: Memory usage without flush after 21504 tasks: 941.5234375 MB +worker1-1 | 2024/06/04 17:14:00.835 FIG10b: Memory usage without flush after 21649 tasks: 941.66796875 MB +worker1-1 | 2024/06/04 17:14:01.838 FIG10b: Memory usage without flush after 21793 tasks: 942.91015625 MB +worker1-1 | 2024/06/04 17:14:02.840 FIG10b: Memory usage without flush after 21935 tasks: 941.91796875 MB +worker1-1 | 2024/06/04 17:14:03.842 FIG10b: Memory usage without flush after 22080 tasks: 951.1796875 MB +worker1-1 | [worker] 2024/06/04 17:14:04.848 FIG10b: Memory usage without flush after 22220 tasks: 952.33984375 MB +worker1-1 | 2024/06/04 17:14:05.855 FIG10b: Memory usage without flush after 22355 tasks: 950.19140625 MB +worker1-1 | [worker]2024/06/04 17:14:06.870 FIG10b: Memory usage without flush after 22496 tasks: 951.20703125 MB +worker1-1 | 2024/06/04 17:14:07.874 FIG10b: Memory usage without flush after 22631 tasks: 952.87890625 MB +worker1-1 | 2024/06/04 17:14:08.877 FIG10b: Memory usage without flush after 22778 tasks: 960.58203125 MB +worker1-1 | 2024/06/04 17:14:09.8902024/06/04 17:14:09.890 FIG10b: Memory usage without flush after 22916 tasks: 960.83203125 MB +worker1-1 | stored...2024/06/04 17:14:10.890 FIG10b: Memory usage without flush after 23064 tasks: 959.015625 MB +worker1-1 | 2024/06/04 17:14:11.895 FIG10b: Memory usage without flush after 23200 tasks: 960.03125 MB +worker1-1 | 2024/06/04 17:14:12.898 FIG10b: Memory usage without flush after 23341 tasks: 959.296875 MB +worker1-1 | 2024/06/04 17:14:13.900 FIG10b: Memory usage without flush after 23467 tasks: 959.8125 MB +worker1-1 | 2024/06/04 17:14:14.904 FIG10b: Memory usage without flush after 23623 tasks: 961.9375 MB +worker1-1 | 2024/06/04 17:14:15.944 FIG10b: Memory usage without flush after 23780 tasks: 962.25 MB +worker1-1 | [worker]2024/06/04 17:14:16.950 FIG10b: Memory usage without flush after 23890 tasks: 964.62109375 MB +worker1-1 | stored...2024/06/04 17:14:17.9542024/06/04 17:14:17.9542024/06/04 17:14:17.954 FIG10b: Memory usage without flush after 24017 tasks: 960.94140625 MB +worker1-1 | 2024/06/04 17:14:18.961 FIG10b: Memory usage without flush after 24174 tasks: 965.21875 MB +worker1-1 | 2024/06/04 17:14:19.9652024/06/04 17:14:19.966 FIG10b: Memory usage without flush after 24288 tasks: 973.1640625 MB +worker1-1 | 2024/06/04 17:14:20.968 FIG10b: Memory usage without flush after 24411 tasks: 972.45703125 MB +worker1-1 | 2024/06/04 17:14:21.981 FIG10b: Memory usage without flush after 24550 tasks: 972.38671875 MB +worker1-1 | 2024/06/04 17:14:22.984 FIG10b: Memory usage without flush after 24688 tasks: 973.90625 MB +worker1-1 | 2024/06/04 17:14:23.9872024/06/04 17:14:23.987 FIG10b: Memory usage without flush after 24821 tasks: 980.11328125 MB +worker1-1 | 2024/06/04 17:14:24.989 FIG10b: Memory usage without flush after 24953 tasks: 984.48828125 MB +worker1-1 | 2024/06/04 17:14:25.994 FIG10b: Memory usage without flush after 25094 tasks: 985.94921875 MB +worker1-1 | 2024/06/04 17:14:26.994 FIG10b: Memory usage without flush after 25245 tasks: 985.22265625 MB +worker1-1 | 2024/06/04 17:14:27.997 2024/06/04 17:14:27.997 FIG10b: Memory usage without flush after 25379 tasks: 986.171875 MB +worker1-1 | 2024/06/04 17:14:29.000 2024/06/04 17:14:29.000[worker] FIG10b: Memory usage without flush after 25512 tasks: 988.28515625 MB +worker1-1 | 2024/06/04 17:14:30.004 FIG10b: Memory usage without flush after 25657 tasks: 987.83984375 MB +worker1-1 | 2024/06/04 17:14:31.006 FIG10b: Memory usage without flush after 25792 tasks: 989.4453125 MB +worker1-1 | 2024/06/04 17:14:32.0112024/06/04 17:14:32.012 FIG10b: Memory usage without flush after 25904 tasks: 989.56640625 MB +worker1-1 | 2024/06/04 17:14:33.016 FIG10b: Memory usage without flush after 26035 tasks: 991.87109375 MB +worker1-1 | 2024/06/04 17:14:34.018 FIG10b: Memory usage without flush after 26175 tasks: 987.86328125 MB +worker1-1 | [worker][worker]2024/06/04 17:14:35.024 FIG10b: Memory usage without flush after 26321 tasks: 988.51171875 MB +worker1-1 | 2024/06/04 17:14:36.030 [worker]2024/06/04 17:14:36.030 FIG10b: Memory usage without flush after 26463 tasks: 989.91796875 MB +worker1-1 | [worker]2024/06/04 17:14:37.039 FIG10b: Memory usage without flush after 26598 tasks: 989.5234375 MB +worker1-1 | 2024/06/04 17:14:38.0472024/06/04 17:14:38.047[worker] 2024/06/04 17:14:38.048 FIG10b: Memory usage without flush after 26735 tasks: 990.3125 MB +worker1-1 | 2024/06/04 17:14:39.051 FIG10b: Memory usage without flush after 26885 tasks: 990.1796875 MB +worker1-1 | 2024/06/04 17:14:40.059 FIG10b: Memory usage without flush after 27007 tasks: 992.046875 MB +worker1-1 | FIG10b: Memory usage without flush after 27135 tasks: 991.65234375 MB +worker1-1 | 2024/06/04 17:14:42.063 2024/06/04 17:14:42.0632024/06/04 17:14:42.063 FIG10b: Memory usage without flush after 27258 tasks: 992.27734375 MB +worker1-1 | 2024/06/04 17:14:43.066 FIG10b: Memory usage without flush after 27388 tasks: 993.0703125 MB +worker1-1 | 2024/06/04 17:14:44.0702024/06/04 17:14:44.071 FIG10b: Memory usage without flush after 27525 tasks: 992.69921875 MB +worker1-1 | 2024/06/04 17:14:45.072 FIG10b: Memory usage without flush after 27664 tasks: 992.83984375 MB +worker1-1 | 2024/06/04 17:14:46.079 2024/06/04 17:14:46.080 FIG10b: Memory usage without flush after 27793 tasks: 993.9609375 MB +worker1-1 | 2024/06/04 17:14:47.084 2024/06/04 17:14:47.085 FIG10b: Memory usage without flush after 27925 tasks: 996.33203125 MB +worker1-1 | 2024/06/04 17:14:48.097 FIG10b: Memory usage without flush after 28056 tasks: 996.55078125 MB +worker1-1 | 2024/06/04 17:14:49.107 FIG10b: Memory usage without flush after 28188 tasks: 997.78125 MB +worker1-1 | 2024/06/04 17:14:50.107 2024/06/04 17:14:50.107 FIG10b: Memory usage without flush after 28327 tasks: 998.1953125 MB +worker1-1 | 2024/06/04 17:14:51.113 FIG10b: Memory usage without flush after 28465 tasks: 999.609375 MB +worker1-1 | 2024/06/04 17:14:52.114 2024/06/04 17:14:52.114FIG10b: Memory usage without flush after 28601 tasks: 1001.1484375 MB +worker1-1 | 2024/06/04 17:14:53.121 FIG10b: Memory usage without flush after 28735 tasks: 1001.94140625 MB +worker1-1 | 2024/06/04 17:14:54.1212024/06/04 17:14:54.121 FIG10b: Memory usage without flush after 28882 tasks: 1001.57421875 MB +worker1-1 | 2024/06/04 17:14:55.1282024/06/04 17:14:55.129 FIG10b: Memory usage without flush after 29017 tasks: 998.4765625 MB +worker1-1 | 2024/06/04 17:14:56.132 FIG10b: Memory usage without flush after 29170 tasks: 999.78125 MB +worker1-1 | 2024/06/04 17:14:57.134 FIG10b: Memory usage without flush after 29320 tasks: 1003.5390625 MB +worker1-1 | 2024/06/04 17:14:58.139 FIG10b: Memory usage without flush after 29475 tasks: 1003.18359375 MB +worker1-1 | 2024/06/04 17:14:59.1422024/06/04 17:14:59.143 FIG10b: Memory usage without flush after 29611 tasks: 1001.6875 MB +worker1-1 | 2024/06/04 17:15:00.1482024/06/04 17:15:00.149 FIG10b: Memory usage without flush after 29740 tasks: 1002.1796875 MB +worker1-1 | 2024/06/04 17:15:01.152 FIG10b: Memory usage without flush after 29871 tasks: 1002.89453125 MB +worker1-1 | 2024/06/04 17:15:02.159 FIG10b: Memory usage without flush after 30003 tasks: 1002.61328125 MB +worker1-1 | 2024/06/04 17:15:03.1652024/06/04 17:15:03.165 FIG10b: Memory usage without flush after 30126 tasks: 1003.5625 MB +worker1-1 | 2024/06/04 17:15:04.173 FIG10b: Memory usage without flush after 30264 tasks: 1007.06640625 MB +worker1-1 | 2024/06/04 17:15:05.1752024/06/04 17:15:05.175 FIG10b: Memory usage without flush after 30402 tasks: 1001.265625 MB +worker1-1 | 2024/06/04 17:15:06.178 FIG10b: Memory usage without flush after 30538 tasks: 1000.1328125 MB +worker1-1 | 2024/06/04 17:15:07.182 FIG10b: Memory usage without flush after 30687 tasks: 1002.62890625 MB +worker1-1 | 2024/06/04 17:15:08.185 FIG10b: Memory usage without flush after 30824 tasks: 1003.28515625 MB +worker1-1 | 2024/06/04 17:15:09.185 FIG10b: Memory usage without flush after 30951 tasks: 1005.31640625 MB +worker1-1 | 2024/06/04 17:15:10.191 FIG10b: Memory usage without flush after 31091 tasks: 1003.46484375 MB +worker1-1 | 2024/06/04 17:15:11.204 FIG10b: Memory usage without flush after 31237 tasks: 1012.5546875 MB +worker1-1 | 2024/06/04 17:15:12.209 FIG10b: Memory usage without flush after 31380 tasks: 1014.01171875 MB2024/06/04 17:15:12.209 +worker1-1 | 2024/06/04 17:15:13.211 FIG10b: Memory usage without flush after 31522 tasks: 1014.55078125 MB +worker1-1 | 2024/06/04 17:15:14.220 [worker]2024/06/04 17:15:14.220 FIG10b: Memory usage without flush after 31666 tasks: 1017.1484375 MB +worker1-1 | 2024/06/04 17:15:15.221 FIG10b: Memory usage without flush after 31812 tasks: 1015.99609375 MB +worker1-1 | [worker] 2024/06/04 17:15:16.2242024/06/04 17:15:16.225 FIG10b: Memory usage without flush after 31947 tasks: 1018.33984375 MB +worker1-1 | 2024/06/04 17:15:17.234 FIG10b: Memory usage without flush after 32078 tasks: 1020.75 MB +worker1-1 | 2024/06/04 17:15:18.2352024/06/04 17:15:18.235 FIG10b: Memory usage without flush after 32218 tasks: 1022.55859375 MB +worker1-1 | 2024/06/04 17:15:19.236 FIG10b: Memory usage without flush after 32362 tasks: 1023.15625 MB +worker1-1 | 2024/06/04 17:15:20.239 FIG10b: Memory usage without flush after 32504 tasks: 1023.6484375 MB +worker1-1 | 2024/06/04 17:15:21.2422024/06/04 17:15:21.243 [worker] FIG10b: Memory usage without flush after 32650 tasks: 1024.34765625 MBExecuted! +worker1-1 | 2024/06/04 17:15:22.244 FIG10b: Memory usage without flush after 32795 tasks: 1026.296875 MB +worker1-1 | 2024/06/04 17:15:23.250 FIG10b: Memory usage without flush after 32937 tasks: 1028.3828125 MB +worker1-1 | 2024/06/04 17:15:24.256 FIG10b: Memory usage without flush after 33068 tasks: 1030.3125 MB +worker1-1 | [worker] stored...2024/06/04 17:15:25.266 FIG10b: Memory usage without flush after 33210 tasks: 1029.61328125 MB +worker1-1 | 2024/06/04 17:15:26.274 FIG10b: Memory usage without flush after 33338 tasks: 1028.609375 MB +worker1-1 | 2024/06/04 17:15:27.2822024/06/04 17:15:27.282 FIG10b: Memory usage without flush after 33480 tasks: 1027.95703125 MB +worker1-1 | 2024/06/04 17:15:28.292 FIG10b: Memory usage without flush after 33621 tasks: 1028.49609375 MB +worker1-1 | 2024/06/04 17:15:29.3032024/06/04 17:15:29.303 FIG10b: Memory usage without flush after 33753 tasks: 1027.9921875 MB +worker1-1 | [worker]2024/06/04 17:15:30.304 FIG10b: Memory usage without flush after 33911 tasks: 1030.65625 MB +worker1-1 | 2024/06/04 17:15:31.306 FIG10b: Memory usage without flush after 34036 tasks: 1031.43359375 MB +worker1-1 | FIG10b: Memory usage without flush after 34185 tasks: 1032.24609375 MB +worker1-1 | 2024/06/04 17:15:33.324 FIG10b: Memory usage without flush after 34322 tasks: 1032.0 MB +worker1-1 | 2024/06/04 17:15:34.326 FIG10b: Memory usage without flush after 34462 tasks: 1032.36328125 MB +worker1-1 | 2024/06/04 17:15:35.326 2024/06/04 17:15:35.327 FIG10b: Memory usage without flush after 34594 tasks: 1033.53125 MB +worker1-1 | 2024/06/04 17:15:36.327 FIG10b: Memory usage without flush after 34736 tasks: 1032.14453125 MB +worker1-1 | [worker]in Run() rn 2024/06/04 17:15:37.331 FIG10b: Memory usage without flush after 34871 tasks: 1031.0546875 MB +worker1-1 | 2024/06/04 17:15:38.339 FIG10b: Memory usage without flush after 35021 tasks: 1033.36328125 MB +worker1-1 | 2024/06/04 17:15:39.3482024/06/04 17:15:39.349 FIG10b: Memory usage without flush after 35166 tasks: 1032.73828125 MB +worker1-1 | 2024/06/04 17:15:40.352 FIG10b: Memory usage without flush after 35261 tasks: 1029.91796875 MB +worker1-1 | 2024/06/04 17:15:41.352 FIG10b: Memory usage without flush after 35391 tasks: 1027.015625 MB +worker1-1 | gonna store now2024/06/04 17:15:42.364 FIG10b: Memory usage without flush after 35543 tasks: 1025.765625 MB +worker1-1 | 2024/06/04 17:15:43.367 FIG10b: Memory usage without flush after 35694 tasks: 1032.625 MB +worker1-1 | 2024/06/04 17:15:44.376 FIG10b: Memory usage without flush after 35830 tasks: 1038.8828125 MB +worker1-1 | 2024/06/04 17:15:45.498[worker] stored...FIG10b: Memory usage without flush after 35970 tasks: 1040.40234375 MB +worker1-1 | 2024/06/04 17:15:46.502 FIG10b: Memory usage without flush after 36112 tasks: 1041.7421875 MB +worker1-1 | 2024/06/04 17:15:47.5042024/06/04 17:15:47.5052024/06/04 17:15:47.505 FIG10b: Memory usage without flush after 36268 tasks: 1035.16796875 MB +worker1-1 | 2024/06/04 17:15:48.5092024/06/04 17:15:48.509 2024/06/04 17:15:48.510 FIG10b: Memory usage without flush after 36393 tasks: 1038.96484375 MB +worker1-1 | 2024/06/04 17:15:49.512 [worker]2024/06/04 17:15:49.513 FIG10b: Memory usage without flush after 36532 tasks: 1039.12890625 MB +worker1-1 | 2024/06/04 17:15:50.519 FIG10b: Memory usage without flush after 36667 tasks: 1039.62109375 MB +worker1-1 | 2024/06/04 17:15:51.522 FIG10b: Memory usage without flush after 36814 tasks: 1039.44921875 MB +worker1-1 | 2024/06/04 17:15:52.524 FIG10b: Memory usage without flush after 36942 tasks: 1041.8984375 MB +worker1-1 | FIG10b: Memory usage without flush after 37074 tasks: 1042.37109375 MB +worker1-1 | 2024/06/04 17:15:54.541 FIG10b: Memory usage without flush after 37204 tasks: 1039.86328125 MB +worker1-1 | 2024/06/04 17:15:55.548 FIG10b: Memory usage without flush after 37342 tasks: 1036.9609375 MB +worker1-1 | 2024/06/04 17:15:56.552 FIG10b: Memory usage without flush after 37472 tasks: 1038.1953125 MB +worker1-1 | 2024/06/04 17:15:57.563 FIG10b: Memory usage without flush after 37612 tasks: 1038.1171875 MB +worker1-1 | 2024/06/04 17:15:58.577 FIG10b: Memory usage without flush after 37740 tasks: 1039.08984375 MB +worker1-1 | [worker]2024/06/04 17:15:59.582 FIG10b: Memory usage without flush after 37885 tasks: 1041.8984375 MB +worker1-1 | 2024/06/04 17:16:00.588 FIG10b: Memory usage without flush after 38018 tasks: 1045.99609375 MB +worker1-1 | 2024/06/04 17:16:01.5942024/06/04 17:16:01.594 FIG10b: Memory usage without flush after 38155 tasks: 1048.20703125 MB +worker1-1 | Executed!2024/06/04 17:16:02.597 FIG10b: Memory usage without flush after 38297 tasks: 1046.03515625 MB +worker1-1 | 2024/06/04 17:16:03.597 FIG10b: Memory usage without flush after 38446 tasks: 1052.859375 MB +worker1-1 | 2024/06/04 17:16:04.600 FIG10b: Memory usage without flush after 38574 tasks: 1053.140625 MB +worker1-1 | 2024/06/04 17:16:05.602 FIG10b: Memory usage without flush after 38708 tasks: 1055.57421875 MB +worker1-1 | 2024/06/04 17:16:06.605 FIG10b: Memory usage without flush after 38842 tasks: 1054.19921875 MB +worker1-1 | 2024/06/04 17:16:07.6082024/06/04 17:16:07.609 FIG10b: Memory usage without flush after 38979 tasks: 1060.4921875 MB +worker1-1 | 2024/06/04 17:16:08.6222024/06/04 17:16:08.623 FIG10b: Memory usage without flush after 39112 tasks: 1065.421875 MB +worker1-1 | 2024/06/04 17:16:09.632 FIG10b: Memory usage without flush after 39233 tasks: 1069.72265625 MB +worker1-1 | 2024/06/04 17:16:10.634 FIG10b: Memory usage without flush after 39369 tasks: 1069.12890625 MB +worker1-1 | 2024/06/04 17:16:11.636 FIG10b: Memory usage without flush after 39500 tasks: 1071.12890625 MB +worker1-1 | 2024/06/04 17:16:12.638 FIG10b: Memory usage without flush after 39640 tasks: 1072.25 MB +worker1-1 | 2024/06/04 17:16:13.640 FIG10b: Memory usage without flush after 39768 tasks: 1081.2890625 MB +worker1-1 | 2024/06/04 17:16:14.644 FIG10b: Memory usage without flush after 39923 tasks: 1080.45703125 MB +worker1-1 | 2024/06/04 17:16:15.646 FIG10b: Memory usage without flush after 40079 tasks: 1086.22265625 MB +worker1-1 | 2024/06/04 17:16:16.648 FIG10b: Memory usage without flush after 40228 tasks: 1088.953125 MB +worker1-1 | 2024/06/04 17:16:46.651 FIG10b: Memory usage without flush after 44386 tasks: 1117.3046875 MB +worker1-1 | 2024/06/04 17:17:16.6552024/06/04 17:17:16.655 [worker] in Run() rnFIG10b: Memory usage without flush after 48572 tasks: 1165.1796875 MB +worker1-1 | 2024/06/04 17:17:46.666 FIG10b: Memory usage without flush after 52603 tasks: 1202.19921875 MB +worker1-1 | 2024/06/04 17:18:16.6612024/06/04 17:18:16.661 FIG10b: Memory usage without flush after 56539 tasks: 1269.734375 MB +worker1-1 | 2024/06/04 17:18:46.664 FIG10b: Memory usage without flush after 60671 tasks: 1289.60546875 MB +worker1-1 | 2024/06/04 17:19:16.666 FIG10b: Memory usage without flush after 64929 tasks: 1303.59375 MB +worker1-1 | 2024/06/04 17:19:46.668 FIG10b: Memory usage without flush after 68960 tasks: 1330.65625 MB +worker1-1 | 2024/06/04 17:20:16.673 FIG10b: Memory usage without flush after 72910 tasks: 1324.15234375 MB +worker1-1 | 2024/06/04 17:20:46.674 FIG10b: Memory usage without flush after 77160 tasks: 1337.40625 MB +worker1-1 | 2024/06/04 17:21:16.676 FIG10b: Memory usage without flush after 81358 tasks: 1349.7890625 MB +worker1-1 | 2024/06/04 17:21:46.677 FIG10b: Memory usage without flush after 85642 tasks: 1381.5703125 MB +worker1-1 | 2024/06/04 17:22:16.684 FIG10b: Memory usage without flush after 89999 tasks: 1412.875 MB +worker1-1 | 2024/06/04 17:22:46.688 FIG10b: Memory usage without flush after 94223 tasks: 1424.2890625 MB +worker1-1 | 2024/06/04 17:23:16.688 FIG10b: Memory usage without flush after 98242 tasks: 1456.50390625 MB +worker1-1 | 2024/06/04 17:23:46.690 FIG10b: Memory usage without flush after 102675 tasks: 1455.07421875 MB +worker1-1 | 2024/06/04 17:24:16.692 FIG10b: Memory usage without flush after 106843 tasks: 1491.23046875 MB +worker1-1 | 2024/06/04 17:24:46.695 FIG10b: Memory usage without flush after 110260 tasks: 1501.765625 MB +worker1-1 | 2024/06/04 17:25:16.695 FIG10b: Memory usage without flush after 114276 tasks: 1513.46875 MB +worker1-1 | 2024/06/04 17:25:46.699 FIG10b: Memory usage without flush after 118268 tasks: 1487.48828125 MB +worker1-1 | 2024/06/04 17:26:16.702 FIG10b: Memory usage without flush after 122774 tasks: 1504.36328125 MB +worker1-1 | 2024/06/04 17:26:46.706 FIG10b: Memory usage without flush after 126885 tasks: 1515.48046875 MB +worker1-1 | 2024/06/04 17:27:16.710 FIG10b: Memory usage without flush after 131135 tasks: 1559.76171875 MB +worker1-1 | 2024/06/04 17:27:46.713 FIG10b: Memory usage without flush after 135333 tasks: 1614.8671875 MB +worker1-1 | 2024/06/04 17:28:16.715 FIG10b: Memory usage without flush after 139528 tasks: 1599.36328125 MB +worker1-1 | 2024/06/04 17:28:46.721 FIG10b: Memory usage without flush after 143095 tasks: 1575.1171875 MB +worker1-1 | 2024/06/04 17:29:16.721 FIG10b: Memory usage without flush after 146849 tasks: 1575.37890625 MB +worker1-1 | 2024/06/04 17:29:46.724 FIG10b: Memory usage without flush after 150948 tasks: 1580.12109375 MB +worker1-1 | 2024/06/04 17:30:16.730 FIG10b: Memory usage without flush after 155098 tasks: 1594.60546875 MB +worker1-1 | 2024/06/04 17:30:46.732 FIG10b: Memory usage without flush after 159007 tasks: 1606.80078125 MB +worker1-1 | 2024/06/04 17:31:16.734 FIG10b: Memory usage without flush after 162942 tasks: 1613.8203125 MB +worker1-1 | 2024/06/04 17:31:46.738 FIG10b: Memory usage without flush after 166558 tasks: 1650.86328125 MB +worker1-1 | 2024/06/04 17:32:16.744 FIG10b: Memory usage without flush after 170461 tasks: 1624.8828125 MB +worker1-1 | 2024/06/04 17:32:46.746 FIG10b: Memory usage without flush after 173707 tasks: 1655.79296875 MB +worker1-1 | 2024/06/04 17:33:16.748 FIG10b: Memory usage without flush after 177633 tasks: 1629.8515625 MB +worker1-1 | 2024/06/04 17:33:46.749 FIG10b: Memory usage without flush after 181601 tasks: 1632.0 MB +worker1-1 | 2024/06/04 17:34:16.752 FIG10b: Memory usage without flush after 185073 tasks: 1665.92578125 MB +worker1-1 | 2024/06/04 17:34:46.755 FIG10b: Memory usage without flush after 189301 tasks: 1641.40625 MB +worker1-1 | 2024/06/04 17:35:16.757 FIG10b: Memory usage without flush after 193460 tasks: 1651.3671875 MB +worker1-1 | 2024/06/04 17:35:46.762 FIG10b: Memory usage without flush after 197325 tasks: 1679.046875 MB +worker1-1 | 2024/06/04 17:36:16.764 FIG10b: Memory usage without flush after 201556 tasks: 1700.41796875 MB +worker1-1 | 2024/06/04 17:36:46.764 FIG10b: Memory usage without flush after 206485 tasks: 1742.9765625 MB +worker1-1 | 2024/06/04 17:37:16.765 FIG10b: Memory usage without flush after 209976 tasks: 1769.1015625 MB +worker1-1 | 2024/06/04 17:37:46.768 FIG10b: Memory usage without flush after 214258 tasks: 1834.1015625 MB +worker1-1 | 2024/06/04 17:38:16.772 FIG10b: Memory usage without flush after 218638 tasks: 1843.01953125 MB +worker1-1 | 2024/06/04 17:38:46.777 FIG10b: Memory usage without flush after 222805 tasks: 1836.50390625 MB +worker1-1 | 2024/06/04 17:39:16.780 FIG10b: Memory usage without flush after 227560 tasks: 1847.5 MB +worker1-1 | 2024/06/04 17:39:46.784 FIG10b: Memory usage without flush after 232082 tasks: 1855.3125 MB +worker1-1 | 2024/06/04 17:40:16.791 FIG10b: Memory usage without flush after 236496 tasks: 1852.5625 MB +worker1-1 | 2024/06/04 17:40:46.793 FIG10b: Memory usage without flush after 240574 tasks: 1874.36328125 MB +worker1-1 | 2024/06/04 17:41:16.795 FIG10b: Memory usage without flush after 245130 tasks: 1870.08203125 MB +worker1-1 | 2024/06/04 17:41:46.798 FIG10b: Memory usage without flush after 249358 tasks: 1878.80078125 MB +worker1-1 | 2024/06/04 17:42:16.798 FIG10b: Memory usage without flush after 253381 tasks: 1861.08203125 MB +worker1-1 | 2024/06/04 17:42:46.816 FIG10b: Memory usage without flush after 257740 tasks: 1869.875 MB +worker1-1 | 2024/06/04 17:43:16.819 FIG10b: Memory usage without flush after 261253 tasks: 1898.765625 MB +worker1-1 | 2024/06/04 17:43:46.822 FIG10b: Memory usage without flush after 265729 tasks: 1860.5234375 MB +worker1-1 | 2024/06/04 17:44:16.828 FIG10b: Memory usage without flush after 270384 tasks: 1857.2265625 MB +worker1-1 | 2024/06/04 17:44:46.829 FIG10b: Memory usage without flush after 274364 tasks: 1858.05859375 MB +worker1-1 | 2024/06/04 17:45:16.830 FIG10b: Memory usage without flush after 278751 tasks: 1870.82421875 MB +worker1-1 | 2024/06/04 17:45:46.835 FIG10b: Memory usage without flush after 283240 tasks: 1900.47265625 MB +worker1-1 | 2024/06/04 17:46:16.837 FIG10b: Memory usage without flush after 287670 tasks: 1909.76953125 MB diff --git a/analysis/output_10b_run5_buggy?_1s_flush_use_AOF_no_garbage_yet.txt b/analysis/output_10b_run5_buggy?_1s_flush_use_AOF_no_garbage_yet.txt new file mode 100644 index 0000000..3bba31b --- /dev/null +++ b/analysis/output_10b_run5_buggy?_1s_flush_use_AOF_no_garbage_yet.txt @@ -0,0 +1,314 @@ +worker1-1 | 2024/06/04 17:51:35.961 FIG10b: HELLO WORLD I AM ANGRY! +worker1-1 | 2024/06/04 17:51:36.000 FIG10b: Starting tasks submission... +worker1-1 | 2024/06/04 17:51:37.003 FIG10b: Memory usage without flush after 111 tasks: 1066.48046875 MB +worker1-1 | 2024/06/04 17:51:38.0042024/06/04 17:51:38.005 FIG10b: Memory usage without flush after 225 tasks: 1080.74609375 MB +worker1-1 | Executed!2024/06/04 17:51:39.009 FIG10b: Memory usage without flush after 341 tasks: 1081.58203125 MB +worker1-1 | 2024/06/04 17:51:40.011 FIG10b: Memory usage without flush after 461 tasks: 1082.59765625 MB +worker1-1 | 2024/06/04 17:51:41.024 FIG10b: Memory usage without flush after 575 tasks: 1087.3671875 MB +worker1-1 | FIG10b: Memory usage without flush after 686 tasks: 1092.6953125 MB +worker1-1 | FIG10b: Memory usage without flush after 801 tasks: 1094.02734375 MB +worker1-1 | 2024/06/04 17:51:44.051 FIG10b: Memory usage without flush after 905 tasks: 1093.44921875 MB +worker1-1 | 2024/06/04 17:51:45.053 FIG10b: Memory usage without flush after 1009 tasks: 1094.83984375 MB +worker1-1 | 2024/06/04 17:51:46.054 FIG10b: Memory usage without flush after 1105 tasks: 1096.97265625 MB +worker1-1 | 2024/06/04 17:51:47.062 FIG10b: Memory usage without flush after 1226 tasks: 1100.5546875 MB +worker1-1 | [worker]2024/06/04 17:51:48.063 FIG10b: Memory usage without flush after 1343 tasks: 1101.35546875 MB +worker1-1 | 2024/06/04 17:51:49.071 FIG10b: Memory usage without flush after 1451 tasks: 1103.0859375 MB +worker1-1 | 2024/06/04 17:51:50.074 FIG10b: Memory usage without flush after 1577 tasks: 1103.578125 MB +worker1-1 | [worker] 2024/06/04 17:51:51.076 FIG10b: Memory usage without flush after 1696 tasks: 1104.5546875 MB +worker1-1 | 2024/06/04 17:51:52.077 FIG10b: Memory usage without flush after 1803 tasks: 1105.09375 MB +worker1-1 | 2024/06/04 17:51:53.079 FIG10b: Memory usage without flush after 1927 tasks: 1104.55078125 MB +worker1-1 | [worker] 2024/06/04 17:51:54.079 FIG10b: Memory usage without flush after 2036 tasks: 1105.546875 MB +worker1-1 | [worker] stored...2024/06/04 17:51:55.113 FIG10b: Memory usage without flush after 2151 tasks: 1110.0390625 MB +worker1-1 | stored...2024/06/04 17:51:56.116 FIG10b: Memory usage without flush after 2238 tasks: 1110.578125 MB +worker1-1 | 2024/06/04 17:51:57.119 FIG10b: Memory usage without flush after 2354 tasks: 1109.95703125 MB +worker1-1 | 2024/06/04 17:51:58.1212024/06/04 17:51:58.121 FIG10b: Memory usage without flush after 2437 tasks: 1109.3125 MB +worker1-1 | 2024/06/04 17:51:59.125 FIG10b: Memory usage without flush after 2521 tasks: 1111.52734375 MB +worker1-1 | 2024/06/04 17:52:00.1282024/06/04 17:52:00.128 [worker] stored...FIG10b: Memory usage without flush after 2620 tasks: 1112.5 MB +worker1-1 | 2024/06/04 17:52:01.130 FIG10b: Memory usage without flush after 2730 tasks: 1113.73046875 MB +worker1-1 | [worker]2024/06/04 17:52:02.134in Run() rn FIG10b: Memory usage without flush after 2854 tasks: 1113.42578125 MB +worker1-1 | 2024/06/04 17:52:03.1382024/06/04 17:52:03.138 FIG10b: Memory usage without flush after 2975 tasks: 1113.96484375 MB +worker1-1 | 2024/06/04 17:52:04.146 FIG10b: Memory usage without flush after 3098 tasks: 1113.765625 MB +worker1-1 | 2024/06/04 17:52:05.1492024/06/04 17:52:05.149 FIG10b: Memory usage without flush after 3217 tasks: 1115.2421875 MB +worker1-1 | [worker] 2024/06/04 17:52:06.1582024/06/04 17:52:06.158 FIG10b: Memory usage without flush after 3341 tasks: 1115.1484375 MB +worker1-1 | 2024/06/04 17:52:07.161 2024/06/04 17:52:07.162 FIG10b: Memory usage without flush after 3435 tasks: 1119.1953125 MB +worker1-1 | Executed! 2024/06/04 17:52:08.170 FIG10b: Memory usage without flush after 3562 tasks: 1119.265625 MB +worker1-1 | 2024/06/04 17:52:09.179 FIG10b: Memory usage without flush after 3686 tasks: 1120.51953125 MB +worker1-1 | 2024/06/04 17:52:10.187 FIG10b: Memory usage without flush after 3813 tasks: 1120.15234375 MB +worker1-1 | in Run() rn2024/06/04 17:52:11.188 FIG10b: Memory usage without flush after 3934 tasks: 1120.69140625 MB +worker1-1 | 2024/06/04 17:52:12.204 FIG10b: Memory usage without flush after 4040 tasks: 1123.9296875 MB +worker1-1 | 2024/06/04 17:52:13.204 FIG10b: Memory usage without flush after 4158 tasks: 1125.33984375 MB +worker1-1 | [worker] 2024/06/04 17:52:14.206 FIG10b: Memory usage without flush after 4279 tasks: 1126.125 MB +worker1-1 | 2024/06/04 17:52:15.2102024/06/04 17:52:15.210 FIG10b: Memory usage without flush after 4398 tasks: 1126.41796875 MB +worker1-1 | FIG10b: Memory usage without flush after 4509 tasks: 1126.48828125 MB +worker1-1 | 2024/06/04 17:52:17.217 FIG10b: Memory usage without flush after 4636 tasks: 1127.2734375 MB +worker1-1 | 2024/06/04 17:52:18.222 2024/06/04 17:52:18.223 FIG10b: Memory usage without flush after 4761 tasks: 1127.14453125 MB +worker1-1 | 2024/06/04 17:52:19.227 FIG10b: Memory usage without flush after 4884 tasks: 1126.45703125 MB +worker1-1 | [worker]2024/06/04 17:52:20.229[worker]2024/06/04 17:52:20.229 FIG10b: Memory usage without flush after 5006 tasks: 1126.51953125 MB +worker1-1 | 2024/06/04 17:52:21.233 FIG10b: Memory usage without flush after 5128 tasks: 1126.5625 MB +worker1-1 | took this many seconds: 0.01422024/06/04 17:52:22.244 FIG10b: Memory usage without flush after 5247 tasks: 1127.1328125 MB +worker1-1 | 2024/06/04 17:52:23.2582024/06/04 17:52:23.258 FIG10b: Memory usage without flush after 5383 tasks: 1128.5703125 MB +worker1-1 | 2024/06/04 17:52:24.259 FIG10b: Memory usage without flush after 5510 tasks: 1128.6015625 MB +worker1-1 | 2024/06/04 17:52:25.2672024/06/04 17:52:25.267 FIG10b: Memory usage without flush after 5644 tasks: 1129.32421875 MB +worker1-1 | 2024/06/04 17:52:26.273 FIG10b: Memory usage without flush after 5764 tasks: 1129.88671875 MB +worker1-1 | [worker]gonna store now2024/06/04 17:52:27.274 FIG10b: Memory usage without flush after 5887 tasks: 1129.6875 MB +worker1-1 | 2024/06/04 17:52:28.2752024/06/04 17:52:28.275 2024/06/04 17:52:28.276 FIG10b: Memory usage without flush after 6014 tasks: 1129.9609375 MB +worker1-1 | 2024/06/04 17:52:29.281 FIG10b: Memory usage without flush after 6145 tasks: 1130.45703125 MB +worker1-1 | took this many seconds: 0.01622024/06/04 17:52:30.288 FIG10b: Memory usage without flush after 6271 tasks: 1130.7109375 MB +worker1-1 | 2024/06/04 17:52:31.297 FIG10b: Memory usage without flush after 6396 tasks: 1130.2578125 MB +worker1-1 | [worker]in Run() rn 2024/06/04 17:52:32.304 FIG10b: Memory usage without flush after 6516 tasks: 1130.8046875 MB +worker1-1 | 2024/06/04 17:52:33.305 FIG10b: Memory usage without flush after 6638 tasks: 1131.09375 MB +worker1-1 | [worker] 2024/06/04 17:52:34.305 FIG10b: Memory usage without flush after 6759 tasks: 1131.171875 MB +worker1-1 | 2024/06/04 17:52:35.306 2024/06/04 17:52:35.307 [worker]FIG10b: Memory usage without flush after 6890 tasks: 1131.17578125 MB +worker1-1 | FIG10b: Memory usage without flush after 7021 tasks: 1131.42578125 MB +worker1-1 | 2024/06/04 17:52:37.311 FIG10b: Memory usage without flush after 7131 tasks: 1131.3046875 MB +worker1-1 | 2024/06/04 17:52:38.320 FIG10b: Memory usage without flush after 7258 tasks: 1131.81640625 MB2024/06/04 17:52:38.320 [lobs] STORED!!! +worker1-1 | 2024/06/04 17:52:39.325 FIG10b: Memory usage without flush after 7392 tasks: 1131.5703125 MB +worker1-1 | took this many seconds: 0.0175[worker]2024/06/04 17:52:40.328 FIG10b: Memory usage without flush after 7519 tasks: 1132.1015625 MB +worker1-1 | 2024/06/04 17:52:41.329 [worker]2024/06/04 17:52:41.330 FIG10b: Memory usage without flush after 7644 tasks: 1131.64453125 MB +worker1-1 | 2024/06/04 17:52:42.336 FIG10b: Memory usage without flush after 7769 tasks: 1131.93359375 MB +worker1-1 | 2024/06/04 17:52:43.337 FIG10b: Memory usage without flush after 7866 tasks: 1132.62109375 MB +worker1-1 | 2024/06/04 17:52:44.339 FIG10b: Memory usage without flush after 7964 tasks: 1132.5703125 MB +worker1-1 | [worker]2024/06/04 17:52:45.359 FIG10b: Memory usage without flush after 8064 tasks: 1133.00390625 MB +worker1-1 | 2024/06/04 17:52:46.721 FIG10b: Memory usage without flush after 8078 tasks: 1133.15234375 MB +worker1-1 | 2024/06/04 17:52:47.669 [worker] 2024/06/04 17:52:47.679 FIG10b: Memory usage without flush after 8093 tasks: 1133.328125 MB +worker1-1 | 2024/06/04 17:52:48.682 2024/06/04 17:52:48.683 FIG10b: Memory usage without flush after 8175 tasks: 1132.875 MB +worker1-1 | 2024/06/04 17:52:49.699 FIG10b: Memory usage without flush after 8270 tasks: 1132.80859375 MB +worker1-1 | 2024/06/04 17:52:50.761 FIG10b: Memory usage without flush after 8363 tasks: 1133.30078125 MB +worker1-1 | 2024/06/04 17:52:51.767 FIG10b: Memory usage without flush after 8466 tasks: 1134.76171875 MB +worker1-1 | 2024/06/04 17:52:52.775 FIG10b: Memory usage without flush after 8586 tasks: 1135.74609375 MB +worker1-1 | 2024/06/04 17:52:53.783 FIG10b: Memory usage without flush after 8691 tasks: 1135.81640625 MB +worker1-1 | 2024/06/04 17:52:54.784 FIG10b: Memory usage without flush after 8799 tasks: 1135.265625 MB +worker1-1 | 2024/06/04 17:52:55.788 FIG10b: Memory usage without flush after 8908 tasks: 1135.0546875 MB +worker1-1 | [worker]2024/06/04 17:52:56.792 FIG10b: Memory usage without flush after 9029 tasks: 1136.5078125 MB +worker1-1 | stored... 2024/06/04 17:52:57.795 FIG10b: Memory usage without flush after 9152 tasks: 1136.1328125 MB +worker1-1 | FIG10b: Memory usage without flush after 9275 tasks: 1135.734375 MB +worker1-1 | 2024/06/04 17:52:59.800 FIG10b: Memory usage without flush after 9401 tasks: 1136.77734375 MB +worker1-1 | 2024/06/04 17:53:00.803 FIG10b: Memory usage without flush after 9508 tasks: 1137.28125 MB +worker1-1 | 2024/06/04 17:53:01.806 FIG10b: Memory usage without flush after 9622 tasks: 1138.0078125 MB +worker1-1 | 2024/06/04 17:53:02.808 FIG10b: Memory usage without flush after 9749 tasks: 1137.2109375 MB +worker1-1 | 2024/06/04 17:53:03.820 FIG10b: Memory usage without flush after 9878 tasks: 1137.984375 MB +worker1-1 | 2024/06/04 17:53:04.8202024/06/04 17:53:04.821 FIG10b: Memory usage without flush after 10001 tasks: 1138.98046875 MB +worker1-1 | 2024/06/04 17:53:05.8242024/06/04 17:53:05.824 FIG10b: Memory usage without flush after 10122 tasks: 1139.765625 MB +worker1-1 | 2024/06/04 17:53:06.827 FIG10b: Memory usage without flush after 10240 tasks: 1138.9453125 MB +worker1-1 | [worker]2024/06/04 17:53:07.828 FIG10b: Memory usage without flush after 10357 tasks: 1139.26171875 MB +worker1-1 | 2024/06/04 17:53:08.832 FIG10b: Memory usage without flush after 10481 tasks: 1139.578125 MB +worker1-1 | 2024/06/04 17:53:09.841 FIG10b: Memory usage without flush after 10602 tasks: 1139.87109375 MB +worker1-1 | [worker]2024/06/04 17:53:10.843 FIG10b: Memory usage without flush after 10719 tasks: 1140.90234375 MB +worker1-1 | 2024/06/04 17:53:11.849 FIG10b: Memory usage without flush after 10834 tasks: 1141.93359375 MB +worker1-1 | 2024/06/04 17:53:12.859 FIG10b: Memory usage without flush after 10930 tasks: 1141.7578125 MB +worker1-1 | 2024/06/04 17:53:13.862 [worker] 2024/06/04 17:53:13.862 FIG10b: Memory usage without flush after 11014 tasks: 1142.51953125 MB +worker1-1 | [worker] 2024/06/04 17:53:14.866 FIG10b: Memory usage without flush after 11122 tasks: 1142.07421875 MB +worker1-1 | in Run() rn2024/06/04 17:53:15.866 FIG10b: Memory usage without flush after 11255 tasks: 1140.20703125 MB +worker1-1 | 2024/06/04 17:53:16.870 FIG10b: Memory usage without flush after 11366 tasks: 1140.734375 MB +worker1-1 | 2024/06/04 17:53:17.872 FIG10b: Memory usage without flush after 11473 tasks: 1140.51171875 MB +worker1-1 | 2024/06/04 17:53:18.877 FIG10b: Memory usage without flush after 11591 tasks: 1142.72265625 MB +worker1-1 | 2024/06/04 17:53:19.879 [worker]2024/06/04 17:53:19.880 FIG10b: Memory usage without flush after 11693 tasks: 1140.58984375 MB +worker1-1 | 2024/06/04 17:53:20.936 FIG10b: Memory usage without flush after 11792 tasks: 1141.75 MB +worker1-1 | 2024/06/04 17:53:21.938 2024/06/04 17:53:21.938 FIG10b: Memory usage without flush after 11912 tasks: 1142.65625 MB +worker1-1 | 2024/06/04 17:53:22.941 FIG10b: Memory usage without flush after 12036 tasks: 1143.30078125 MB +worker1-1 | 2024/06/04 17:53:23.958 FIG10b: Memory usage without flush after 12161 tasks: 1143.16796875 MB +worker1-1 | 2024/06/04 17:53:24.9622024/06/04 17:53:24.9632024/06/04 17:53:24.963 FIG10b: Memory usage without flush after 12281 tasks: 1143.296875 MB +worker1-1 | 2024/06/04 17:53:25.966 FIG10b: Memory usage without flush after 12399 tasks: 1143.90234375 MB +worker1-1 | 2024/06/04 17:53:26.968 FIG10b: Memory usage without flush after 12510 tasks: 1143.60546875 MB +worker1-1 | 2024/06/04 17:53:27.969in Run() rn FIG10b: Memory usage without flush after 12632 tasks: 1143.95703125 MB +worker1-1 | [worker]2024/06/04 17:53:28.972 FIG10b: Memory usage without flush after 12746 tasks: 1144.84375 MB +worker1-1 | [worker] 2024/06/04 17:53:29.977 FIG10b: Memory usage without flush after 12876 tasks: 1144.6796875 MB +worker1-1 | [worker][worker]2024/06/04 17:53:30.984 FIG10b: Memory usage without flush after 12996 tasks: 1146.03515625 MB +worker1-1 | 2024/06/04 17:53:31.9842024/06/04 17:53:31.984 [worker] FIG10b: Memory usage without flush after 13126 tasks: 1145.625 MBin Run() rn +worker1-1 | 2024/06/04 17:53:32.986 FIG10b: Memory usage without flush after 13234 tasks: 1146.79296875 MB +worker1-1 | 2024/06/04 17:53:33.988 FIG10b: Memory usage without flush after 13349 tasks: 1154.3125 MB +worker1-1 | FIG10b: Memory usage without flush after 13456 tasks: 1154.5703125 MB +worker1-1 | 2024/06/04 17:53:36.004 FIG10b: Memory usage without flush after 13576 tasks: 1156.3984375 MB +worker1-1 | FIG10b: Memory usage without flush after 13698 tasks: 1156.79296875 MB +worker1-1 | 2024/06/04 17:53:38.010 FIG10b: Memory usage without flush after 13827 tasks: 1151.52734375 MB +worker1-1 | 2024/06/04 17:53:39.015 FIG10b: Memory usage without flush after 13957 tasks: 1151.87109375 MB +worker1-1 | 2024/06/04 17:53:40.017 FIG10b: Memory usage without flush after 14074 tasks: 1149.55078125 MB +worker1-1 | 2024/06/04 17:53:41.020 FIG10b: Memory usage without flush after 14199 tasks: 1150.41796875 MB +worker1-1 | 2024/06/04 17:53:42.032 FIG10b: Memory usage without flush after 14326 tasks: 1150.734375 MB +worker1-1 | 2024/06/04 17:53:43.034 FIG10b: Memory usage without flush after 14451 tasks: 1150.578125 MB +worker1-1 | 2024/06/04 17:53:44.035 2024/06/04 17:53:44.035 FIG10b: Memory usage without flush after 14573 tasks: 1150.9375 MB +worker1-1 | 2024/06/04 17:53:45.035 2024/06/04 17:53:45.036 FIG10b: Memory usage without flush after 14696 tasks: 1151.0703125 MB +worker1-1 | 2024/06/04 17:53:46.041 FIG10b: Memory usage without flush after 14809 tasks: 1151.46484375 MB +worker1-1 | in Run() rn2024/06/04 17:53:47.042 FIG10b: Memory usage without flush after 14931 tasks: 1151.5703125 MB +worker1-1 | 2024/06/04 17:53:48.045 FIG10b: Memory usage without flush after 15053 tasks: 1151.9296875 MB +worker1-1 | 2024/06/04 17:53:49.046 FIG10b: Memory usage without flush after 15175 tasks: 1152.3125 MB +worker1-1 | 2024/06/04 17:53:50.047 FIG10b: Memory usage without flush after 15301 tasks: 1144.1484375 MB +worker1-1 | 2024/06/04 17:53:51.051 FIG10b: Memory usage without flush after 15421 tasks: 1145.4765625 MB +worker1-1 | 2024/06/04 17:53:52.055 FIG10b: Memory usage without flush after 15550 tasks: 1146.1015625 MB +worker1-1 | 2024/06/04 17:53:53.062 FIG10b: Memory usage without flush after 15671 tasks: 1146.44921875 MB +worker1-1 | 2024/06/04 17:53:54.063 FIG10b: Memory usage without flush after 15786 tasks: 1146.38671875 MB +worker1-1 | 2024/06/04 17:53:55.072 FIG10b: Memory usage without flush after 15913 tasks: 1146.64453125 MB +worker1-1 | [worker]2024/06/04 17:53:56.077 FIG10b: Memory usage without flush after 16025 tasks: 1145.60546875 MB +worker1-1 | 2024/06/04 17:53:57.089 2024/06/04 17:53:57.090 FIG10b: Memory usage without flush after 16137 tasks: 1146.7109375 MB +worker1-1 | 2024/06/04 17:53:58.091 FIG10b: Memory usage without flush after 16233 tasks: 1146.19140625 MB +worker1-1 | 2024/06/04 17:53:59.094 FIG10b: Memory usage without flush after 16290 tasks: 1147.15625 MB +worker1-1 | 2024/06/04 17:54:00.098 FIG10b: Memory usage without flush after 16404 tasks: 1137.625 MB +worker1-1 | 2024/06/04 17:54:01.105 FIG10b: Memory usage without flush after 16507 tasks: 1137.79296875 MB +worker1-1 | 2024/06/04 17:54:02.120 FIG10b: Memory usage without flush after 16621 tasks: 1138.90625 MB +worker1-1 | 2024/06/04 17:54:03.122 FIG10b: Memory usage without flush after 16739 tasks: 1139.12890625 MB +worker1-1 | 2024/06/04 17:54:04.126 [worker] stored...2024/06/04 17:54:04.1262024/06/04 17:54:04.127 FIG10b: Memory usage without flush after 16849 tasks: 1138.421875 MB +worker1-1 | [worker]2024/06/04 17:54:05.129 FIG10b: Memory usage without flush after 16966 tasks: 1139.22265625 MB +worker1-1 | gonna store now 2024/06/04 17:54:06.1302024/06/04 17:54:06.130 FIG10b: Memory usage without flush after 17089 tasks: 1140.7890625 MB +worker1-1 | stored...2024/06/04 17:54:07.130 FIG10b: Memory usage without flush after 17208 tasks: 1140.51171875 MB +worker1-1 | 2024/06/04 17:54:08.1362024/06/04 17:54:08.137 FIG10b: Memory usage without flush after 17324 tasks: 1140.02734375 MB +worker1-1 | 2024/06/04 17:54:09.150 FIG10b: Memory usage without flush after 17420 tasks: 1139.7578125 MB +worker1-1 | 2024/06/04 17:54:10.160 FIG10b: Memory usage without flush after 17508 tasks: 1141.15234375 MB +worker1-1 | [worker]2024/06/04 17:54:11.170 FIG10b: Memory usage without flush after 17603 tasks: 1139.875 MB +worker1-1 | [worker] 2024/06/04 17:54:12.172 FIG10b: Memory usage without flush after 17700 tasks: 1140.3125 MB +worker1-1 | [worker]2024/06/04 17:54:13.178 FIG10b: Memory usage without flush after 17810 tasks: 1140.9921875 MB +worker1-1 | 2024/06/04 17:54:14.182 FIG10b: Memory usage without flush after 17911 tasks: 1140.7421875 MB +worker1-1 | 2024/06/04 17:54:15.185 FIG10b: Memory usage without flush after 18022 tasks: 1140.453125 MB +worker1-1 | 2024/06/04 17:54:16.186 FIG10b: Memory usage without flush after 18128 tasks: 1141.609375 MB +worker1-1 | 2024/06/04 17:54:17.1862024/06/04 17:54:17.186 FIG10b: Memory usage without flush after 18243 tasks: 1141.55078125 MB +worker1-1 | 2024/06/04 17:54:18.192 2024/06/04 17:54:18.193[worker]2024/06/04 17:54:18.193 FIG10b: Memory usage without flush after 18350 tasks: 1141.2890625 MB +worker1-1 | FIG10b: Memory usage without flush after 18464 tasks: 1141.28125 MB +worker1-1 | 2024/06/04 17:54:20.2002024/06/04 17:54:20.200 FIG10b: Memory usage without flush after 18577 tasks: 1141.4765625 MB +worker1-1 | [worker] 2024/06/04 17:54:21.208 FIG10b: Memory usage without flush after 18696 tasks: 1142.69140625 MB +worker1-1 | 2024/06/04 17:54:22.218 FIG10b: Memory usage without flush after 18816 tasks: 1143.0 MB +worker1-1 | 2024/06/04 17:54:23.219 FIG10b: Memory usage without flush after 18939 tasks: 1143.05859375 MB +worker1-1 | 2024/06/04 17:54:24.227in Run() rn2024/06/04 17:54:24.228 FIG10b: Memory usage without flush after 19052 tasks: 1143.6640625 MB +worker1-1 | 2024/06/04 17:54:25.231 FIG10b: Memory usage without flush after 19178 tasks: 1143.70703125 MB +worker1-1 | stored...2024/06/04 17:54:26.234 FIG10b: Memory usage without flush after 19301 tasks: 1144.0078125 MB +worker1-1 | 2024/06/04 17:54:27.2532024/06/04 17:54:27.253 [worker] Executed!FIG10b: Memory usage without flush after 19426 tasks: 1144.29296875 MB +worker1-1 | 2024/06/04 17:54:28.256 FIG10b: Memory usage without flush after 19541 tasks: 1144.40234375 MB +worker1-1 | 2024/06/04 17:54:29.260 2024/06/04 17:54:29.261 FIG10b: Memory usage without flush after 19636 tasks: 1144.85546875 MB +worker1-1 | 2024/06/04 17:54:30.266 FIG10b: Memory usage without flush after 19714 tasks: 1145.16015625 MB +worker1-1 | 2024/06/04 17:54:31.268 FIG10b: Memory usage without flush after 19825 tasks: 1147.01953125 MB +worker1-1 | 2024/06/04 17:54:32.320 FIG10b: Memory usage without flush after 19938 tasks: 1146.8828125 MB +worker1-1 | 2024/06/04 17:54:33.341 FIG10b: Memory usage without flush after 20060 tasks: 1146.9375 MB +worker1-1 | 2024/06/04 17:54:34.346 FIG10b: Memory usage without flush after 20156 tasks: 1146.41796875 MB +worker1-1 | 2024/06/04 17:54:35.395 FIG10b: Memory usage without flush after 20260 tasks: 1147.53125 MB +worker1-1 | took this many seconds: 0.02142024/06/04 17:54:36.409 FIG10b: Memory usage without flush after 20364 tasks: 1147.203125 MB +worker1-1 | 2024/06/04 17:54:37.413 FIG10b: Memory usage without flush after 20475 tasks: 1147.32421875 MB +worker1-1 | 2024/06/04 17:54:38.418 [worker]2024/06/04 17:54:38.418 FIG10b: Memory usage without flush after 20557 tasks: 1147.59765625 MB +worker1-1 | took this many seconds: 0.0115 2024/06/04 17:54:39.4182024/06/04 17:54:39.418 FIG10b: Memory usage without flush after 20673 tasks: 1147.4453125 MB +worker1-1 | 2024/06/04 17:54:40.4292024/06/04 17:54:40.429 FIG10b: Memory usage without flush after 20773 tasks: 1147.625 MB +worker1-1 | 2024/06/04 17:54:41.441 FIG10b: Memory usage without flush after 20873 tasks: 1149.32421875 MB +worker1-1 | 2024/06/04 17:54:42.446 FIG10b: Memory usage without flush after 20968 tasks: 1149.02734375 MB +worker1-1 | 2024/06/04 17:54:43.447 2024/06/04 17:54:43.447[worker] FIG10b: Memory usage without flush after 21083 tasks: 1149.37890625 MB +worker1-1 | 2024/06/04 17:54:44.448 FIG10b: Memory usage without flush after 21185 tasks: 1149.0546875 MB +worker1-1 | 2024/06/04 17:54:45.458 FIG10b: Memory usage without flush after 21273 tasks: 1149.53125 MB +worker1-1 | 2024/06/04 17:54:46.4602024/06/04 17:54:46.460 FIG10b: Memory usage without flush after 21372 tasks: 1149.96484375 MB +worker1-1 | gonna store nowin Run() rn2024/06/04 17:54:47.461 FIG10b: Memory usage without flush after 21481 tasks: 1149.6015625 MB +worker1-1 | 2024/06/04 17:54:48.467 FIG10b: Memory usage without flush after 21574 tasks: 1149.79296875 MB +worker1-1 | 2024/06/04 17:54:49.4682024/06/04 17:54:49.468 FIG10b: Memory usage without flush after 21684 tasks: 1149.68359375 MB +worker1-1 | 2024/06/04 17:54:50.471 FIG10b: Memory usage without flush after 21797 tasks: 1150.765625 MB +worker1-1 | [worker]2024/06/04 17:54:51.4752024/06/04 17:54:51.4752024/06/04 17:54:51.475 FIG10b: Memory usage without flush after 21908 tasks: 1150.83203125 MB +worker1-1 | 2024/06/04 17:54:52.476 [worker]gonna store now2024/06/04 17:54:52.477 FIG10b: Memory usage without flush after 22020 tasks: 1151.71875 MB +worker1-1 | 2024/06/04 17:54:53.483 2024/06/04 17:54:53.4842024/06/04 17:54:53.485 FIG10b: Memory usage without flush after 22130 tasks: 1153.14453125 MB +worker1-1 | 2024/06/04 17:54:54.487 FIG10b: Memory usage without flush after 22240 tasks: 1153.86328125 MB +worker1-1 | FIG10b: Memory usage without flush after 22345 tasks: 1152.5078125 MB +worker1-1 | Executed!2024/06/04 17:54:56.494 FIG10b: Memory usage without flush after 22446 tasks: 1152.77734375 MB +worker1-1 | stored...2024/06/04 17:54:57.4982024/06/04 17:54:57.498 FIG10b: Memory usage without flush after 22554 tasks: 1153.44921875 MB +worker1-1 | FIG10b: Memory usage without flush after 22658 tasks: 1154.15234375 MB +worker1-1 | 2024/06/04 17:54:59.5082024/06/04 17:54:59.508 stored...2024/06/04 17:54:59.509 FIG10b: Memory usage without flush after 22764 tasks: 1153.2890625 MB +worker1-1 | 2024/06/04 17:55:00.5132024/06/04 17:55:00.5142024/06/04 17:55:00.514 FIG10b: Memory usage without flush after 22866 tasks: 1153.59375 MB +worker1-1 | 2024/06/04 17:55:01.516 FIG10b: Memory usage without flush after 22966 tasks: 1146.75390625 MB +worker1-1 | 2024/06/04 17:55:02.522 FIG10b: Memory usage without flush after 23071 tasks: 1147.3984375 MB +worker1-1 | 2024/06/04 17:55:03.5252024/06/04 17:55:03.5252024/06/04 17:55:03.525 [worker]2024/06/04 17:55:03.526 FIG10b: Memory usage without flush after 23170 tasks: 1147.75 MB +worker1-1 | 2024/06/04 17:55:04.530 FIG10b: Memory usage without flush after 23265 tasks: 1148.02734375 MB +worker1-1 | 2024/06/04 17:55:05.532gonna store now FIG10b: Memory usage without flush after 23373 tasks: 1148.5625 MB +worker1-1 | stored...2024/06/04 17:55:06.538 FIG10b: Memory usage without flush after 23476 tasks: 1148.30859375 MB +worker1-1 | 2024/06/04 17:55:07.549 FIG10b: Memory usage without flush after 23579 tasks: 1149.0078125 MB +worker1-1 | 2024/06/04 17:55:08.5552024/06/04 17:55:08.555 FIG10b: Memory usage without flush after 23686 tasks: 1149.26953125 MB +worker1-1 | 2024/06/04 17:55:09.555 2024/06/04 17:55:09.5562024/06/04 17:55:09.556[worker] FIG10b: Memory usage without flush after 23793 tasks: 1149.5390625 MB +worker1-1 | FIG10b: Memory usage without flush after 23900 tasks: 1150.2421875 MB +worker1-1 | Executed!2024/06/04 17:55:11.580 2024/06/04 17:55:11.580 FIG10b: Memory usage without flush after 24006 tasks: 1150.45703125 MB +worker1-1 | 2024/06/04 17:55:12.5802024/06/04 17:55:12.581 FIG10b: Memory usage without flush after 24106 tasks: 1150.4765625 MB +worker1-1 | 2024/06/04 17:55:13.586 FIG10b: Memory usage without flush after 24208 tasks: 1151.46484375 MBExecuted! +worker1-1 | 2024/06/04 17:55:14.593 FIG10b: Memory usage without flush after 24318 tasks: 1151.48046875 MB +worker1-1 | 2024/06/04 17:55:15.596 FIG10b: Memory usage without flush after 24429 tasks: 1151.9609375 MB +worker1-1 | [worker] in Run() rn2024/06/04 17:55:16.597 FIG10b: Memory usage without flush after 24529 tasks: 1151.734375 MB +worker1-1 | FIG10b: Memory usage without flush after 24629 tasks: 1152.2421875 MB +worker1-1 | 2024/06/04 17:55:18.602 [worker]2024/06/04 17:55:18.602 FIG10b: Memory usage without flush after 24715 tasks: 1152.42578125 MB +worker1-1 | 2024/06/04 17:55:19.6102024/06/04 17:55:19.611 FIG10b: Memory usage without flush after 24801 tasks: 1152.6953125 MB +worker1-1 | 2024/06/04 17:55:20.617 FIG10b: Memory usage without flush after 24897 tasks: 1153.1640625 MB +worker1-1 | 2024/06/04 17:55:21.618 FIG10b: Memory usage without flush after 24992 tasks: 1151.421875 MB +worker1-1 | 2024/06/04 17:55:22.6212024/06/04 17:55:22.622 FIG10b: Memory usage without flush after 25089 tasks: 1148.91015625 MB +worker1-1 | 2024/06/04 17:55:23.627 FIG10b: Memory usage without flush after 25187 tasks: 1150.38671875 MB +worker1-1 | 2024/06/04 17:55:24.637 FIG10b: Memory usage without flush after 25293 tasks: 1150.31640625 MB +worker1-1 | 2024/06/04 17:55:25.640 FIG10b: Memory usage without flush after 25387 tasks: 1150.5546875 MB +worker1-1 | 2024/06/04 17:55:26.645 FIG10b: Memory usage without flush after 25477 tasks: 1150.83984375 MB +worker1-1 | 2024/06/04 17:55:27.648 FIG10b: Memory usage without flush after 25573 tasks: 1150.34765625 MB +worker1-1 | 2024/06/04 17:55:28.655 FIG10b: Memory usage without flush after 25666 tasks: 1150.8515625 MB +worker1-1 | 2024/06/04 17:55:29.657 FIG10b: Memory usage without flush after 25762 tasks: 1150.8671875 MB +worker1-1 | 2024/06/04 17:55:30.659 2024/06/04 17:55:30.660 FIG10b: Memory usage without flush after 25858 tasks: 1151.12109375 MB +worker1-1 | 2024/06/04 17:55:31.668 FIG10b: Memory usage without flush after 25944 tasks: 1151.1484375 MB +worker1-1 | 2024/06/04 17:55:32.6792024/06/04 17:55:32.679 FIG10b: Memory usage without flush after 26032 tasks: 1153.4609375 MB +worker1-1 | 2024/06/04 17:55:33.6822024/06/04 17:55:33.683 FIG10b: Memory usage without flush after 26120 tasks: 1153.9921875 MB +worker1-1 | [worker] gonna store now2024/06/04 17:55:34.688 FIG10b: Memory usage without flush after 26207 tasks: 1153.80859375 MB +worker1-1 | 2024/06/04 17:55:35.692 FIG10b: Memory usage without flush after 26277 tasks: 1153.6953125 MB +worker1-1 | 2024/06/04 17:55:36.699 FIG10b: Memory usage without flush after 26352 tasks: 1153.81640625 MB +worker1-1 | 2024/06/04 17:55:37.702 FIG10b: Memory usage without flush after 26423 tasks: 1153.94921875 MB +worker1-1 | 2024/06/04 17:55:38.706 FIG10b: Memory usage without flush after 26520 tasks: 1153.72265625 MB +worker1-1 | 2024/06/04 17:55:39.719 FIG10b: Memory usage without flush after 26600 tasks: 1153.56640625 MB +worker1-1 | 2024/06/04 17:55:40.723 FIG10b: Memory usage without flush after 26699 tasks: 1163.3046875 MB +worker1-1 | 2024/06/04 17:55:41.724 FIG10b: Memory usage without flush after 26786 tasks: 1164.203125 MB +worker1-1 | 2024/06/04 17:55:42.725 FIG10b: Memory usage without flush after 26880 tasks: 1164.9296875 MB +worker1-1 | 2024/06/04 17:55:43.733 FIG10b: Memory usage without flush after 26984 tasks: 1165.4140625 MB +worker1-1 | [worker]2024/06/04 17:55:44.7372024/06/04 17:55:44.737 FIG10b: Memory usage without flush after 27080 tasks: 1166.13671875 MB +worker1-1 | 2024/06/04 17:55:45.745 FIG10b: Memory usage without flush after 27161 tasks: 1165.96484375 MB +worker1-1 | 2024/06/04 17:55:46.746 FIG10b: Memory usage without flush after 27255 tasks: 1166.48046875 MB +worker1-1 | 2024/06/04 17:55:47.748 FIG10b: Memory usage without flush after 27355 tasks: 1167.19140625 MB +worker1-1 | 2024/06/04 17:55:48.7482024/06/04 17:55:48.748 [worker]2024/06/04 17:55:48.749 FIG10b: Memory usage without flush after 27456 tasks: 1167.42578125 MB +worker1-1 | 2024/06/04 17:55:49.7532024/06/04 17:55:49.753 FIG10b: Memory usage without flush after 27563 tasks: 1172.82421875 MB +worker1-1 | 2024/06/04 17:55:50.755 FIG10b: Memory usage without flush after 27659 tasks: 1172.80078125 MB +worker1-1 | [worker]2024/06/04 17:55:51.758 2024/06/04 17:55:51.759 FIG10b: Memory usage without flush after 27762 tasks: 1173.64453125 MB +worker1-1 | 2024/06/04 17:55:52.768 FIG10b: Memory usage without flush after 27859 tasks: 1175.81640625 MB +worker1-1 | 2024/06/04 17:55:53.772 FIG10b: Memory usage without flush after 27947 tasks: 1177.01953125 MB +worker1-1 | 2024/06/04 17:55:54.777 2024/06/04 17:55:54.777 FIG10b: Memory usage without flush after 28036 tasks: 1177.4609375 MB +worker1-1 | 2024/06/04 17:55:55.7832024/06/04 17:55:55.783 FIG10b: Memory usage without flush after 28128 tasks: 1177.91796875 MB +worker1-1 | 2024/06/04 17:55:56.7862024/06/04 17:55:56.786 FIG10b: Memory usage without flush after 28230 tasks: 1177.03125 MB +worker1-1 | 2024/06/04 17:55:57.7862024/06/04 17:55:57.787 FIG10b: Memory usage without flush after 28334 tasks: 1176.80078125 MB +worker1-1 | 2024/06/04 17:55:58.791 FIG10b: Memory usage without flush after 28427 tasks: 1177.3046875 MB +worker1-1 | FIG10b: Memory usage without flush after 28494 tasks: 1177.78125 MB +worker1-1 | 2024/06/04 17:56:00.803 FIG10b: Memory usage without flush after 28573 tasks: 1177.72265625 MB +worker1-1 | 2024/06/04 17:56:01.807 FIG10b: Memory usage without flush after 28662 tasks: 1177.25390625 MB +worker1-1 | 2024/06/04 17:56:02.809 [worker] 2024/06/04 17:56:02.810 FIG10b: Memory usage without flush after 28756 tasks: 1177.80078125 MB +worker1-1 | 2024/06/04 17:56:03.820 FIG10b: Memory usage without flush after 28851 tasks: 1181.1171875 MB +worker1-1 | 2024/06/04 17:56:04.8352024/06/04 17:56:04.835 FIG10b: Memory usage without flush after 28946 tasks: 1181.34765625 MB +worker1-1 | 2024/06/04 17:56:05.8412024/06/04 17:56:05.841 FIG10b: Memory usage without flush after 29033 tasks: 1181.546875 MB +worker1-1 | 2024/06/04 17:56:06.857 FIG10b: Memory usage without flush after 29129 tasks: 1181.5859375 MB +worker1-1 | 2024/06/04 17:56:07.857 FIG10b: Memory usage without flush after 29212 tasks: 1182.09375 MB +worker1-1 | 2024/06/04 17:56:08.863 FIG10b: Memory usage without flush after 29300 tasks: 1181.640625 MB +worker1-1 | 2024/06/04 17:56:09.868 FIG10b: Memory usage without flush after 29414 tasks: 1182.57421875 MB +worker1-1 | 2024/06/04 17:56:10.873 FIG10b: Memory usage without flush after 29521 tasks: 1183.7265625 MB +worker1-1 | 2024/06/04 17:56:11.875 FIG10b: Memory usage without flush after 29623 tasks: 1183.546875 MB +worker1-1 | 2024/06/04 17:56:12.8832024/06/04 17:56:12.883 [worker] gonna store nowFIG10b: Memory usage without flush after 29732 tasks: 1183.8046875 MB +worker1-1 | 2024/06/04 17:56:13.893 FIG10b: Memory usage without flush after 29839 tasks: 1179.33984375 MB +worker1-1 | 2024/06/04 17:56:14.894 FIG10b: Memory usage without flush after 29939 tasks: 1179.7890625 MB +worker1-1 | 2024/06/04 17:56:15.905 FIG10b: Memory usage without flush after 30042 tasks: 1180.2890625 MB +worker1-1 | [worker]2024/06/04 17:56:16.920 FIG10b: Memory usage without flush after 30122 tasks: 1180.04296875 MB +worker1-1 | 2024/06/04 17:56:17.9212024/06/04 17:56:17.922 FIG10b: Memory usage without flush after 30206 tasks: 1179.5859375 MB +worker1-1 | 2024/06/04 17:56:18.923 FIG10b: Memory usage without flush after 30310 tasks: 1180.7734375 MB +worker1-1 | [worker]2024/06/04 17:56:19.930 FIG10b: Memory usage without flush after 30416 tasks: 1180.55859375 MB +worker1-1 | 2024/06/04 17:56:20.930 [worker] stored...2024/06/04 17:56:20.931 FIG10b: Memory usage without flush after 30518 tasks: 1180.31640625 MB +worker1-1 | [worker]2024/06/04 17:56:21.938 FIG10b: Memory usage without flush after 30626 tasks: 1181.28515625 MB +worker1-1 | 2024/06/04 17:56:22.942 FIG10b: Memory usage without flush after 30726 tasks: 1180.00390625 MB +worker1-1 | 2024/06/04 17:56:23.954 FIG10b: Memory usage without flush after 30815 tasks: 1182.046875 MB +worker1-1 | 2024/06/04 17:56:24.958 FIG10b: Memory usage without flush after 30874 tasks: 1183.4375 MB +worker1-1 | 2024/06/04 17:56:25.976 [worker]2024/06/04 17:56:25.977 FIG10b: Memory usage without flush after 30966 tasks: 1183.21875 MB +worker1-1 | 2024/06/04 17:56:26.986[worker] FIG10b: Memory usage without flush after 31058 tasks: 1183.97265625 MB +worker1-1 | [worker][worker]2024/06/04 17:56:27.987 FIG10b: Memory usage without flush after 31155 tasks: 1183.51171875 MB +worker1-1 | gonna store nowtook this many seconds: 0.02612024/06/04 17:56:28.998 FIG10b: Memory usage without flush after 31251 tasks: 1184.0078125 MB +worker1-1 | 2024/06/04 17:56:30.006 FIG10b: Memory usage without flush after 31365 tasks: 1183.65625 MB +worker1-1 | 2024/06/04 17:56:31.0152024/06/04 17:56:31.015 FIG10b: Memory usage without flush after 31473 tasks: 1184.14453125 MB +worker1-1 | 2024/06/04 17:56:32.0182024/06/04 17:56:32.018 FIG10b: Memory usage without flush after 31576 tasks: 1182.58984375 MB +worker1-1 | 2024/06/04 17:56:33.022 FIG10b: Memory usage without flush after 31680 tasks: 1182.53125 MB +worker1-1 | 2024/06/04 17:56:34.030 FIG10b: Memory usage without flush after 31787 tasks: 1186.67578125 MB +worker1-1 | 2024/06/04 17:56:35.033 FIG10b: Memory usage without flush after 31900 tasks: 1185.92578125 MB +worker1-1 | 2024/06/04 17:56:36.033 FIG10b: Memory usage without flush after 32003 tasks: 1174.68359375 MB +worker1-1 | 2024/06/04 17:57:06.033 2024/06/04 17:57:06.0342024/06/04 17:57:06.034 FIG10b: Memory usage without flush after 35229 tasks: 1180.8984375 MB +worker1-1 | 2024/06/04 17:57:36.0392024/06/04 17:57:36.040 FIG10b: Memory usage without flush after 38298 tasks: 1184.65234375 MB +worker1-1 | 2024/06/04 17:58:06.046 FIG10b: Memory usage without flush after 41191 tasks: 1191.8828125 MB +worker1-1 | [worker] 2024/06/04 17:58:36.050 FIG10b: Memory usage without flush after 44182 tasks: 1196.6171875 MB +worker1-1 | 2024/06/04 17:59:06.058 FIG10b: Memory usage without flush after 47052 tasks: 1204.25390625 MB +worker1-1 | 2024/06/04 17:59:36.059 FIG10b: Memory usage without flush after 49891 tasks: 1207.82421875 MB +worker1-1 | 2024/06/04 18:00:06.0692024/06/04 18:00:06.069 FIG10b: Memory usage without flush after 52681 tasks: 1214.05859375 MB +worker1-1 | 2024/06/04 18:00:36.072 2024/06/04 18:00:36.072 FIG10b: Memory usage without flush after 55884 tasks: 1232.2421875 MB +worker1-1 | 2024/06/04 18:01:06.075 FIG10b: Memory usage without flush after 59003 tasks: 1239.90234375 MB +worker1-1 | 2024/06/04 18:01:36.080 FIG10b: Memory usage without flush after 62247 tasks: 1239.94140625 MB +worker1-1 | 2024/06/04 18:02:06.084 [worker]2024/06/04 18:02:06.085 FIG10b: Memory usage without flush after 65488 tasks: 1240.8671875 MB +worker1-1 | FIG10b: Memory usage without flush after 68727 tasks: 1243.9453125 MB +worker1-1 | 2024/06/04 18:03:06.099[worker]2024/06/04 18:03:06.100 FIG10b: Memory usage without flush after 71805 tasks: 1250.90234375 MB +worker1-1 | 2024/06/04 18:03:36.1072024/06/04 18:03:36.107 FIG10b: Memory usage without flush after 74937 tasks: 1256.14453125 MB diff --git a/analysis/output_10b_run6_flush.txt b/analysis/output_10b_run6_flush.txt new file mode 100644 index 0000000..52c51c4 --- /dev/null +++ b/analysis/output_10b_run6_flush.txt @@ -0,0 +1,212 @@ +gcs-1 | 2024/06/04 18:35:55.978 0.4712982177734375 +gcs-1 | 2024/06/04 18:36:00.979 0.6151580810546875 +gcs-1 | 2024/06/04 18:36:05.980 0.730987548828125 +gcs-1 | 2024/06/04 18:36:10.980 0.8366241455078125 +gcs-1 | 2024/06/04 18:36:15.982 0.7630157470703125 +gcs-1 | 2024/06/04 18:36:20.984 0.8029861450195312 +gcs-1 | 2024/06/04 18:36:25.985 0.806304931640625 +gcs-1 | 2024/06/04 18:36:30.986 0.9662704467773438 +gcs-1 | 2024/06/04 18:36:35.986 0.882080078125 +gcs-1 | 2024/06/04 18:36:40.990 0.8871078491210938 +gcs-1 | 2024/06/04 18:36:45.991 0.8853683471679688 +gcs-1 | 2024/06/04 18:36:50.992 0.8856353759765625 +gcs-1 | 2024/06/04 18:36:55.993 0.903106689453125 +gcs-1 | 2024/06/04 18:37:00.994 1.2060394287109375 +gcs-1 | 2024/06/04 18:37:05.995 1.0459136962890625 +gcs-1 | 2024/06/04 18:37:10.998 1.0422439575195312 +gcs-1 | 2024/06/04 18:37:16.001 1.0460891723632812 +gcs-1 | 2024/06/04 18:37:21.002 1.0367889404296875 +gcs-1 | 2024/06/04 18:37:26.005 1.0477371215820312 +gcs-1 | 2024/06/04 18:37:31.007 1.0630111694335938 +gcs-1 | 2024/06/04 18:37:36.008 1.0644607543945312 +gcs-1 | 2024/06/04 18:37:41.008 1.06597900390625 +gcs-1 | 2024/06/04 18:37:46.010 1.07586669921875 +gcs-1 | 2024/06/04 18:37:51.010 1.0880126953125 +gcs-1 | 2024/06/04 18:37:56.011 1.7011642456054688 +gcs-1 | 2024/06/04 18:38:01.012 1.6937637329101562 +gcs-1 | 2024/06/04 18:38:06.015 1.35693359375 +gcs-1 | 2024/06/04 18:38:11.016 1.3575210571289062 +gcs-1 | 2024/06/04 18:38:16.016 1.3592376708984375 +gcs-1 | 2024/06/04 18:38:21.017 1.355438232421875 +gcs-1 | 2024/06/04 18:38:26.018 1.3642654418945312 +gcs-1 | 2024/06/04 18:38:31.019 1.3612899780273438 +gcs-1 | 2024/06/04 18:38:36.020 1.35736083984375 +gcs-1 | 2024/06/04 18:38:41.021 1.36566162109375 +gcs-1 | 2024/06/04 18:38:46.025 1.3725051879882812 +gcs-1 | 2024/06/04 18:38:51.027 1.3767852783203125 +gcs-1 | 2024/06/04 18:38:56.028 1.3851394653320312 +gcs-1 | 2024/06/04 18:39:01.029 1.391815185546875 +gcs-1 | 2024/06/04 18:39:06.030 1.3970184326171875 +gcs-1 | 2024/06/04 18:39:11.032 1.4043045043945312 +gcs-1 | 2024/06/04 18:39:16.034 1.413177490234375 +gcs-1 | 2024/06/04 18:39:21.034 1.4253005981445312 +gcs-1 | 2024/06/04 18:39:26.036 1.4256057739257812 +gcs-1 | 2024/06/04 18:39:31.038 1.4508438110351562 +gcs-1 | 2024/06/04 18:39:36.038 1.4605636596679688 +gcs-1 | 2024/06/04 18:39:41.040 2.6618728637695312 +gcs-1 | 2024/06/04 18:39:46.042 2.6613845825195312 +gcs-1 | 2024/06/04 18:39:51.043 2.6645889282226562 +gcs-1 | 2024/06/04 18:39:56.044 2.64996337890625 +gcs-1 | 2024/06/04 18:40:01.046 2.6588897705078125 +gcs-1 | 2024/06/04 18:40:06.048 1.9665908813476562 +gcs-1 | 2024/06/04 18:40:11.049 1.9753646850585938 +gcs-1 | 2024/06/04 18:40:16.050 1.9753799438476562 +gcs-1 | 2024/06/04 18:40:21.050 1.9930038452148438 +gcs-1 | 2024/06/04 18:40:26.051 1.9843673706054688 +gcs-1 | 2024/06/04 18:40:31.052 1.9817047119140625 +gcs-1 | 2024/06/04 18:40:36.053 1.9836349487304688 +gcs-1 | 2024/06/04 18:40:41.053 1.9745941162109375 +gcs-1 | 2024/06/04 18:40:46.057 1.9837646484375 +gcs-1 | 2024/06/04 18:40:51.059 1.9785537719726562 +gcs-1 | 2024/06/04 18:40:56.059 1.990203857421875 +gcs-1 | 2024/06/04 18:41:01.061 1.7728424072265625 +gcs-1 | 2024/06/04 18:41:06.063 1.79400634765625 +gcs-1 | 2024/06/04 18:41:11.065 1.7856674194335938 +gcs-1 | 2024/06/04 18:41:16.066 1.793914794921875 +gcs-1 | 2024/06/04 18:41:21.067 1.7599563598632812 +gcs-1 | 2024/06/04 18:41:26.069 1.794189453125 +gcs-1 | 2024/06/04 18:41:31.070 1.7983551025390625 +gcs-1 | 2024/06/04 18:41:36.072 1.7949447631835938 +gcs-1 | 2024/06/04 18:41:41.073 1.8064117431640625 +gcs-1 | 2024/06/04 18:41:46.077 1.8233795166015625 +gcs-1 | 2024/06/04 18:41:51.079 1.81158447265625 +gcs-1 | 2024/06/04 18:41:56.080 1.8395156860351562 +gcs-1 | 2024/06/04 18:42:01.081 1.827056884765625 +gcs-1 | 2024/06/04 18:42:06.082 1.8432846069335938 +gcs-1 | 2024/06/04 18:42:11.083 1.8462371826171875 +gcs-1 | 2024/06/04 18:42:16.083 1.8495559692382812 +gcs-1 | 2024/06/04 18:42:21.084 1.8671875 +gcs-1 | 2024/06/04 18:42:26.085 1.8811187744140625 +gcs-1 | 2024/06/04 18:42:31.086 1.895721435546875 +gcs-1 | 2024/06/04 18:42:36.087 1.8936309814453125 +gcs-1 | 2024/06/04 18:42:41.088 1.8928680419921875 +gcs-1 | 2024/06/04 18:42:46.093 1.9055862426757812 +gcs-1 | 2024/06/04 18:42:51.094 1.9277496337890625 +gcs-1 | 2024/06/04 18:42:56.095 1.9287490844726562 +gcs-1 | 2024/06/04 18:43:01.097 1.937286376953125 +gcs-1 | 2024/06/04 18:43:06.097 1.9503936767578125 +gcs-1 | 2024/06/04 18:43:11.099 1.9545745849609375 +gcs-1 | 2024/06/04 18:43:16.101 1.988128662109375 +gcs-1 | 2024/06/04 18:43:21.102 4.376167297363281 +gcs-1 | 2024/06/04 18:43:26.103 4.376502990722656 +gcs-1 | 2024/06/04 18:43:31.105 4.377189636230469 +gcs-1 | 2024/06/04 18:43:36.106 4.394599914550781 +gcs-1 | 2024/06/04 18:43:41.106 4.396209716796875 +gcs-1 | 2024/06/04 18:43:46.109 4.377449035644531 +gcs-1 | 2024/06/04 18:43:51.112 4.3657379150390625 +gcs-1 | 2024/06/04 18:43:56.114 4.391792297363281 +gcs-1 | 2024/06/04 18:44:01.116 4.393150329589844 +gcs-1 | 2024/06/04 18:44:06.118 4.379981994628906 +gcs-1 | 2024/06/04 18:44:11.118 3.0070571899414062 +gcs-1 | 2024/06/04 18:44:16.122 3.0190658569335938 +gcs-1 | 2024/06/04 18:44:21.124 3.0153961181640625 +gcs-1 | 2024/06/04 18:44:26.126 3.0196151733398438 +gcs-1 | 2024/06/04 18:44:31.127 3.0075531005859375 +gcs-1 | 2024/06/04 18:44:36.129 3.0203094482421875 +gcs-1 | 2024/06/04 18:44:41.130 3.0154876708984375 +gcs-1 | 2024/06/04 18:44:46.132 3.0289764404296875 +gcs-1 | 2024/06/04 18:44:51.133 3.02105712890625 +gcs-1 | 2024/06/04 18:44:56.135 3.0169143676757812 +gcs-1 | 2024/06/04 18:45:01.152 3.0152587890625 +gcs-1 | 2024/06/04 18:45:06.153 3.0446090698242188 +gcs-1 | 2024/06/04 18:45:11.156 3.0236053466796875 +gcs-1 | 2024/06/04 18:45:16.158 3.037078857421875 +gcs-1 | 2024/06/04 18:45:21.159 3.031402587890625 +gcs-1 | 2024/06/04 18:45:26.162 3.00067138671875 +gcs-1 | 2024/06/04 18:45:31.163 3.0385284423828125 +gcs-1 | 2024/06/04 18:45:36.164 3.0350189208984375 +gcs-1 | 2024/06/04 18:45:41.166 3.0415420532226562 +gcs-1 | 2024/06/04 18:45:46.169 3.027008056640625 +gcs-1 | 2024/06/04 18:45:51.170 3.0510482788085938 +gcs-1 | 2024/06/04 18:45:56.171 3.0405120849609375 +gcs-1 | 2024/06/04 18:46:01.173 3.040008544921875 +gcs-1 | 2024/06/04 18:46:06.174 3.0400543212890625 +gcs-1 | 2024/06/04 18:46:11.176 3.0528411865234375 +gcs-1 | 2024/06/04 18:46:16.179 3.0454940795898438 +gcs-1 | 2024/06/04 18:46:21.180 3.0629653930664062 +gcs-1 | 2024/06/04 18:46:26.181 3.064178466796875 +gcs-1 | 2024/06/04 18:46:31.183 3.056488037109375 +gcs-1 | 2024/06/04 18:46:36.184 3.0559768676757812 +gcs-1 | 2024/06/04 18:46:41.186 3.049346923828125 +gcs-1 | 2024/06/04 18:46:46.187 3.0491104125976562 +gcs-1 | 2024/06/04 18:46:51.188 3.070220947265625 +gcs-1 | 2024/06/04 18:46:56.189 3.0629425048828125 +gcs-1 | 2024/06/04 18:47:01.190 3.0534286499023438 +gcs-1 | 2024/06/04 18:47:06.191 3.0541229248046875 +gcs-1 | 2024/06/04 18:47:11.193 3.063262939453125 +gcs-1 | 2024/06/04 18:47:16.197 3.066314697265625 +gcs-1 | 2024/06/04 18:47:21.198 3.0680618286132812 +gcs-1 | 2024/06/04 18:47:26.199 3.0597457885742188 +gcs-1 | 2024/06/04 18:47:31.213 3.02667236328125 +gcs-1 | 2024/06/04 18:47:36.214 3.0867996215820312 +gcs-1 | 2024/06/04 18:47:41.215 3.0983428955078125 +gcs-1 | 2024/06/04 18:47:46.217 3.0848159790039062 +gcs-1 | 2024/06/04 18:47:51.218 3.1069259643554688 +gcs-1 | 2024/06/04 18:47:56.219 3.1214981079101562 +gcs-1 | 2024/06/04 18:48:01.220 3.0757522583007812 +gcs-1 | 2024/06/04 18:48:06.221 3.1439437866210938 +gcs-1 | 2024/06/04 18:48:11.224 3.1282882690429688 +gcs-1 | 2024/06/04 18:48:16.226 3.134124755859375 +gcs-1 | 2024/06/04 18:48:21.227 3.1048660278320312 +gcs-1 | 2024/06/04 18:48:26.227 3.1502838134765625 +gcs-1 | 2024/06/04 18:48:31.228 3.1458282470703125 +gcs-1 | 2024/06/04 18:48:36.229 3.1410675048828125 +gcs-1 | 2024/06/04 18:48:41.237 3.1520156860351562 +gcs-1 | 2024/06/04 18:48:46.239 3.1770706176757812 +gcs-1 | 2024/06/04 18:48:51.240 3.1921844482421875 +gcs-1 | 2024/06/04 18:48:56.241 3.1827392578125 +gcs-1 | 2024/06/04 18:49:01.243 3.1824722290039062 +gcs-1 | 2024/06/04 18:49:06.244 3.2339859008789062 +gcs-1 | 2024/06/04 18:49:11.245 3.1619110107421875 +gcs-1 | 2024/06/04 18:49:16.247 3.2211456298828125 +gcs-1 | 2024/06/04 18:49:21.270 3.2269287109375 +gcs-1 | 2024/06/04 18:49:26.271 3.24176025390625 +gcs-1 | 2024/06/04 18:49:31.272 3.265350341796875 +gcs-1 | 2024/06/04 18:49:36.274 3.272674560546875 +gcs-1 | 2024/06/04 18:49:41.276 3.2856063842773438 +gcs-1 | 2024/06/04 18:49:46.277 3.3036575317382812 +gcs-1 | 2024/06/04 18:49:51.278 3.3234634399414062 +gcs-1 | 2024/06/04 18:49:56.279 3.321624755859375 +gcs-1 | 2024/06/04 18:50:01.281 3.3337173461914062 +gcs-1 | 2024/06/04 18:50:06.282 3.352447509765625 +gcs-1 | 2024/06/04 18:50:11.283 3.3528671264648438 +gcs-1 | 2024/06/04 18:50:16.285 3.3671417236328125 +gcs-1 | 2024/06/04 18:50:21.287 3.3683624267578125 +gcs-1 | 2024/06/04 18:50:26.288 3.394073486328125 +gcs-1 | 2024/06/04 18:50:31.290 3.3882980346679688 +gcs-1 | 2024/06/04 18:50:36.291 3.4113845825195312 +gcs-1 | 2024/06/04 18:50:41.298 3.3700027465820312 +gcs-1 | 2024/06/04 18:50:46.298 3.4124832153320312 +gcs-1 | 2024/06/04 18:50:51.300 3.3822402954101562 +gcs-1 | 2024/06/04 18:50:56.302 3.4241943359375 +gcs-1 | 2024/06/04 18:51:01.303 3.4392852783203125 +gcs-1 | 2024/06/04 18:51:06.310 3.3993606567382812 +gcs-1 | 2024/06/04 18:51:11.313 3.456268310546875 +gcs-1 | 2024/06/04 18:51:16.316 3.444488525390625 +gcs-1 | 2024/06/04 18:51:21.317 3.4685821533203125 +gcs-1 | 2024/06/04 18:51:26.318 8.228248596191406 +gcs-1 | 2024/06/04 18:51:31.385 8.261550903320312 +gcs-1 | 2024/06/04 18:51:36.387 8.248344421386719 +gcs-1 | 2024/06/04 18:51:41.388 8.290199279785156 +gcs-1 | 2024/06/04 18:51:46.389 8.281494140625 +gcs-1 | 2024/06/04 18:51:51.390 8.303192138671875 +gcs-1 | 2024/06/04 18:51:56.391 8.26129150390625 +gcs-1 | 2024/06/04 18:52:01.393 8.272514343261719 +gcs-1 | 2024/06/04 18:52:06.396 8.297195434570312 +gcs-1 | 2024/06/04 18:52:11.418 8.257728576660156 +gcs-1 | 2024/06/04 18:52:16.421 8.254081726074219 +gcs-1 | 2024/06/04 18:52:21.422 8.299392700195312 +gcs-1 | 2024/06/04 18:52:26.424 8.278854370117188 +gcs-1 | 2024/06/04 18:52:31.425 8.324920654296875 +gcs-1 | 2024/06/04 18:52:36.426 8.291427612304688 +gcs-1 | 2024/06/04 18:52:41.427 8.329216003417969 +gcs-1 | 2024/06/04 18:52:46.428 8.279190063476562 +gcs-1 | 2024/06/04 18:52:51.429 8.315376281738281 +gcs-1 | 2024/06/04 18:52:56.430 8.319190979003906 +gcs-1 | 2024/06/04 18:53:01.431 8.326942443847656 +gcs-1 | 2024/06/04 18:53:06.433 8.319160461425781 +gcs-1 | 2024/06/04 18:53:11.433 8.320426940917969 +gcs-1 | 2024/06/04 18:53:16.435 8.341056823730469 +gcs-1 | 2024/06/04 18:53:21.437 5.562217712402344 +gcs-1 | 2024/06/04 18:53:26.440 5.5629730224609375 +gcs-1 | 2024/06/04 18:53:31.441 5.592796325683594 diff --git a/analysis/output_10b_run7_BADRESULT_dontflush.txt b/analysis/output_10b_run7_BADRESULT_dontflush.txt new file mode 100644 index 0000000..36f5def --- /dev/null +++ b/analysis/output_10b_run7_BADRESULT_dontflush.txt @@ -0,0 +1,114 @@ +gcs-1 | 2024/06/04 18:58:14 server listening at [::]:50000 +gcs-1 | 2024/06/04 18:58:14.414 server listening at [::]:50001 +gcs-1 | 2024/06/04 18:58:14.416 0.4748687744140625 +gcs-1 | 2024/06/04 18:58:19.417 0.7540206909179688 +gcs-1 | 2024/06/04 18:58:24.418 1.065032958984375 +gcs-1 | 2024/06/04 18:58:29.419 1.0816116333007812 +gcs-1 | 2024/06/04 18:58:34.420 1.4495162963867188 +gcs-1 | 2024/06/04 18:58:39.422 1.53350830078125 +gcs-1 | 2024/06/04 18:58:44.423 1.61444091796875 +gcs-1 | 2024/06/04 18:58:49.424 2.6472244262695312 +gcs-1 | 2024/06/04 18:58:54.425 2.9524612426757812 +gcs-1 | 2024/06/04 18:58:59.426 2.374725341796875 +gcs-1 | 2024/06/04 18:59:04.427 2.5830154418945312 +gcs-1 | 2024/06/04 18:59:09.429 2.9528121948242188 +gcs-1 | 2024/06/04 18:59:14.432 3.2568206787109375 +gcs-1 | 2024/06/04 18:59:19.433 3.0411911010742188 +gcs-1 | 2024/06/04 18:59:24.433 2.9720611572265625 +gcs-1 | 2024/06/04 18:59:29.435 3.58203125 +gcs-1 | 2024/06/04 18:59:34.436 2.4860000610351562 +gcs-1 | 2024/06/04 18:59:39.437 3.1765060424804688 +gcs-1 | 2024/06/04 18:59:44.439 2.0571365356445312 +gcs-1 | 2024/06/04 18:59:49.441 2.7280654907226562 +gcs-1 | 2024/06/04 18:59:54.441 3.5274429321289062 +gcs-1 | 2024/06/04 18:59:59.442 3.6893539428710938 +gcs-1 | 2024/06/04 19:00:04.444 2.2642440795898438 +gcs-1 | 2024/06/04 19:00:09.446 2.71295166015625 +gcs-1 | 2024/06/04 19:00:14.447 6.010200500488281 +gcs-1 | 2024/06/04 19:00:19.449 5.968513488769531 +gcs-1 | 2024/06/04 19:00:24.450 3.9825515747070312 +gcs-1 | 2024/06/04 19:00:29.450 3.7501296997070312 +gcs-1 | 2024/06/04 19:00:34.451 3.5324172973632812 +gcs-1 | 2024/06/04 19:00:39.453 3.3953094482421875 +gcs-1 | 2024/06/04 19:00:44.456 3.1057891845703125 +gcs-1 | 2024/06/04 19:00:49.458 2.8861236572265625 +gcs-1 | 2024/06/04 19:00:54.459 5.432861328125 +gcs-1 | 2024/06/04 19:00:59.460 5.0107574462890625 +gcs-1 | 2024/06/04 19:01:04.461 4.550201416015625 +gcs-1 | 2024/06/04 19:01:09.462 4.207878112792969 +gcs-1 | 2024/06/04 19:01:14.462 3.9461746215820312 +gcs-1 | 2024/06/04 19:01:19.463 3.7596893310546875 +gcs-1 | 2024/06/04 19:01:24.465 3.5427474975585938 +gcs-1 | 2024/06/04 19:01:29.465 3.2588424682617188 +gcs-1 | 2024/06/04 19:01:34.466 5.787635803222656 +gcs-1 | 2024/06/04 19:01:39.467 5.4891204833984375 +gcs-1 | 2024/06/04 19:01:44.469 5.175819396972656 +gcs-1 | 2024/06/04 19:01:49.470 4.7996063232421875 +gcs-1 | 2024/06/04 19:01:54.471 4.439849853515625 +gcs-1 | 2024/06/04 19:01:59.471 7.001838684082031 +gcs-1 | 2024/06/04 19:02:04.473 9.541595458984375 +gcs-1 | 2024/06/04 19:02:09.474 12.116615295410156 +gcs-1 | 2024/06/04 19:02:14.477 8.2061767578125 +gcs-1 | 2024/06/04 19:02:19.479 10.763824462890625 +gcs-1 | 2024/06/04 19:02:24.480 5.2975616455078125 +gcs-1 | 2024/06/04 19:02:29.481 7.9277191162109375 +gcs-1 | 2024/06/04 19:02:34.482 6.046714782714844 +gcs-1 | 2024/06/04 19:02:39.483 8.58746337890625 +gcs-1 | 2024/06/04 19:02:44.483 6.536651611328125 +gcs-1 | 2024/06/04 19:02:49.484 8.646133422851562 +gcs-1 | 2024/06/04 19:02:54.485 6.558677673339844 +gcs-1 | 2024/06/04 19:02:59.486 9.062904357910156 +gcs-1 | 2024/06/04 19:03:04.488 6.9549560546875 +gcs-1 | 2024/06/04 19:03:09.490 4.9092559814453125 +gcs-1 | 2024/06/04 19:03:14.491 7.502021789550781 +gcs-1 | 2024/06/04 19:03:19.492 5.123695373535156 +gcs-1 | 2024/06/04 19:03:24.493 7.7623138427734375 +gcs-1 | 2024/06/04 19:03:29.495 5.789070129394531 +gcs-1 | 2024/06/04 19:03:34.497 8.377128601074219 +gcs-1 | 2024/06/04 19:03:39.498 6.492942810058594 +gcs-1 | 2024/06/04 19:03:44.501 4.676490783691406 +gcs-1 | 2024/06/04 19:03:49.502 7.218788146972656 +gcs-1 | 2024/06/04 19:03:54.503 5.489463806152344 +gcs-1 | 2024/06/04 19:03:59.504 8.080680847167969 +gcs-1 | 2024/06/04 19:04:04.505 6.2022552490234375 +gcs-1 | 2024/06/04 19:04:09.506 8.754859924316406 +gcs-1 | 2024/06/04 19:04:14.509 6.789039611816406 +gcs-1 | 2024/06/04 19:04:19.510 9.362159729003906 +gcs-1 | 2024/06/04 19:04:24.512 7.197380065917969 +gcs-1 | 2024/06/04 19:04:29.513 5.1575927734375 +gcs-1 | 2024/06/04 19:04:34.514 7.810218811035156 +gcs-1 | 2024/06/04 19:04:39.516 5.3560943603515625 +gcs-1 | 2024/06/04 19:04:44.647 5.6973419189453125 +gcs-1 | 2024/06/04 19:04:49.667 7.562110900878906 +gcs-1 | 2024/06/04 19:04:54.669 5.394920349121094 +gcs-1 | 2024/06/04 19:04:59.673 7.6736907958984375 +gcs-1 | 2024/06/04 19:10:15.133 6.317527770996094 +gcs-1 | 2024/06/04 19:10:20.135 7.560111999511719 +gcs-1 | 2024/06/04 19:10:25.151 9.531242370605469 +gcs-1 | 2024/06/04 19:10:30.279 9.783424377441406 +gcs-1 | 2024/06/04 19:10:35.300 5.319740295410156 +gcs-1 | 2024/06/04 19:10:40.301 5.928047180175781 +gcs-1 | 2024/06/04 19:10:45.148 6.508811950683594 +gcs-1 | 2024/06/04 19:10:50.152 7.157768249511719 +gcs-1 | 2024/06/04 19:10:55.246 7.735542297363281 +gcs-1 | 2024/06/04 19:12:29.466 7.840553283691406 +gcs-1 | 2024/06/04 19:12:34.468 5.777587890625 +gcs-1 | 2024/06/04 19:12:39.469 8.519912719726562 +gcs-1 | 2024/06/04 19:12:44.471 9.306671142578125 +gcs-1 | 2024/06/04 19:12:49.476 9.999237060546875 +gcs-1 | 2024/06/04 19:12:54.488 5.756217956542969 +gcs-1 | 2024/06/04 19:12:59.491 6.480560302734375 +gcs-1 | 2024/06/04 19:18:21.785 6.981689453125 +gcs-1 | 2024/06/04 19:18:26.785 7.7218780517578125 +gcs-1 | 2024/06/04 19:19:00.886 10.090690612792969 +gcs-1 | 2024/06/04 19:19:05.890 14.087242126464844 +gcs-1 | 2024/06/04 19:19:10.892 15.348991394042969 +gcs-1 | 2024/06/04 19:19:15.894 16.777236938476562 +gcs-1 | 2024/06/04 19:19:20.895 19.112342834472656 +gcs-1 | 2024/06/04 19:19:25.896 14.24517822265625 +gcs-1 | 2024/06/04 19:19:30.902 16.9776611328125 +gcs-1 | 2024/06/04 19:19:35.905 19.4801025390625 +gcs-1 | 2024/06/04 19:19:40.906 21.99610137939453 +gcs-1 | 2024/06/04 19:19:45.988 9.858772277832031 +gcs-1 | 2024/06/04 19:19:50.993 11.033058166503906 +gcs-1 | 2024/06/04 19:19:55.995 13.123298645019531 diff --git a/analysis/output_10b_run8_dontflush.txt b/analysis/output_10b_run8_dontflush.txt new file mode 100644 index 0000000..f2fa4a8 --- /dev/null +++ b/analysis/output_10b_run8_dontflush.txt @@ -0,0 +1,192 @@ +gcs-1 | 2024/06/04 19:21:27.363 0.47129058837890625 +gcs-1 | 2024/06/04 19:21:32.370 0.7487564086914062 +gcs-1 | 2024/06/04 19:21:37.371 0.8387985229492188 +gcs-1 | 2024/06/04 19:21:42.372 3.419403076171875 +gcs-1 | 2024/06/04 19:21:47.373 3.4430770874023438 +gcs-1 | 2024/06/04 19:21:52.373 1.003143310546875 +gcs-1 | 2024/06/04 19:21:57.374 0.9983901977539062 +gcs-1 | 2024/06/04 19:22:02.375 2.0103912353515625 +gcs-1 | 2024/06/04 19:22:07.377 2.3386611938476562 +gcs-1 | 2024/06/04 19:22:12.378 2.3165206909179688 +gcs-1 | 2024/06/04 19:22:17.380 2.5964279174804688 +gcs-1 | 2024/06/04 19:22:22.381 2.657073974609375 +gcs-1 | 2024/06/04 19:22:27.382 2.8733978271484375 +gcs-1 | 2024/06/04 19:22:32.385 2.5427169799804688 +gcs-1 | 2024/06/04 19:22:37.387 2.2776947021484375 +gcs-1 | 2024/06/04 19:22:42.388 2.8539962768554688 +gcs-1 | 2024/06/04 19:22:47.390 3.3134765625 +gcs-1 | 2024/06/04 19:22:52.391 2.057342529296875 +gcs-1 | 2024/06/04 19:22:57.392 2.6971817016601562 +gcs-1 | 2024/06/04 19:23:02.393 3.2243499755859375 +gcs-1 | 2024/06/04 19:23:07.395 3.4471054077148438 +gcs-1 | 2024/06/04 19:23:12.395 1.925506591796875 +gcs-1 | 2024/06/04 19:23:17.397 2.5724411010742188 +gcs-1 | 2024/06/04 19:23:22.398 2.974029541015625 +gcs-1 | 2024/06/04 19:23:27.399 3.4613037109375 +gcs-1 | 2024/06/04 19:23:32.399 6.0098876953125 +gcs-1 | 2024/06/04 19:23:37.400 5.898078918457031 +gcs-1 | 2024/06/04 19:23:42.402 3.8481826782226562 +gcs-1 | 2024/06/04 19:23:47.403 3.360137939453125 +gcs-1 | 2024/06/04 19:23:52.404 3.246429443359375 +gcs-1 | 2024/06/04 19:23:57.407 3.1309356689453125 +gcs-1 | 2024/06/04 19:24:02.410 3.0131988525390625 +gcs-1 | 2024/06/04 19:24:07.410 2.9094314575195312 +gcs-1 | 2024/06/04 19:24:12.411 5.452568054199219 +gcs-1 | 2024/06/04 19:24:17.412 4.906715393066406 +gcs-1 | 2024/06/04 19:24:22.413 4.477287292480469 +gcs-1 | 2024/06/04 19:24:27.415 4.377899169921875 +gcs-1 | 2024/06/04 19:24:32.418 4.270896911621094 +gcs-1 | 2024/06/04 19:24:37.419 4.158775329589844 +gcs-1 | 2024/06/04 19:24:42.420 4.088165283203125 +gcs-1 | 2024/06/04 19:24:47.420 3.9782638549804688 +gcs-1 | 2024/06/04 19:24:52.422 3.7663192749023438 +gcs-1 | 2024/06/04 19:24:57.423 3.4873809814453125 +gcs-1 | 2024/06/04 19:25:02.426 3.2296524047851562 +gcs-1 | 2024/06/04 19:25:07.427 5.927833557128906 +gcs-1 | 2024/06/04 19:25:12.428 5.4492340087890625 +gcs-1 | 2024/06/04 19:25:17.429 7.936981201171875 +gcs-1 | 2024/06/04 19:25:22.432 10.604972839355469 +gcs-1 | 2024/06/04 19:25:27.432 6.833213806152344 +gcs-1 | 2024/06/04 19:25:32.434 9.40277099609375 +gcs-1 | 2024/06/04 19:25:37.435 12.065711975097656 +gcs-1 | 2024/06/04 19:25:42.436 6.60052490234375 +gcs-1 | 2024/06/04 19:25:47.437 4.7006988525390625 +gcs-1 | 2024/06/04 19:25:52.439 7.328437805175781 +gcs-1 | 2024/06/04 19:25:57.440 5.4693603515625 +gcs-1 | 2024/06/04 19:26:02.439 7.106376647949219 +gcs-1 | 2024/06/04 19:26:07.440 4.885688781738281 +gcs-1 | 2024/06/04 19:26:12.441 7.247566223144531 +gcs-1 | 2024/06/04 19:26:17.442 5.273231506347656 +gcs-1 | 2024/06/04 19:26:22.443 7.742225646972656 +gcs-1 | 2024/06/04 19:26:27.444 5.845916748046875 +gcs-1 | 2024/06/04 19:26:32.448 8.254135131835938 +gcs-1 | 2024/06/04 19:26:37.449 6.340934753417969 +gcs-1 | 2024/06/04 19:26:42.449 8.846717834472656 +gcs-1 | 2024/06/04 19:26:47.450 6.9485931396484375 +gcs-1 | 2024/06/04 19:26:52.452 5.043464660644531 +gcs-1 | 2024/06/04 19:26:57.453 7.529380798339844 +gcs-1 | 2024/06/04 19:27:02.454 5.5513153076171875 +gcs-1 | 2024/06/04 19:27:07.455 7.921234130859375 +gcs-1 | 2024/06/04 19:27:12.455 5.634376525878906 +gcs-1 | 2024/06/04 19:27:17.457 8.166412353515625 +gcs-1 | 2024/06/04 19:27:22.460 5.962432861328125 +gcs-1 | 2024/06/04 19:27:27.460 8.487312316894531 +gcs-1 | 2024/06/04 19:27:32.462 6.260498046875 +gcs-1 | 2024/06/04 19:27:37.463 8.73968505859375 +gcs-1 | 2024/06/04 19:27:42.464 6.531364440917969 +gcs-1 | 2024/06/04 19:27:47.465 9.033126831054688 +gcs-1 | 2024/06/04 19:27:52.468 6.6288299560546875 +gcs-1 | 2024/06/04 19:27:57.469 9.079505920410156 +gcs-1 | 2024/06/04 19:28:02.471 6.7116241455078125 +gcs-1 | 2024/06/04 19:28:07.473 9.220306396484375 +gcs-1 | 2024/06/04 19:28:12.676 6.7909088134765625 +gcs-1 | 2024/06/04 19:28:17.677 8.9853515625 +gcs-1 | 2024/06/04 19:28:22.679 6.484397888183594 +gcs-1 | 2024/06/04 19:28:27.680 8.861106872558594 +gcs-1 | 2024/06/04 19:28:32.681 6.506874084472656 +gcs-1 | 2024/06/04 19:28:37.682 9.062728881835938 +gcs-1 | 2024/06/04 19:28:42.683 6.430274963378906 +gcs-1 | 2024/06/04 19:28:47.684 8.7381591796875 +gcs-1 | 2024/06/04 19:28:52.685 6.049659729003906 +gcs-1 | 2024/06/04 19:28:57.687 8.569046020507812 +gcs-1 | 2024/06/04 19:29:02.685 13.198020935058594 +gcs-1 | 2024/06/04 19:29:07.686 15.540107727050781 +gcs-1 | 2024/06/04 19:29:12.687 17.877342224121094 +gcs-1 | 2024/06/04 19:29:17.688 12.938446044921875 +gcs-1 | 2024/06/04 19:29:22.690 15.421417236328125 +gcs-1 | 2024/06/04 19:29:27.690 17.76190948486328 +gcs-1 | 2024/06/04 19:29:32.694 20.038368225097656 +gcs-1 | 2024/06/04 19:29:37.696 22.490615844726562 +gcs-1 | 2024/06/04 19:29:42.697 10.978309631347656 +gcs-1 | 2024/06/04 19:29:47.698 13.442184448242188 +gcs-1 | 2024/06/04 19:29:52.699 15.766464233398438 +gcs-1 | 2024/06/04 19:29:57.701 18.161224365234375 +gcs-1 | 2024/06/04 19:30:02.703 9.691146850585938 +gcs-1 | 2024/06/04 19:30:07.704 11.990638732910156 +gcs-1 | 2024/06/04 19:30:12.706 14.3824462890625 +gcs-1 | 2024/06/04 19:30:17.708 8.759857177734375 +gcs-1 | 2024/06/04 19:30:22.709 11.142829895019531 +gcs-1 | 2024/06/04 19:30:27.710 13.604194641113281 +gcs-1 | 2024/06/04 19:30:32.712 16.05791473388672 +gcs-1 | 2024/06/04 19:30:37.713 10.415145874023438 +gcs-1 | 2024/06/04 19:30:42.713 12.878974914550781 +gcs-1 | 2024/06/04 19:30:47.714 15.367881774902344 +gcs-1 | 2024/06/04 19:30:52.715 9.372314453125 +gcs-1 | 2024/06/04 19:30:57.718 11.754783630371094 +gcs-1 | 2024/06/04 19:31:02.719 14.211502075195312 +gcs-1 | 2024/06/04 19:31:07.720 16.67999267578125 +gcs-1 | 2024/06/04 19:31:12.721 10.782791137695312 +gcs-1 | 2024/06/04 19:31:17.722 13.280471801757812 +gcs-1 | 2024/06/04 19:31:22.724 15.736763000488281 +gcs-1 | 2024/06/04 19:31:27.725 10.063522338867188 +gcs-1 | 2024/06/04 19:31:32.728 12.499763488769531 +gcs-1 | 2024/06/04 19:31:37.730 14.886199951171875 +gcs-1 | 2024/06/04 19:31:42.731 9.108489990234375 +gcs-1 | 2024/06/04 19:31:47.733 11.585563659667969 +gcs-1 | 2024/06/04 19:31:52.735 14.075881958007812 +gcs-1 | 2024/06/04 19:31:57.736 16.557769775390625 +gcs-1 | 2024/06/04 19:32:02.737 10.810867309570312 +gcs-1 | 2024/06/04 19:32:07.739 13.184783935546875 +gcs-1 | 2024/06/04 19:32:12.740 15.646293640136719 +gcs-1 | 2024/06/04 19:32:17.740 9.857810974121094 +gcs-1 | 2024/06/04 19:32:22.742 12.212295532226562 +gcs-1 | 2024/06/04 19:32:27.743 14.596710205078125 +gcs-1 | 2024/06/04 19:32:32.744 9.035865783691406 +gcs-1 | 2024/06/04 19:32:37.745 11.638259887695312 +gcs-1 | 2024/06/04 19:32:42.746 14.198501586914062 +gcs-1 | 2024/06/04 19:32:47.748 16.848388671875 +gcs-1 | 2024/06/04 19:32:52.748 11.137123107910156 +gcs-1 | 2024/06/04 19:32:57.749 13.722923278808594 +gcs-1 | 2024/06/04 19:33:02.751 16.373641967773438 +gcs-1 | 2024/06/04 19:33:07.753 9.919609069824219 +gcs-1 | 2024/06/04 19:33:12.754 12.312255859375 +gcs-1 | 2024/06/04 19:33:17.756 14.829246520996094 +gcs-1 | 2024/06/04 19:33:22.757 9.050392150878906 +gcs-1 | 2024/06/04 19:33:27.758 11.583648681640625 +gcs-1 | 2024/06/04 19:33:32.760 14.016876220703125 +gcs-1 | 2024/06/04 19:33:37.761 16.556686401367188 +gcs-1 | 2024/06/04 19:33:42.773 10.625007629394531 +gcs-1 | 2024/06/04 19:33:47.775 13.024528503417969 +gcs-1 | 2024/06/04 19:33:52.776 15.596565246582031 +gcs-1 | 2024/06/04 19:33:57.777 9.745750427246094 +gcs-1 | 2024/06/04 19:34:02.778 12.34716796875 +gcs-1 | 2024/06/04 19:34:07.779 14.959617614746094 +gcs-1 | 2024/06/04 19:34:12.780 17.524765014648438 +gcs-1 | 2024/06/04 19:34:17.782 11.546844482421875 +gcs-1 | 2024/06/04 19:34:22.784 14.118873596191406 +gcs-1 | 2024/06/04 19:34:27.785 16.70843505859375 +gcs-1 | 2024/06/04 19:34:32.784 10.193046569824219 +gcs-1 | 2024/06/04 19:34:37.814 12.485702514648438 +gcs-1 | 2024/06/04 19:34:42.816 14.736854553222656 +gcs-1 | 2024/06/04 19:34:47.817 17.065345764160156 +gcs-1 | 2024/06/04 19:34:52.818 10.659378051757812 +gcs-1 | 2024/06/04 19:34:57.819 13.1251220703125 +gcs-1 | 2024/06/04 19:35:02.823 15.710685729980469 +gcs-1 | 2024/06/04 19:35:07.824 18.30474090576172 +gcs-1 | 2024/06/04 19:35:12.825 12.003318786621094 +gcs-1 | 2024/06/04 19:35:17.827 14.653106689453125 +gcs-1 | 2024/06/04 19:35:22.828 17.357589721679688 +gcs-1 | 2024/06/04 19:35:27.829 11.02642822265625 +gcs-1 | 2024/06/04 19:35:32.830 13.667793273925781 +gcs-1 | 2024/06/04 19:35:37.831 16.328025817871094 +gcs-1 | 2024/06/04 19:35:42.832 9.955978393554688 +gcs-1 | 2024/06/04 19:35:47.833 12.561546325683594 +gcs-1 | 2024/06/04 19:35:52.836 15.2049560546875 +gcs-1 | 2024/06/04 19:35:57.838 17.817832946777344 +gcs-1 | 2024/06/04 19:36:02.840 11.227790832519531 +gcs-1 | 2024/06/04 19:36:07.841 13.8360595703125 +gcs-1 | 2024/06/04 19:36:12.842 16.432937622070312 +gcs-1 | 2024/06/04 19:36:17.843 19.054405212402344 +gcs-1 | 2024/06/04 19:36:22.844 12.429069519042969 +gcs-1 | 2024/06/04 19:36:27.846 15.116043090820312 +gcs-1 | 2024/06/04 19:36:32.848 26.552085876464844 +gcs-1 | 2024/06/04 19:36:37.849 29.160964965820312 +gcs-1 | 2024/06/04 19:36:42.850 31.53943634033203 +gcs-1 | 2024/06/04 19:36:47.851 34.08723449707031 +gcs-1 | 2024/06/04 19:36:52.852 36.65630340576172 +gcs-1 | 2024/06/04 19:36:57.853 39.16461181640625 +gcs-1 | 2024/06/04 19:37:02.855 41.622657775878906 +gcs-1 | 2024/06/04 19:37:07.856 25.137908935546875 +gcs-1 | 2024/06/04 19:37:12.857 27.570457458496094 +gcs-1 | 2024/06/04 19:37:17.858 30.110198974609375 +gcs-1 | 2024/06/04 19:37:22.860 32.5474853515625 diff --git a/analysis/output_10b_run9_flush_mysql.txt b/analysis/output_10b_run9_flush_mysql.txt new file mode 100644 index 0000000..d97db98 --- /dev/null +++ b/analysis/output_10b_run9_flush_mysql.txt @@ -0,0 +1,270 @@ +gcs-1 | 2024/06/04 19:42:41 server listening at [::]:50000 +gcs-1 | 2024/06/04 19:42:41.847 server listening at [::]:50001 +gcs-1 | 2024/06/04 19:42:41.851 0.541778564453125 +gcs-1 | 2024/06/04 19:42:42.849 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:43.848 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:44.850 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:45.849 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:46.848 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:42:46.851 0.7560653686523438 +gcs-1 | 2024/06/04 19:42:51.853 1.3027725219726562 +gcs-1 | 2024/06/04 19:42:56.853 1.3161163330078125 +gcs-1 | 2024/06/04 19:43:01.861 1.3270416259765625 +gcs-1 | 2024/06/04 19:43:06.863 1.3676834106445312 +gcs-1 | 2024/06/04 19:43:11.864 1.397064208984375 +gcs-1 | 2024/06/04 19:43:16.873 1.5728912353515625 +gcs-1 | 2024/06/04 19:43:21.874 1.5522232055664062 +gcs-1 | 2024/06/04 19:43:26.876 1.4987716674804688 +gcs-1 | 2024/06/04 19:43:31.879 1.5016555786132812 +gcs-1 | 2024/06/04 19:43:36.881 1.460968017578125 +gcs-1 | 2024/06/04 19:43:41.881 1.4944305419921875 +gcs-1 | 2024/06/04 19:43:46.883 1.909912109375 +gcs-1 | 2024/06/04 19:43:51.883 1.6782989501953125 +gcs-1 | 2024/06/04 19:43:56.885 1.6936721801757812 +gcs-1 | 2024/06/04 19:44:01.887 1.6722183227539062 +gcs-1 | 2024/06/04 19:44:06.890 1.7350997924804688 +gcs-1 | 2024/06/04 19:44:11.891 1.7082443237304688 +gcs-1 | 2024/06/04 19:44:16.893 1.725311279296875 +gcs-1 | 2024/06/04 19:44:20.160 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:44:21.895 1.7982254028320312 +gcs-1 | 2024/06/04 19:44:26.897 1.6686325073242188 +gcs-1 | 2024/06/04 19:44:31.900 1.7048492431640625 +gcs-1 | 2024/06/04 19:44:36.900 1.8752365112304688 +gcs-1 | 2024/06/04 19:44:41.902 2.2953033447265625 +gcs-1 | 2024/06/04 19:44:46.903 2.2530364990234375 +gcs-1 | 2024/06/04 19:44:51.903 1.9637451171875 +gcs-1 | 2024/06/04 19:44:56.905 1.8707351684570312 +gcs-1 | 2024/06/04 19:45:01.906 2.0524215698242188 +gcs-1 | 2024/06/04 19:45:06.907 2.0872573852539062 +gcs-1 | 2024/06/04 19:45:11.909 1.9512405395507812 +gcs-1 | 2024/06/04 19:45:16.910 2.0131683349609375 +gcs-1 | 2024/06/04 19:45:21.911 1.9726409912109375 +gcs-1 | 2024/06/04 19:45:26.912 1.9984664916992188 +gcs-1 | 2024/06/04 19:45:31.915 2.1372756958007812 +gcs-1 | 2024/06/04 19:45:36.917 2.053253173828125 +gcs-1 | 2024/06/04 19:45:41.919 2.1457443237304688 +gcs-1 | 2024/06/04 19:45:46.920 2.1833419799804688 +gcs-1 | 2024/06/04 19:45:51.923 2.18670654296875 +gcs-1 | 2024/06/04 19:45:56.924 2.1562881469726562 +gcs-1 | 2024/06/04 19:46:01.926 2.2276992797851562 +gcs-1 | 2024/06/04 19:46:06.929 2.1830291748046875 +gcs-1 | 2024/06/04 19:46:11.931 2.2578582763671875 +gcs-1 | 2024/06/04 19:46:16.932 2.3423690795898438 +gcs-1 | 2024/06/04 19:46:21.933 3.4385833740234375 +gcs-1 | 2024/06/04 19:46:26.934 3.4460372924804688 +gcs-1 | 2024/06/04 19:46:31.936 3.4427566528320312 +gcs-1 | 2024/06/04 19:46:36.937 3.405426025390625 +gcs-1 | 2024/06/04 19:46:41.939 2.6701431274414062 +gcs-1 | 2024/06/04 19:46:46.939 2.7786788940429688 +gcs-1 | 2024/06/04 19:46:51.942 2.7306900024414062 +gcs-1 | 2024/06/04 19:46:56.943 2.7115554809570312 +gcs-1 | 2024/06/04 19:47:01.946 2.7729339599609375 +gcs-1 | 2024/06/04 19:47:06.947 2.6181869506835938 +gcs-1 | 2024/06/04 19:47:11.948 2.6241455078125 +gcs-1 | 2024/06/04 19:47:16.950 2.7991256713867188 +gcs-1 | 2024/06/04 19:47:21.951 2.685302734375 +gcs-1 | 2024/06/04 19:47:26.953 2.6424407958984375 +gcs-1 | 2024/06/04 19:47:31.954 2.83978271484375 +gcs-1 | 2024/06/04 19:47:36.955 2.7719802856445312 +gcs-1 | 2024/06/04 19:47:41.957 2.825897216796875 +gcs-1 | 2024/06/04 19:47:46.958 2.7015762329101562 +gcs-1 | 2024/06/04 19:47:51.961 2.6591110229492188 +gcs-1 | 2024/06/04 19:47:56.964 2.637481689453125 +gcs-1 | 2024/06/04 19:48:01.966 2.5851974487304688 +gcs-1 | 2024/06/04 19:48:06.967 2.6280364990234375 +gcs-1 | 2024/06/04 19:48:11.967 2.8046035766601562 +gcs-1 | 2024/06/04 19:48:16.968 2.7517776489257812 +gcs-1 | 2024/06/04 19:48:21.969 2.7795562744140625 +gcs-1 | 2024/06/04 19:48:26.970 2.7211380004882812 +gcs-1 | 2024/06/04 19:48:31.973 2.813690185546875 +gcs-1 | 2024/06/04 19:48:36.974 2.6896209716796875 +gcs-1 | 2024/06/04 19:48:41.975 2.9103164672851562 +gcs-1 | 2024/06/04 19:48:46.977 2.8389129638671875 +gcs-1 | 2024/06/04 19:48:51.978 2.7414321899414062 +gcs-1 | 2024/06/04 19:48:56.979 2.7215194702148438 +gcs-1 | 2024/06/04 19:49:01.980 2.7321243286132812 +gcs-1 | 2024/06/04 19:49:06.981 2.8215179443359375 +gcs-1 | 2024/06/04 19:49:11.982 2.7779006958007812 +gcs-1 | 2024/06/04 19:49:16.984 2.6975250244140625 +gcs-1 | 2024/06/04 19:49:21.988 2.7195587158203125 +gcs-1 | 2024/06/04 19:49:26.991 2.7353515625 +gcs-1 | 2024/06/04 19:49:31.996 5.162193298339844 +gcs-1 | 2024/06/04 19:49:36.997 5.352867126464844 +gcs-1 | 2024/06/04 19:49:41.998 5.135749816894531 +gcs-1 | 2024/06/04 19:49:47.001 5.236991882324219 +gcs-1 | 2024/06/04 19:49:52.002 5.279327392578125 +gcs-1 | 2024/06/04 19:49:57.003 5.358299255371094 +gcs-1 | 2024/06/04 19:50:02.005 5.383903503417969 +gcs-1 | 2024/06/04 19:50:07.007 5.256683349609375 +gcs-1 | 2024/06/04 19:50:12.008 3.896575927734375 +gcs-1 | 2024/06/04 19:50:17.009 3.8473968505859375 +gcs-1 | 2024/06/04 19:50:22.010 3.86724853515625 +gcs-1 | 2024/06/04 19:50:27.011 3.9947586059570312 +gcs-1 | 2024/06/04 19:50:32.010 4.004676818847656 +gcs-1 | 2024/06/04 19:50:37.010 3.9845046997070312 +gcs-1 | 2024/06/04 19:50:42.013 3.6995315551757812 +gcs-1 | 2024/06/04 19:50:47.015 3.89434814453125 +gcs-1 | 2024/06/04 19:50:52.015 3.8594207763671875 +gcs-1 | 2024/06/04 19:50:57.016 3.8529205322265625 +gcs-1 | 2024/06/04 19:51:02.021 3.822540283203125 +gcs-1 | 2024/06/04 19:51:07.022 3.98785400390625 +gcs-1 | 2024/06/04 19:51:12.023 3.8061752319335938 +gcs-1 | 2024/06/04 19:51:17.025 3.90289306640625 +gcs-1 | 2024/06/04 19:51:22.026 3.8698196411132812 +gcs-1 | 2024/06/04 19:51:27.028 3.9017105102539062 +gcs-1 | 2024/06/04 19:51:32.029 3.9386215209960938 +gcs-1 | 2024/06/04 19:51:37.031 3.9354934692382812 +gcs-1 | 2024/06/04 19:51:42.032 3.8931045532226562 +gcs-1 | 2024/06/04 19:51:47.033 4.007530212402344 +gcs-1 | 2024/06/04 19:51:52.034 3.92266845703125 +gcs-1 | 2024/06/04 19:51:57.036 3.8879776000976562 +gcs-1 | 2024/06/04 19:52:02.038 3.94140625 +gcs-1 | 2024/06/04 19:52:07.039 3.96728515625 +gcs-1 | 2024/06/04 19:52:12.040 4.01123046875 +gcs-1 | 2024/06/04 19:52:17.042 4.004814147949219 +gcs-1 | 2024/06/04 19:52:22.043 3.931427001953125 +gcs-1 | 2024/06/04 19:52:27.044 3.9497604370117188 +gcs-1 | 2024/06/04 19:52:32.046 4.100624084472656 +gcs-1 | 2024/06/04 19:52:37.049 3.8479232788085938 +gcs-1 | 2024/06/04 19:52:42.051 3.8843002319335938 +gcs-1 | 2024/06/04 19:52:47.054 3.8787612915039062 +gcs-1 | 2024/06/04 19:52:49.385 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:52:52.055 3.9020767211914062 +gcs-1 | 2024/06/04 19:52:53.949 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:52:57.057 3.3466644287109375 +gcs-1 | 2024/06/04 19:53:02.106 3.7256851196289062 +gcs-1 | 2024/06/04 19:53:07.109 3.9135284423828125 +gcs-1 | 2024/06/04 19:53:12.109 3.94085693359375 +gcs-1 | 2024/06/04 19:53:17.112 3.9469451904296875 +gcs-1 | 2024/06/04 19:53:22.113 4.1459503173828125 +gcs-1 | 2024/06/04 19:53:27.114 4.072853088378906 +gcs-1 | 2024/06/04 19:53:32.115 3.9909515380859375 +gcs-1 | 2024/06/04 19:53:37.116 3.9355239868164062 +gcs-1 | 2024/06/04 19:53:42.117 3.9438705444335938 +gcs-1 | 2024/06/04 19:53:47.119 4.1521148681640625 +gcs-1 | 2024/06/04 19:53:52.120 4.1165771484375 +gcs-1 | 2024/06/04 19:53:57.122 3.9645462036132812 +gcs-1 | 2024/06/04 19:54:02.123 4.008697509765625 +gcs-1 | 2024/06/04 19:54:07.126 3.9752349853515625 +gcs-1 | 2024/06/04 19:54:12.127 4.1572265625 +gcs-1 | 2024/06/04 19:54:14.593 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:54:17.135 3.5205612182617188 +gcs-1 | 2024/06/04 19:54:19.404 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:54:22.136 4.158515930175781 +gcs-1 | 2024/06/04 19:54:27.137 4.0652313232421875 +gcs-1 | 2024/06/04 19:54:31.919 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:54:32.127 3.5928192138671875 +gcs-1 | 2024/06/04 19:54:37.133 3.4572525024414062 +gcs-1 | 2024/06/04 19:54:42.134 4.091117858886719 +gcs-1 | 2024/06/04 19:54:47.136 4.213432312011719 +gcs-1 | 2024/06/04 19:54:52.136 4.0318450927734375 +gcs-1 | 2024/06/04 19:54:57.138 4.234596252441406 +gcs-1 | 2024/06/04 19:55:02.159 4.1728057861328125 +gcs-1 | 2024/06/04 19:55:07.161 4.115531921386719 +gcs-1 | 2024/06/04 19:55:12.161 4.257545471191406 +gcs-1 | 2024/06/04 19:55:17.164 4.3677978515625 +gcs-1 | 2024/06/04 19:55:22.164 4.366447448730469 +gcs-1 | 2024/06/04 19:55:27.166 4.489662170410156 +gcs-1 | 2024/06/04 19:55:32.168 4.238433837890625 +gcs-1 | 2024/06/04 19:55:37.170 4.231101989746094 +gcs-1 | 2024/06/04 19:55:42.171 4.193916320800781 +gcs-1 | 2024/06/04 19:55:47.172 4.2407073974609375 +gcs-1 | 2024/06/04 19:55:52.175 4.24365234375 +gcs-1 | 2024/06/04 19:55:57.178 4.167854309082031 +gcs-1 | 2024/06/04 19:56:02.179 9.159080505371094 +gcs-1 | 2024/06/04 19:56:07.180 9.027717590332031 +gcs-1 | 2024/06/04 19:56:11.936 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:56:12.181 8.815505981445312 +gcs-1 | 2024/06/04 19:56:13.885 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:56:17.183 9.644462585449219 +gcs-1 | 2024/06/04 19:56:22.184 9.179283142089844 +gcs-1 | 2024/06/04 19:56:27.185 9.074821472167969 +gcs-1 | 2024/06/04 19:56:32.193 9.318168640136719 +gcs-1 | 2024/06/04 19:56:37.194 9.010856628417969 +gcs-1 | 2024/06/04 19:56:42.195 9.003768920898438 +gcs-1 | 2024/06/04 19:56:47.197 9.423927307128906 +gcs-1 | 2024/06/04 19:56:52.199 9.155105590820312 +gcs-1 | 2024/06/04 19:56:57.200 9.069099426269531 +gcs-1 | 2024/06/04 19:57:02.201 9.386749267578125 +gcs-1 | 2024/06/04 19:57:07.203 9.30413818359375 +gcs-1 | 2024/06/04 19:57:12.206 9.159759521484375 +gcs-1 | 2024/06/04 19:57:17.207 6.3763427734375 +gcs-1 | 2024/06/04 19:57:22.208 6.4787445068359375 +gcs-1 | 2024/06/04 19:57:27.208 6.401405334472656 +gcs-1 | 2024/06/04 19:57:32.219 5.834190368652344 +gcs-1 | 2024/06/04 19:57:37.221 6.562278747558594 +gcs-1 | 2024/06/04 19:57:42.223 6.282386779785156 +gcs-1 | 2024/06/04 19:57:47.223 6.442680358886719 +gcs-1 | 2024/06/04 19:57:52.226 6.404937744140625 +gcs-1 | 2024/06/04 19:57:57.226 6.2392578125 +gcs-1 | 2024/06/04 19:58:02.228 6.125709533691406 +gcs-1 | 2024/06/04 19:58:07.253 6.2435150146484375 +gcs-1 | 2024/06/04 19:58:09.190 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:58:12.255 6.359710693359375 +gcs-1 | 2024/06/04 19:58:17.258 6.2640838623046875 +gcs-1 | 2024/06/04 19:58:22.259 6.612174987792969 +gcs-1 | 2024/06/04 19:58:27.261 6.179771423339844 +gcs-1 | 2024/06/04 19:58:32.263 6.527679443359375 +gcs-1 | 2024/06/04 19:58:37.266 6.354972839355469 +gcs-1 | 2024/06/04 19:58:42.267 6.275825500488281 +gcs-1 | 2024/06/04 19:58:47.268 6.4571990966796875 +gcs-1 | 2024/06/04 19:58:52.273 6.233741760253906 +gcs-1 | 2024/06/04 19:58:54.893 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:58:57.278 6.1600494384765625 +gcs-1 | 2024/06/04 19:59:02.277 6.147499084472656 +gcs-1 | 2024/06/04 19:59:07.279 6.4554595947265625 +gcs-1 | 2024/06/04 19:59:12.280 6.414337158203125 +gcs-1 | 2024/06/04 19:59:17.281 6.2513885498046875 +gcs-1 | 2024/06/04 19:59:22.282 6.390953063964844 +gcs-1 | 2024/06/04 19:59:27.284 6.281791687011719 +gcs-1 | 2024/06/04 19:59:32.289 6.332313537597656 +gcs-1 | 2024/06/04 19:59:37.375 6.100608825683594 +gcs-1 | 2024/06/04 19:59:37.945 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:59:39.899 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 19:59:42.376 5.796142578125 +gcs-1 | 2024/06/04 19:59:47.377 6.586235046386719 +gcs-1 | 2024/06/04 19:59:52.378 6.6221466064453125 +gcs-1 | 2024/06/04 19:59:57.389 6.4645538330078125 +gcs-1 | 2024/06/04 20:00:02.391 6.424186706542969 +gcs-1 | 2024/06/04 20:00:07.392 6.462791442871094 +gcs-1 | 2024/06/04 20:00:09.892 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:12.394 6.034141540527344 +gcs-1 | 2024/06/04 20:00:17.395 6.5565948486328125 +gcs-1 | 2024/06/04 20:00:22.396 6.491447448730469 +gcs-1 | 2024/06/04 20:00:27.397 6.366546630859375 +gcs-1 | 2024/06/04 20:00:32.401 5.9578704833984375 +gcs-1 | 2024/06/04 20:00:35.903 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:37.404 6.703330993652344 +gcs-1 | 2024/06/04 20:00:42.407 5.6876983642578125 +gcs-1 | 2024/06/04 20:00:47.407 6.803955078125 +gcs-1 | 2024/06/04 20:00:52.038 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:52.409 6.2969512939453125 +gcs-1 | 2024/06/04 20:00:53.894 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:56.003 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:00:57.410 7.14398193359375 +gcs-1 | 2024/06/04 20:00:57.945 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:01:02.412 6.350189208984375 +gcs-1 | 2024/06/04 20:01:07.413 5.6882781982421875 +gcs-1 | 2024/06/04 20:01:12.427 5.788665771484375 +gcs-1 | 2024/06/04 20:01:17.429 5.781158447265625 +gcs-1 | 2024/06/04 20:01:22.431 5.6903533935546875 +gcs-1 | 2024/06/04 20:01:27.432 6.453605651855469 +gcs-1 | 2024/06/04 20:01:32.437 6.3587493896484375 +gcs-1 | 2024/06/04 20:01:37.438 5.68914794921875 +gcs-1 | 2024/06/04 20:01:42.439 6.609962463378906 +gcs-1 | 2024/06/04 20:01:47.440 5.8092193603515625 +gcs-1 | 2024/06/04 20:01:52.441 6.6129302978515625 +gcs-1 | 2024/06/04 20:01:57.441 5.7250823974609375 +gcs-1 | 2024/06/04 20:02:02.442 5.694068908691406 +gcs-1 | 2024/06/04 20:02:07.442 6.5864715576171875 +gcs-1 | 2024/06/04 20:02:12.443 5.6991119384765625 +gcs-1 | 2024/06/04 20:02:17.444 6.4114227294921875 +gcs-1 | 2024/06/04 20:02:22.445 6.061088562011719 +gcs-1 | 2024/06/04 20:02:27.448 6.7945404052734375 +gcs-1 | 2024/06/04 20:02:32.451 5.9176483154296875 +gcs-1 | 2024/06/04 20:02:37.453 6.7691192626953125 +gcs-1 | 2024/06/04 20:02:42.113 Error flushing to disk: incomplete input +gcs-1 | 2024/06/04 20:02:42.456 6.22845458984375 +gcs-1 | 2024/06/04 20:02:47.457 6.676582336425781 +gcs-1 | 2024/06/04 20:02:52.459 6.405731201171875 +gcs-1 | 2024/06/04 20:02:57.460 5.7739105224609375 diff --git a/analysis/parse_10b.py b/analysis/parse_10b.py new file mode 100644 index 0000000..6c60916 --- /dev/null +++ b/analysis/parse_10b.py @@ -0,0 +1,54 @@ +import re +import csv +from datetime import datetime + +def parse_log_line(line): + pattern = r'^worker\d+\-\d+\s+\|\s+(\d{4}\/\d{2}\/\d{2}\s+\d{2}:\d{2}:\d{2}\.\d{3})\s+FIG10b:\s+Memory\s+usage\s+without\s+flush\s+after\s+(\d+)\s+tasks:\s+(\d+\.\d+)\s+MB' + match = re.match(pattern, line) + if match: + timestamp = datetime.strptime(match.group(1), '%Y/%m/%d %H:%M:%S.%f') + tasks = int(match.group(2)) + memory_usage = float(match.group(3)) + return (timestamp, memory_usage) + else: + return None + +def parse_logs(log_data): + result = [] + for line in log_data.split('\n'): + parsed = parse_log_line(line) + if parsed: + result.append(parsed) + return result + +def write_to_csv(data, filename): + with open(filename, 'w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(['Timestamp', 'Memory Usage (MB)']) + for timestamp, memory_usage in data: + writer.writerow([timestamp, memory_usage]) + +# Unit test +def test_parse_log_line(): + test_line = 'worker1-1 | 2024/06/04 05:02:21.134 FIG10b: Memory usage without flush after 1459 tasks: 1060.06640625 MB' + expected_output = (datetime(2024, 6, 4, 5, 2, 21, 134000), 1060.06640625) + assert parse_log_line(test_line) == expected_output + +# Example usage +if __name__ == '__main__': + test_parse_log_line() # Run the unit test + print("unit test worked") + + with open('output_10b_run2_300s_flush_2b_memory_limit.txt', 'r') as f: + log_data = f.read() + + parsed_logs = parse_logs(log_data) + write_to_csv(parsed_logs, 'output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv') + + # ================================= + + with open('output_10b_run3_40s_flush_2gb_memory_limit.txt', 'r') as f: + log_data = f.read() + + parsed_logs = parse_logs(log_data) + write_to_csv(parsed_logs, 'output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv') \ No newline at end of file diff --git a/analysis/plot_10b.py b/analysis/plot_10b.py new file mode 100644 index 0000000..614f1f0 --- /dev/null +++ b/analysis/plot_10b.py @@ -0,0 +1,114 @@ +import matplotlib.pyplot as plt +import csv +from datetime import datetime + +# Read data from the CSV file +x1 = [] # Elapsed time (seconds) +y1 = [] # GCS Used Memory (MB) + +with open('output_10b_run1_cleaned.csv', 'r') as csvfile: + plots = csv.reader(csvfile) + header = next(plots) # Skip the header row + timestamp_baseline = None + memory_baseline = None + + for row in plots: + timestamp_str, memory = row + timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + + if timestamp_baseline is None: + timestamp_baseline = timestamp + memory_baseline = float(memory) + + elapsed_time = (timestamp - timestamp_baseline).total_seconds() + memory_used = float(memory) - memory_baseline + + x1.append(elapsed_time) + y1.append(memory_used) + +# ===== +# Read data from the CSV file +x2 = [] # Elapsed time (seconds) +y2 = [] # GCS Used Memory (MB) + +with open('output_10b_run3_40s_flush_2gb_memory_limit_cleaned.csv', 'r') as csvfile: + plots = csv.reader(csvfile) + header = next(plots) # Skip the header row + timestamp_baseline = None + memory_baseline = None + + for row in plots: + timestamp_str, memory = row + timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + + if timestamp_baseline is None: + timestamp_baseline = timestamp + memory_baseline = float(memory) + + elapsed_time = (timestamp - timestamp_baseline).total_seconds() + memory_used = float(memory) - memory_baseline + + x2.append(elapsed_time) + y2.append(memory_used) + +# ===== + +# Read data from the CSV file +x3 = [] # Elapsed time (seconds) +y3 = [] # GCS Used Memory (MB) + +with open('output_10b_run2_300s_flush_2b_memory_limit_cleaned.csv', 'r') as csvfile: + plots = csv.reader(csvfile) + header = next(plots) # Skip the header row + timestamp_baseline = None + memory_baseline = None + + for row in plots: + timestamp_str, memory = row + timestamp = datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S.%f') + + if timestamp_baseline is None: + timestamp_baseline = timestamp + memory_baseline = float(memory) + + elapsed_time = (timestamp - timestamp_baseline).total_seconds() + memory_used = float(memory) - memory_baseline + + x3.append(elapsed_time) + y3.append(memory_used) + +# ===== + +def attempt1(): + # Create the plot + plt.plot(x1, y1, label="Baby Ray, no GCS flush", linestyle='--', color='blue') + plt.plot(x2, y2, label="Baby Ray, GCS flush per 40s", color='orange') + + # Customize the plot + plt.xlabel("Elapsed time (seconds)") + plt.ylabel("GCS Used Memory (MB)") + plt.title("Memory Usage Over Time") + plt.legend(loc="lower right") + + # Show the plot + plt.grid(True) + plt.show() + +def attempt2(): + # Create the plot + plt.plot(x1, y1, label="Baby Ray, no GCS flush", linestyle='--', color='blue') + plt.plot(x3, y3, label="Baby Ray, GCS flush per 300s", color='orange') + + # Customize the plot + plt.xlabel("Elapsed time (seconds)") + plt.ylabel("GCS Used Memory (MB)") + plt.title("50 million no-op tasks") + plt.legend(loc="lower right") + + # Show the plot + # Save the plot as a high-quality PNG + plt.savefig("fig10b_graphs/attempt2_sharp.png", dpi=300) + plt.show() + +if __name__ == '__main__': + attempt2() \ No newline at end of file diff --git a/config/app_config.yaml b/config/app_config.yaml index 04892f4..c196ce4 100644 --- a/config/app_config.yaml +++ b/config/app_config.yaml @@ -15,5 +15,5 @@ ports: dns: node_prefix: "node" gcs: - flush_interval_sec: -1 # put -1 to never flush - enable_chain_replication: true \ No newline at end of file + flush_interval_sec: 1 # put -1 to never flush + enable_chain_replication: true diff --git a/docker-compose.fig11.yml b/docker-compose.fig11.yml new file mode 100644 index 0000000..45e7c63 --- /dev/null +++ b/docker-compose.fig11.yml @@ -0,0 +1,12 @@ +version: '3' + +services: + worker1: + # -u for the python script to flush output immediately + command: ["/bin/sh", "-c", "python3 -u -m pythonserver.worker & ./go/bin/localobjstore & ./go/bin/localscheduler & sleep 5 && python3 -u scripts/fig11.py & sleep infinity"] + depends_on: + # - zookeeper + - worker2 + - worker3 + - gcs + - global_scheduler diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 4121f3f..a5ae596 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -2,7 +2,7 @@ version: '3' services: worker1: - command: ["/bin/sh", "-c", "./go/bin/worker & ./go/bin/localobjstore & ./go/bin/localscheduler & sleep 2 && pytest -v /app/tests/integration"] + command: ["/bin/sh", "-c", "python3 -u -m pythonserver.worker & ./go/bin/localobjstore & ./go/bin/localscheduler & sleep 2 && pytest -v /app/tests/integration"] #command: ["/bin/sh", "-c", "./go/bin/localscheduler & ./go/bin/localobjstore & ./go/bin/worker & sleep 5 && pytest -v /app/tests/integration"] depends_on: - gcs diff --git a/go/cmd/gcsfunctable/main.go b/go/cmd/gcsfunctable/main.go index d6739cb..2dea283 100644 --- a/go/cmd/gcsfunctable/main.go +++ b/go/cmd/gcsfunctable/main.go @@ -29,7 +29,7 @@ func main() { _ = lis s := grpc.NewServer(util.GetServerOptions()...) pb.RegisterGCSFuncServer(s, NewGCSFuncServer()) - log.Printf("server listening at %v", lis.Addr()) + // log.Printf("server listening at %v", lis.Addr()) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } diff --git a/go/cmd/gcsobjtable/db.go b/go/cmd/gcsobjtable/db.go index 57e7b05..540a211 100644 --- a/go/cmd/gcsobjtable/db.go +++ b/go/cmd/gcsobjtable/db.go @@ -5,7 +5,11 @@ import ( "strings" ) -// TODO: Address the fact that SQLite does not hold uint64 that exceeds positive int64 +func setCacheSizeToZero(db *sql.DB) error { + _, err := db.Exec("PRAGMA cache_size = 0;") + return err +} + func createObjectLocationsTable(db *sql.DB) error { createTableSQL := `CREATE TABLE IF NOT EXISTS object_locations ( "object_uid" INTEGER NOT NULL, diff --git a/go/cmd/gcsobjtable/db_test.go b/go/cmd/gcsobjtable/db_test.go index f18d897..97fef9c 100644 --- a/go/cmd/gcsobjtable/db_test.go +++ b/go/cmd/gcsobjtable/db_test.go @@ -7,6 +7,29 @@ import ( _ "github.com/mattn/go-sqlite3" ) +func TestSetCacheSizeToZero(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("Failed to open database: %v", err) + } + defer db.Close() + + err = setCacheSizeToZero(db) + if err != nil { + t.Errorf("Failed to set cache size to zero: %v", err) + } + + var cacheSize int + err = db.QueryRow("PRAGMA cache_size;").Scan(&cacheSize) + if err != nil { + t.Fatalf("Failed to query cache size: %v", err) + } + + if cacheSize != 0 { + t.Errorf("Cache size is not zero, got: %d", cacheSize) + } +} + func TestCreateObjectLocationsTable(t *testing.T) { // Use an in-memory SQLite database for testing database, err := sql.Open("sqlite3", ":memory:") diff --git a/go/cmd/gcsobjtable/main.go b/go/cmd/gcsobjtable/main.go index 0e23584..c6b9ceb 100644 --- a/go/cmd/gcsobjtable/main.go +++ b/go/cmd/gcsobjtable/main.go @@ -1,7 +1,9 @@ package main import ( + "bufio" "context" + "os" "time" // "errors" @@ -9,6 +11,7 @@ import ( "log" "math/rand" "net" + "runtime" "strconv" "sync" @@ -39,7 +42,7 @@ func LocalLog(format string, v ...interface{}) { } else { logMessage = fmt.Sprintf(format, v...) } - log.Printf("[lobs] %s", logMessage) + log.Printf("[gcsobjtable] %s", logMessage) } func main() { @@ -55,7 +58,7 @@ func main() { _ = lis s := grpc.NewServer(util.GetServerOptions()...) pb.RegisterGCSObjServer(s, NewGCSObjServer(cfg.GCS.FlushIntervalSec)) - log.Printf("server listening at %v", lis.Addr()) + // log.Printf("server listening at %v", lis.Addr()) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } @@ -64,12 +67,24 @@ func main() { // server is used to implement your gRPC service. type GCSObjServer struct { pb.UnimplementedGCSObjServer - objectLocations map[uint64][]uint64 // object uid -> list of nodeIds as uint64 - waitlist map[uint64][]string // object uid -> list of IP addresses as string - mu sync.Mutex // lock should be used for both objectLocations and waitlist - database *sql.DB // connection to SQLite persistent datastore - ticker *time.Ticker // for GCS flushing - objectSizes map[uint64]uint64 + objectLocations map[uint64][]uint64 // object uid -> list of nodeIds as uint64 + waitlist map[uint64][]string // object uid -> list of IP addresses as string + mu sync.Mutex // lock should be used for both objectLocations and waitlist + database *sql.DB // connection to SQLite persistent datastore + ticker *time.Ticker // for GCS flushing + lineageMu sync.RWMutex // lock should be used for s.lineage + objectSizes map[uint64]uint64 + lineage map[uint64]*pb.GlobalScheduleRequest + globalSchedulerClient SchedulerClient + liveNodes map[uint64]bool + generating map[uint64]uint64 //object uid -> node id of original generator + //used to determine when an original creation of a uid should be restarted + globalCtx context.Context +} + +type SchedulerClient interface { + Schedule(ctx context.Context, req *pb.GlobalScheduleRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error) + Heartbeat(ctx context.Context, req *pb.HeartbeatRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error) } /* set flushIntervalSec to -1 to disable GCS flushing */ @@ -80,17 +95,28 @@ func NewGCSObjServer(flushIntervalSec int) *GCSObjServer { if err != nil { log.Fatal(err) } + setCacheSizeToZero(database) // Creates table if it doesn't already exist createObjectLocationsTable(database) /* Create server object */ + + globalSchedulerAddress := fmt.Sprintf("%s%d:%d", cfg.DNS.NodePrefix, cfg.NodeIDs.GlobalScheduler, cfg.Ports.GlobalScheduler) + conn, _ := grpc.Dial(globalSchedulerAddress, grpc.WithInsecure()) + globalSchedulerClient := pb.NewGlobalSchedulerClient(conn) + server := &GCSObjServer{ - objectLocations: make(map[uint64][]uint64), - waitlist: make(map[uint64][]string), - mu: sync.Mutex{}, - database: database, - ticker: nil, - objectSizes: make(map[uint64]uint64), + objectLocations: make(map[uint64][]uint64), + waitlist: make(map[uint64][]string), + mu: sync.Mutex{}, + database: database, + ticker: nil, + objectSizes: make(map[uint64]uint64), + lineage: make(map[uint64]*pb.GlobalScheduleRequest), + globalSchedulerClient: globalSchedulerClient, + liveNodes: make(map[uint64]bool), + generating: make(map[uint64]uint64), + globalCtx: context.Background(), } if flushIntervalSec != -1 { // Launch periodic disk flushing @@ -102,30 +128,111 @@ func NewGCSObjServer(flushIntervalSec int) *GCSObjServer { if err != nil { log.Printf("Error flushing to disk: %v", err) } else { - log.Printf("Successfully flushed to disk!") + //log.Printf("Successfully flushed to disk!") } } }() } + + logMemoryUsage := true // TODO: REMOVE HARDCODE + if logMemoryUsage { + go func() { + for { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + // HeapAlloc: Bytes of allocated heap objects (heap memory in use). in MB + log.Printf("%v\n", bToMb(memStats.HeapAlloc)) + + // Sleep for 5 seconds before the next iteration + time.Sleep(5 * time.Second) + } + }() + } + return server } +func bToMb(b uint64) float64 { + return float64(b) / 1024 / 1024 +} + func (s *GCSObjServer) flushToDisk() error { s.mu.Lock() defer s.mu.Unlock() + garbage_collect := true // TODO: REMOVE HARDCODED + flush_to_AOF := false // TODO: REMOVE HARDCODED + aof_filename := "./aof.txt" // TODO: REMOVE HARDCODED - err := insertOrUpdateObjectLocations(s.database, s.objectLocations) - if err != nil { - return err + // Can either flush to append-only file or to DB + if flush_to_AOF { + err := WriteObjectLocationsToAOF(aof_filename, s.objectLocations) + if err != nil { + return err + } + //log.Printf("Successfully flushed to AOF!") + } else { + // Flush to SQLite3 Disk Database + err := insertOrUpdateObjectLocations(s.database, s.objectLocations) + if err != nil { + return err + } } + // Completely delete the current map in memory and start blank s.objectLocations = make(map[uint64][]uint64) // orphaning the old map will get it garbage collected + // Manually trigger garbage collection if desired + if garbage_collect { + runtime.GC() + //log.Println("Garbage collection triggered") + } + return nil +} + +// WriteObjectLocationsToAOF appends the contents of the objectLocations map to a file +func WriteObjectLocationsToAOF(filename string, objectLocations map[uint64][]uint64) error { + // Open file in append mode, create if it doesn't exist + file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("error opening file: %w", err) + } + defer file.Close() + + // Create a buffered writer + writer := bufio.NewWriter(file) + + // Iterate over the map and write to file + for key, values := range objectLocations { + // Convert key to string + keyStr := strconv.FormatUint(key, 10) + + // Convert values slice to a comma-separated string + var valuesStr string + for i, val := range values { + if i > 0 { + valuesStr += "," + } + valuesStr += strconv.FormatUint(val, 10) + } + + // Write key and values to file + _, err := writer.WriteString(fmt.Sprintf("%s: [%s]\n", keyStr, valuesStr)) + if err != nil { + return fmt.Errorf("error writing to file: %w", err) + } + } + + // Flush the buffered writer + if err := writer.Flush(); err != nil { + return fmt.Errorf("error flushing buffer: %w", err) + } + return nil } /* -Returns a nodeId that has object uid. If it doesn't exist anywhere, -then the second return value will be false. +Returns a nodeId that has object uid. If it has never been added to objectLocations, +then return false. Otherwise, return True. Assumes that s's mutex is locked. The return values are the location, boolean if it exists, and error */ @@ -147,9 +254,20 @@ func (s *GCSObjServer) getNodeId(uid uint64) (*uint64, bool, error) { // Proceed below } + nodesToReturn := make([]uint64, 1, 1) + for _, n := range nodeIds { + if !s.liveNodes[n] { + nodesToReturn = append(nodesToReturn, n) + } + } + + if len(nodesToReturn) == 0 { + return nil, false, nil + } + // Note: policy is to pick a random one; in the future it will need to be locality-based - randomIndex := rand.Intn(len(nodeIds)) - nodeId := &nodeIds[randomIndex] + randomIndex := rand.Intn(len(nodesToReturn)) + nodeId := &nodesToReturn[randomIndex] return nodeId, true, nil } @@ -162,7 +280,7 @@ func (s *GCSObjServer) sendCallback(clientAddress string, uid uint64, nodeId uin conn, err := grpc.Dial(clientAddress, util.GetDialOptions()...) if err != nil { // Log the error instead of returning it - log.Printf("Failed to connect back to client at %s: %v", clientAddress, err) + // log.Printf("Failed to connect back to client at %s: %v", clientAddress, err) return } defer conn.Close() // TODO: remove in some eventual universe @@ -175,7 +293,7 @@ func (s *GCSObjServer) sendCallback(clientAddress string, uid uint64, nodeId uin // Call LocationFound and handle any potential error _, err = localObjStoreClient.LocationFound(ctx, &pb.LocationFoundCallback{Uid: uid, Location: nodeId}) if err != nil { - log.Printf("Failed to send LocationFound callback for UID %d to client at %s: %v", uid, clientAddress, err) + // log.Printf("Failed to send LocationFound callback for UID %d to client at %s: %v", uid, clientAddress, err) } } @@ -183,10 +301,8 @@ func (s *GCSObjServer) NotifyOwns(ctx context.Context, req *pb.NotifyOwnsRequest s.mu.Lock() defer s.mu.Unlock() - log.Printf("WAS JUST NOTIFYOWNS()ED") - uid, nodeId := req.Uid, req.NodeId - + delete(s.generating, uid) // Append the nodeId to the list for the given object uid if _, exists := s.objectLocations[uid]; !exists { s.objectLocations[uid] = []uint64{} // Initialize slice if it doesn't exist @@ -234,7 +350,7 @@ func (s *GCSObjServer) RequestLocation(ctx context.Context, req *pb.RequestLocat if !exists { // Add client to waiting list - if _, exists := s.waitlist[uid]; !exists { + if _, waiting := s.waitlist[uid]; !waiting { s.waitlist[uid] = []string{} // Initialize slice if it doesn't exist } s.waitlist[uid] = append(s.waitlist[uid], clientAddress) @@ -256,11 +372,6 @@ func (s *GCSObjServer) RequestLocation(ctx context.Context, req *pb.RequestLocat func (s *GCSObjServer) GetObjectLocations(ctx context.Context, req *pb.ObjectLocationsRequest) (*pb.ObjectLocationsResponse, error) { locations := make(map[uint64]*pb.LocationByteTuple) - // log.Printf("DEEP PRINT!") - // log.Printf("length = %v", len(s.objectLocations)) - // for k, v := range s.objectLocations { - // log.Printf("s.objectLocations[%v] = %v", k, v) - // } for _, u := range req.Args { if _, ok := s.objectLocations[uint64(u)]; ok { @@ -269,3 +380,93 @@ func (s *GCSObjServer) GetObjectLocations(ctx context.Context, req *pb.ObjectLoc } return &pb.ObjectLocationsResponse{Locations: locations}, nil } + +func (s *GCSObjServer) RegisterLineage(ctx context.Context, req *pb.GlobalScheduleRequest) (*pb.StatusResponse, error) { + s.lineageMu.RLock() + if _, ok := s.lineage[req.Uid]; ok { + s.lineageMu.RUnlock() + return &pb.StatusResponse{Success: false}, status.Error(codes.Internal, "tried to register duplicate lineage") + } + s.lineageMu.RUnlock() + s.lineageMu.Lock() + s.lineage[req.Uid] = req + s.lineageMu.Unlock() + return &pb.StatusResponse{Success: false}, nil +} + +func (s *GCSObjServer) getNodes(uid uint64) []uint64 { + var nodes []uint64 + if node, ok := s.generating[uid]; ok { + nodes = append(nodes, node) + } + + if objectNodes, ok := s.objectLocations[uid]; ok { + for _, node := range objectNodes { + nodes = append(nodes, node) + } + } + + return nodes +} + +func (s *GCSObjServer) allDead(nodes []uint64) bool { + for _, node := range nodes { + if s.liveNodes[node] { + return false + } + } + return true +} + +func (s *GCSObjServer) RegisterLiveNodes(ctx context.Context, req *pb.LiveNodesRequest) (*pb.StatusResponse, error) { + + // LocalLog("got RegisterLiveNodes() call.") + + s.liveNodes = req.LiveNodes + + // if literally everything is dead: generating AND object locations + // then reschedule. but if there is at least one live node that is either + // gonna generate it or storing it then don't + + var regenerateList []uint64 + allUids := make(map[uint64]bool) + + for uid, _ := range s.generating { + allUids[uid] = true + } + for uid, _ := range s.objectLocations { + allUids[uid] = true + } + + for uid, _ := range allUids { + nodes := s.getNodes(uid) + + // LocalLog("the nodes gotten here is: %v", nodes) + + if s.allDead(nodes) { + regenerateList = append(regenerateList, uid) + } + } + + for _, uid := range regenerateList { + // LocalLog("uid = %v, rescheduling", uid) + go func() { + _, err := s.globalSchedulerClient.Schedule(s.globalCtx, s.lineage[uid]) + if err != nil { + // LocalLog("cannot contact global scheduler, err = %v", err) + } else { + // LocalLog("Just ran it on global!") + } + }() + } + return &pb.StatusResponse{Success: true}, nil +} + +func (s *GCSObjServer) RegisterGenerating(ctx context.Context, req *pb.GeneratingRequest) (*pb.StatusResponse, error) { + // LocalLog("trying to register node %v as a generating node", req.NodeId) + // if id, ok := s.generating[req.Uid]; ok { + // return &pb.StatusResponse{Success: false}, status.Error(codes.Internal, fmt.Sprintf("node %d is already generating uid %d", id, req.Uid)) + // } + s.generating[req.Uid] = req.NodeId + return &pb.StatusResponse{Success: true}, nil +} diff --git a/go/cmd/gcsobjtable/main_test.go b/go/cmd/gcsobjtable/main_test.go index 5be32b2..8bcc17a 100644 --- a/go/cmd/gcsobjtable/main_test.go +++ b/go/cmd/gcsobjtable/main_test.go @@ -6,10 +6,13 @@ import ( "database/sql" "errors" "fmt" + "io/ioutil" "log" "math/rand" "net" + "os" "strconv" + "strings" "sync" "testing" "time" @@ -42,6 +45,48 @@ func bufDialer(context.Context, string) (net.Conn, error) { return lis.Dial() } +func TestWriteObjectLocationsToAOF(t *testing.T) { + // Create a temporary file + tmpfile, err := ioutil.TempFile("", "test_object_locations_*.txt") + if err != nil { + t.Fatalf("Unable to create temp file: %v", err) + } + defer os.Remove(tmpfile.Name()) // Clean up the temp file after the test + + // Close the file so WriteObjectLocations can open it + tmpfile.Close() + + // Sample map for testing + objectLocations := map[uint64][]uint64{ + 1: {10, 20, 30}, + 2: {40, 50, 60}, + 3: {70, 80, 90}, + } + + // Call the function to write the map to the file + err = WriteObjectLocationsToAOF(tmpfile.Name(), objectLocations) + if err != nil { + t.Fatalf("WriteObjectLocations failed: %v", err) + } + + // Read the file content + content, err := ioutil.ReadFile(tmpfile.Name()) + if err != nil { + t.Fatalf("Unable to read temp file: %v", err) + } + + // Expected content + expectedContent := `1: [10,20,30] +2: [40,50,60] +3: [70,80,90] +` + + // Check if the content matches the expected content + if strings.TrimSpace(string(content)) != strings.TrimSpace(expectedContent) { + t.Errorf("Content mismatch\nExpected:\n%s\nGot:\n%s", expectedContent, string(content)) + } +} + func TestGetNodeId(t *testing.T) { // Seed the random number generator for reproducibility in tests rand.Seed(1) @@ -397,7 +442,7 @@ func TestSendCallback_Hit(t *testing.T) { s := grpc.NewServer() mock := NewMockLocalObjStoreServer(1) pb.RegisterLocalObjStoreServer(s, mock) - log.Printf("server listening at %v", lis.Addr()) + // log.Printf("server listening at %v", lis.Addr()) // Run the server in a goroutine go func() { @@ -550,6 +595,64 @@ func TestRequestLocation(t *testing.T) { if resp.ImmediatelyFound { t.Errorf("Expected location not to be found immediately, but it was found") } +} + +type mockSchedulerClient struct { + pb.GlobalSchedulerClient + requestsReceived map[uint64]*pb.GlobalScheduleRequest +} + +func (m *mockSchedulerClient) Schedule(ctx context.Context , req *pb.GlobalScheduleRequest, opts ...grpc.CallOption ) (*pb.StatusResponse, error) { + m.requestsReceived[req.Uid] = req + return nil, nil +} +func (m *mockSchedulerClient) Heartbeat(ctx context.Context, req *pb.HeartbeatRequest, opts ...grpc.CallOption ) (*pb.StatusResponse, error) { + return nil, nil +} +func (m *mockSchedulerClient) LiveNodesHeartbeat(ctx context.Context) (error) { + return nil +} +func (m *mockSchedulerClient) SendLiveNodes(ctx context.Context) { + return nil +} +func TestNodeDiesWhileGenerating(t *testing.T) { + /* + -Worker tells GCS that its generating + -Global Scheduler tells it that that node is dead + -another node asks for object + -global scheduler should receive a schedule request for that object + */ + ctx = context.Background() + req := &pb.GlobalScheduleRequest{Uid: 200, Name: "func_name"} + m := mockSchedulerClient { + requestsReceived: make(map[uint64]bool), + } + + s := GCSObjServer { + generating: make(map[uint64]uint64), + globalSchedulerClient: m, + lineage: make(map[uint64]*pb.GlobalScheduleRequest), + liveNodes: make(map[uint64]bool), + } + + s.RegisterLineage(ctx, req) + + s.RegisterGenerating(ctx, &pb.GeneratingRequest{Uid: 200, NodeId: 5}) + + //Node 5 is assumed to be dead b/c we're not sending heartbeats. + + s.RequestLocation(ctx, &pb.RequestLocationRequest{Uid: 200}) + + if val, ok := m.requestsReceived[200]; !ok || val.Name != "func_name"{ + t.Errorf("Global scheduler never received new schedule request") + } + + + + + + + } // // MockGCSObjServer inherits GCSObjServer and overrides sendCallback diff --git a/go/cmd/globalscheduler/main.go b/go/cmd/globalscheduler/main.go index 7f790e8..2097df7 100644 --- a/go/cmd/globalscheduler/main.go +++ b/go/cmd/globalscheduler/main.go @@ -8,9 +8,12 @@ import ( "math/rand" "math" // "bytes" - "time" + +// "os" +// "time" // "os" "sync" + "time" "fmt" "google.golang.org/grpc" pb "github.com/rodrigo-castellon/babyray/pkg" @@ -21,47 +24,27 @@ import ( // "github.com/go-zookeeper/zk" ) var cfg *config.Config - +const LIVE_NODE_TIMEOUT time.Duration = 400 * time.Millisecond +const HEARTBEAT_WAIT = 100 * time.Millisecond var mu sync.RWMutex -// func connectToZookeeper(servers []string) (*zk.Conn, error) { -// var conn *zk.Conn -// var err error -// for i := 0; i < 10; i++ { // retry 10 times -// conn, _, err = zk.Connect(servers, time.Second*10) -// if err == nil { -// return conn, nil -// } -// log.Printf("Failed to connect to Zookeeper, retrying in 5 seconds... (attempt %d/10)", i+1) -// time.Sleep(5 * time.Second) -// } -// return nil, err -// } +const MAX_CONCURRENT_TASKS = 10 + +// LocalLog formats the message and logs it with a specific prefix +func LocalLog(format string, v ...interface{}) { + var logMessage string + if len(v) == 0 { + logMessage = format // No arguments, use the format string as-is + } else { + logMessage = fmt.Sprintf(format, v...) + } + log.Printf("[global scheduler] %s", logMessage) +} func main() { customlog.Init() - cfg = config.LoadConfig() // Load configuration - - // connect to zookeeper - // servers := []string{"zookeeper:2181"} - // // zkConn, _, err := zk.Connect(servers, time.Second) - // zkConn, err := connectToZookeeper(servers) - // if err != nil { - // log.Fatalf("Unable to connect to Zookeeper: %v", err) - // } - - // defer zkConn.Close() - - // path := "/services/globalsched" - // data := []byte("node1") - - // _, err = zkConn.Create(path, data, zk.FlagEphemeral, zk.WorldACL(zk.PermAll)) - // if err != nil { - // log.Fatalf("Unable to create znode: %v", err) - // } - - // log.Println("Service registered with Zookeeper") - + ctx := context.Background() + cfg = config.GetConfig() // Load configuration address := ":" + strconv.Itoa(cfg.Ports.GlobalScheduler) // Prepare the network address lis, err := net.Listen("tcp", address) @@ -72,15 +55,18 @@ func main() { s := grpc.NewServer(util.GetServerOptions()...) gcsAddress := fmt.Sprintf("%s%d:%d", cfg.DNS.NodePrefix, cfg.NodeIDs.GCS, cfg.Ports.GCSObjectTable) conn, _ := grpc.Dial(gcsAddress, util.GetDialOptions()...) - pb.RegisterGlobalSchedulerServer(s, &server{gcsClient: pb.NewGCSObjClient(conn), status: make(map[uint64]HeartbeatEntry)}) + serverInstance := &server{gcsClient: pb.NewGCSObjClient(conn), status: make(map[uint64]HeartbeatEntry)} + pb.RegisterGlobalSchedulerServer(s, serverInstance) defer conn.Close() - log.Printf("server listening at %v", lis.Addr()) + // log.Printf("server listening at %v", lis.Addr()) + go serverInstance.LiveNodesHeartbeat(ctx) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } } type HeartbeatEntry struct { + timeReceived time.Time numRunningTasks uint32 numQueuedTasks uint32 avgRunningTime float32 @@ -89,24 +75,58 @@ type HeartbeatEntry struct { // server is used to implement your gRPC service. type server struct { pb.UnimplementedGlobalSchedulerServer - gcsClient pb.GCSObjClient + gcsClient ObjClient status map[uint64]HeartbeatEntry } +type ObjClient interface { + RegisterLiveNodes(ctx context.Context, req *pb.LiveNodesRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error) + RequestLocation(ctx context.Context, req *pb.RequestLocationRequest, opts ...grpc.CallOption) (*pb.RequestLocationResponse, error) + RegisterGenerating(ctx context.Context, req *pb.GeneratingRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error) + GetObjectLocations(ctx context.Context, req *pb.ObjectLocationsRequest, opts ...grpc.CallOption) (*pb.ObjectLocationsResponse, error) +} + func (s *server) Heartbeat(ctx context.Context, req *pb.HeartbeatRequest ) (*pb.StatusResponse, error) { - // log.Printf("heartbeat from %v", req.NodeId) + //log.Printf("heartbeat from %v", req.NodeId) mu.Lock() numQueuedTasks := req.QueuedTasks - if (req.RunningTasks == 10) { + if (req.RunningTasks == MAX_CONCURRENT_TASKS) { numQueuedTasks = numQueuedTasks + 1 // also need to wait for something currently running to finish } - s.status[req.NodeId] = HeartbeatEntry{numRunningTasks: req.RunningTasks, numQueuedTasks: numQueuedTasks, avgRunningTime: req.AvgRunningTime, avgBandwidth: req.AvgBandwidth} + s.status[req.NodeId] = HeartbeatEntry{timeReceived: time.Now(), numRunningTasks: req.RunningTasks, numQueuedTasks: numQueuedTasks, avgRunningTime: req.AvgRunningTime, avgBandwidth: req.AvgBandwidth} mu.Unlock() return &pb.StatusResponse{Success: true}, nil } +func (s *server) LiveNodesHeartbeat(ctx context.Context) (error) { + ctx = context.Background() + for { + s.SendLiveNodes(ctx) + time.Sleep(HEARTBEAT_WAIT) + } + return nil + +} + +func(s *server) SendLiveNodes(ctx context.Context) (error) { + liveNodes := make(map[uint64]bool) + mu.RLock() + for uid, heartbeat := range s.status { + liveNodes[uid] = time.Since(heartbeat.timeReceived) < LIVE_NODE_TIMEOUT + } + mu.RUnlock() + // LocalLog("sending RegisterLiveNodes() call now") + // go func() { + // if _, err := s.gcsClient.RegisterLiveNodes(ctx, &pb.LiveNodesRequest{LiveNodes: liveNodes}); err != nil { + // LocalLog("Error registering live nodes: %v", err) + // } + // }() + s.gcsClient.RegisterLiveNodes(ctx, &pb.LiveNodesRequest{LiveNodes: liveNodes}) + return nil +} + func (s *server) Schedule(ctx context.Context , req *pb.GlobalScheduleRequest ) (*pb.StatusResponse, error) { localityFlag := req.LocalityFlag // localityFlag := false @@ -114,9 +134,9 @@ func (s *server) Schedule(ctx context.Context , req *pb.GlobalScheduleRequest ) // localityFlag = true // } - log.Printf("locality aware? it's: %v", localityFlag) + // log.Printf("locality aware? it's: %v", localityFlag) - log.Printf("THE REQ UIDS ARE = %v", req.Uids) + // log.Printf("THE REQ UIDS ARE = %v", req.Uids) // gives us back the node id of the worker node_id := req.NodeId @@ -124,26 +144,23 @@ func (s *server) Schedule(ctx context.Context , req *pb.GlobalScheduleRequest ) node_id = getBestWorker(ctx, s, localityFlag, req.Uids) } - log.Printf("best worker was node_id = %v", node_id) + // log.Printf("best worker was node_id = %v", node_id) workerAddress := fmt.Sprintf("%s%d:%d", cfg.DNS.NodePrefix, node_id, cfg.Ports.LocalWorkerStart) conn, err := grpc.Dial(workerAddress, util.GetDialOptions()...) if err != nil { - log.Printf("failed to connect to %s: %v", workerAddress, err) + // log.Printf("failed to connect to %s: %v", workerAddress, err) return nil, err } defer conn.Close() workerClient := pb.NewWorkerClient(conn) - - log.Printf("gonna call Run() now") - - // Example of creating a context with a timeout - // For some reason this custom context is necessary... ¯\_(ツ)_/¯ - customCtx, cancel := context.WithTimeout(context.Background(), time.Hour) - defer cancel() - - output_result, err := workerClient.Run(customCtx, &pb.RunRequest{Uid: req.Uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs}) + // LocalLog("Contacted the worker") + // if req.NewObject { + // LocalLog("registering this as a generating node now.") + s.gcsClient.RegisterGenerating(ctx, &pb.GeneratingRequest{Uid: req.Uid, NodeId: node_id}) + // } + output_result, err := workerClient.Run(ctx, &pb.RunRequest{Uid: req.Uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs}) if err != nil || !output_result.Success { log.Fatalf(fmt.Sprintf("global scheduler failed to contact node %d. Err: %v", node_id, err)) } @@ -158,8 +175,8 @@ func getBestWorker(ctx context.Context, s *server, localityFlag bool, uids []uin minTime = math.MaxFloat32 if (localityFlag && len(uids) > 0) { - log.Printf("LOCALITY FLAG IS ON!") - log.Printf("ASKING THE GCS FOR THESE OBJECTS: %v", uids) + // log.Printf("LOCALITY FLAG IS ON!") + // log.Printf("ASKING THE GCS FOR THESE OBJECTS: %v", uids) locationsResp, err := s.gcsClient.GetObjectLocations(ctx, &pb.ObjectLocationsRequest{Args: uids}) if err != nil { log.Fatalf("Failed to ask gcs for object locations: %v", err) @@ -176,20 +193,38 @@ func getBestWorker(ctx context.Context, s *server, localityFlag bool, uids []uin total = 0 for _, val := range locationsResp.Locations { locs := val.Locations - log.Printf("the locations resp val = %v", val) + // log.Printf("the locations resp val = %v", val) for _, loc := range locs { locationToBytes[uint64(loc)] += val.Bytes total += val.Bytes } } - for loc, bytes := range locationToBytes { + // Collect keys from the map + keys := make([]uint64, 0, len(s.status)) + for id := range s.status { + keys = append(keys, id) + } + + // Shuffle the keys + rand.Shuffle(len(keys), func(i, j int) { + keys[i], keys[j] = keys[j], keys[i] + }) + + for _, loc := range keys { + + // for loc, bytes := range locationToBytes { + bytes := locationToBytes[loc] + // skip dead nodes + if heartbeat, _ := s.status[loc]; !(time.Since(heartbeat.timeReceived) < LIVE_NODE_TIMEOUT) { + continue + } mu.RLock() queueingTime := float32(s.status[loc].numQueuedTasks) * s.status[loc].avgRunningTime transferTime := float32(total - bytes) / s.status[loc].avgBandwidth mu.RUnlock() - log.Printf("worker = %v; queueing and transfer = %v, %v", loc, queueingTime, transferTime) + // log.Printf("worker = %v; queueing and transfer = %v, %v", loc, queueingTime, transferTime) waitingTime := queueingTime + transferTime if waitingTime < minTime { @@ -199,7 +234,7 @@ func getBestWorker(ctx context.Context, s *server, localityFlag bool, uids []uin } } } else { - log.Printf("doing the statuses rn") + // log.Printf("doing the statuses rn") // TODO: make iteration order random for maximum fairness mu.RLock() @@ -216,7 +251,11 @@ func getBestWorker(ctx context.Context, s *server, localityFlag bool, uids []uin for _, id := range keys { heartbeat := s.status[id] - log.Printf("worker = %v; queued time = %v", id, float32(heartbeat.numQueuedTasks)*heartbeat.avgRunningTime) + if !(time.Since(heartbeat.timeReceived) < LIVE_NODE_TIMEOUT) { + continue + } + + // log.Printf("worker = %v; queued time = %v", id, float32(heartbeat.numQueuedTasks)*heartbeat.avgRunningTime) if float32(heartbeat.numQueuedTasks)*heartbeat.avgRunningTime < minTime { minId = id minTime = float32(heartbeat.numQueuedTasks) * heartbeat.avgRunningTime diff --git a/go/cmd/globalscheduler/main_test.go b/go/cmd/globalscheduler/main_test.go new file mode 100644 index 0000000..4532a10 --- /dev/null +++ b/go/cmd/globalscheduler/main_test.go @@ -0,0 +1,78 @@ +package main + +import ( + + "context" + + "testing" + "time" + pb "github.com/rodrigo-castellon/babyray/pkg" + "google.golang.org/grpc" + "google.golang.org/grpc/test/bufconn" +// "github.com/rodrigo-castellon/babyray/config" +) + +const bufSize = 1024 * 1024 + +var lis *bufconn.Listener + +func init() { + //cfg = config.GetConfig() +} + +// type mockFuncClient struct { +// pb.GCSFuncClient // Embedding the interface for forward compatibility +// resp *pb.FetchResponse +// err error +// } + +// func (m *mockFuncClient) FetchFunc(ctx context.Context, in *pb.FetchRequest, opts ...grpc.CallOption) (*pb.FetchResponse, error) { +// return m.resp, m.err +// } + +type mockGCSClient struct { + pb.GCSObjClient + liveNodes map[uint64]bool +} +func (m *mockGCSClient) RegisterLiveNodes(ctx context.Context, req *pb.LiveNodesRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error) { + m.liveNodes = req.LiveNodes; + + return &pb.StatusResponse{Success: true}, nil +} + + +func (m *mockGCSClient) RequestLocation(ctx context.Context, req *pb.RequestLocationRequest, opts ...grpc.CallOption) (*pb.RequestLocationResponse, error){ + return nil, nil +} +func (m *mockGCSClient) RegisterGenerating(ctx context.Context, req *pb.GeneratingRequest, opts ...grpc.CallOption) (*pb.StatusResponse, error){ + return nil, nil +} +func (m *mockGCSClient) GetObjectLocations(ctx context.Context, req *pb.ObjectLocationsRequest, opts ...grpc.CallOption) (*pb.ObjectLocationsResponse, error) { + return nil, nil +} +func TestHeartbeats(t *testing.T) { + ctx :=context.Background() + + m := mockGCSClient { + liveNodes: make(map[uint64]bool), + } + s := server { + gcsClient: &m, + status: make(map[uint64]HeartbeatEntry), + } + + s.Heartbeat(ctx, &pb.HeartbeatRequest{NodeId: 200}) + s.SendLiveNodes(ctx) + if val, ok := m.liveNodes[200]; val != true || !ok { + t.Errorf("Node was not correctly registered as alive with GCS") + } + + time.Sleep(2 * LIVE_NODE_TIMEOUT) + + s.SendLiveNodes(ctx); + + if val, _ := m.liveNodes[200]; val == true { + t.Errorf("Node was not correctly registered as dead with GCS") + } +} + diff --git a/go/cmd/localobjstore/main.go b/go/cmd/localobjstore/main.go index d3ddfc9..27f2b44 100644 --- a/go/cmd/localobjstore/main.go +++ b/go/cmd/localobjstore/main.go @@ -25,7 +25,7 @@ const DEFAULT_AVG_BANDWIDTH = 0.1 // to start with func createSharedMemory(name string, size int) ([]byte, int, error) { // Create a shared memory segment - LocalLog("WERE GONNA CREATE A /DEV FILE WITH THIS NAME: %s", "/dev/shm/"+name) + // LocalLog("WERE GONNA CREATE A /DEV FILE WITH THIS NAME: %s", "/dev/shm/"+name) shmFd, err := syscall.Open("/dev/shm/"+name, syscall.O_CREAT|syscall.O_RDWR, 0666) if err != nil { return nil, -1, err @@ -42,14 +42,14 @@ func createSharedMemory(name string, size int) ([]byte, int, error) { return nil, -1, err } - LocalLog("/DEV FILE CREATED!") + // LocalLog("/DEV FILE CREATED!") return shmAddr, shmFd, nil } // Function to open and read from a shared memory segment func readSharedMemory(name string, size int) ([]byte, error) { - LocalLog("WERE GONNA READ A /DEV FILE WITH THIS NAME: %s", "/dev/shm/"+name) + // LocalLog("WERE GONNA READ A /DEV FILE WITH THIS NAME: %s", "/dev/shm/"+name) // Open the shared memory object shmFd, err := syscall.Open("/dev/shm/"+name, syscall.O_RDWR, 0666) if err != nil { @@ -121,7 +121,7 @@ func startServer(port string) (*grpc.Server, error) { pb.RegisterLocalObjStoreServer(s, &server{localObjectStore: make(map[uint64][]byte), objectSizes: make(map[uint64]uint64), localObjectChannels: make(map[uint64]chan *pb.LocationFoundCallback), gcsObjClient: pb.NewGCSObjClient(conn), localNodeID: uint64(nodeId), avgBandwidth: DEFAULT_AVG_BANDWIDTH}) - LocalLog("lobs server listening at %v", lis.Addr()) + // LocalLog("lobs server listening at %v", lis.Addr()) go func() { if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) @@ -153,12 +153,12 @@ func (s *server) Store(ctx context.Context, req *pb.StoreRequest) (*pb.StatusRes return nil, err } - LocalLog("NOW ATTEMPTING TO COPY OBJECT BYTES INTO SHARED MEMORY SLICE") + // LocalLog("NOW ATTEMPTING TO COPY OBJECT BYTES INTO SHARED MEMORY SLICE") copy(shmSlice, req.ObjectBytes) - LocalLog("COPIED!") + // LocalLog("COPIED!") mu.Lock() // s.localObjectStore[req.Uid] = req.ObjectBytes @@ -166,7 +166,7 @@ func (s *server) Store(ctx context.Context, req *pb.StoreRequest) (*pb.StatusRes s.objectSizes[req.Uid] = size; mu.Unlock() - LocalLog("STORED!!!") + // LocalLog("STORED!!!") s.gcsObjClient.NotifyOwns(ctx, &pb.NotifyOwnsRequest{Uid: req.Uid, NodeId: s.localNodeID, ObjectSize: size}) return &pb.StatusResponse{Success: true}, nil @@ -176,10 +176,10 @@ func (s *server) Get(ctx context.Context, req *pb.GetRequest) (*pb.GetResponse, mu.RLock() if val, ok := s.objectSizes[req.Uid]; ok { mu.RUnlock() - LocalLog("ITS IN OBJECT SIZES!") + // LocalLog("ITS IN OBJECT SIZES!") return &pb.GetResponse{Uid: req.Uid, ObjectBytes: []byte{}, Local: true, Size: val}, nil } - LocalLog("THE UID WAS NOT IN OBJECT SIZES: %v", req.Uid) + // LocalLog("THE UID WAS NOT IN OBJECT SIZES: %v", req.Uid) // if val, ok := s.localObjectStore[req.Uid]; ok { // mu.RUnlock() // return &pb.GetResponse{Uid: req.Uid, ObjectBytes: val, Local: true}, nil @@ -281,7 +281,7 @@ func (s *server) Copy(ctx context.Context, req *pb.CopyRequest) (*pb.CopyRespons data, err := readSharedMemory(strconv.FormatUint(req.Uid, 10), int(size)) if err != nil { - LocalLog("got err from read: %v", err) + // LocalLog("got err from read: %v", err) return nil, err } diff --git a/go/cmd/localscheduler/main.go b/go/cmd/localscheduler/main.go index d19f8e0..5c12f03 100644 --- a/go/cmd/localscheduler/main.go +++ b/go/cmd/localscheduler/main.go @@ -56,15 +56,20 @@ func main() { globalSchedulerClient := pb.NewGlobalSchedulerClient(conn) nodeId, _ := strconv.Atoi(os.Getenv("NODE_ID")) - pb.RegisterLocalSchedulerServer(s, &server{globalSchedulerClient: globalSchedulerClient, workerClient: workerClient, globalCtx: context.Background(), localNodeID: uint64(nodeId)}) + gcsObjAddress := fmt.Sprintf("%s%d:%d", cfg.DNS.NodePrefix, cfg.NodeIDs.GCS, cfg.Ports.GCSObjectTable) + gcsConn, _ := grpc.Dial(gcsObjAddress, grpc.WithInsecure()) + gcsObjClient := pb.NewGCSObjClient(gcsConn) + server := &server{globalSchedulerClient: globalSchedulerClient, workerClient: workerClient, globalCtx: context.Background(), localNodeID: uint64(nodeId), gcsClient: gcsObjClient, alive: true} + pb.RegisterLocalSchedulerServer(s, server) ctx := context.Background() - go SendHeartbeats(ctx, globalSchedulerClient, uint64(nodeId)) + go server.SendHeartbeats(ctx, globalSchedulerClient, uint64(nodeId)) - LocalLog("localsched server listening at %v", lis.Addr()) + // LocalLog("localsched server listening at %v", lis.Addr()) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } + } // server is used to implement your gRPC service. @@ -72,8 +77,10 @@ type server struct { pb.UnimplementedLocalSchedulerServer globalSchedulerClient pb.GlobalSchedulerClient workerClient pb.WorkerClient + gcsClient pb.GCSObjClient globalCtx context.Context localNodeID uint64 + alive bool } // TruncateHighBit truncates the most significant bit of a uint64 value @@ -85,19 +92,24 @@ func TruncateHighBit(u64 uint64) uint64 { // Implement your service methods here. func (s *server) Schedule(ctx context.Context, req *pb.ScheduleRequest) (*pb.ScheduleResponse, error) { - var worker_id int - worker_id, _ = strconv.Atoi(os.Getenv("NODE_ID")) + // var worker_id int + // worker_id, _ = strconv.Atoi(os.Getenv("NODE_ID")) uid := rand.Uint64() uid = TruncateHighBit(uid) // NOTE: THIS IS A WORKAROUND FOR THE FACT THAT SQLite3 IN THE GCS OBJECT TABLE's STORAGE CANNOT TAKE HIGH BIT OF UINT64 + _, err := s.gcsClient.RegisterLineage(ctx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids}) + if err != nil { + // LocalLog("cant hit gcs: %v", err) + } + // custom behavior if the client itself specifies where we should send this // computation if req.NodeId != 0 { - LocalLog("Doing something special with req.NodeId = %v", req.NodeId) + // LocalLog("Doing something special with req.NodeId = %v", req.NodeId) go func() { - _, err := s.globalSchedulerClient.Schedule(s.globalCtx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids, NodeId: req.NodeId}) + _, err := s.globalSchedulerClient.Schedule(s.globalCtx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids, NewObject: true, NodeId: req.NodeId}) if err != nil { - LocalLog("cannot contact global scheduler") + // LocalLog("cannot contact global scheduler") } else { // LocalLog("Just ran it on global!") } @@ -105,28 +117,29 @@ func (s *server) Schedule(ctx context.Context, req *pb.ScheduleRequest) (*pb.Sch return &pb.ScheduleResponse{Uid: uid}, nil } - LocalLog("asking about the worker status") + // LocalLog("asking about the worker status") scheduleLocally, _ := s.workerClient.WorkerStatus(ctx, &pb.StatusResponse{}) - LocalLog("heard back from the worker status: %v", scheduleLocally.NumRunningTasks) + // LocalLog("heard back from the worker status: %v", scheduleLocally.NumRunningTasks) if scheduleLocally.NumRunningTasks < MAX_TASKS { - LocalLog("Just running locally") + // LocalLog("Just running locally") go func() { + s.gcsClient.RegisterGenerating(ctx, &pb.GeneratingRequest{Uid: uid, NodeId: s.localNodeID}) _, err := s.workerClient.Run(s.globalCtx, &pb.RunRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs}) if err != nil { - LocalLog("cannot contact worker %d: %v", worker_id, err) + // LocalLog("cannot contact worker %d: %v", worker_id, err) } else { // LocalLog("Just ran it!") } }() } else { - LocalLog("contacting global scheduler") + // LocalLog("contacting global scheduler") go func() { - LocalLog("THE REQ UIDS AT LOCAL SCHEDULER ARE %v", req.Uids) - _, err := s.globalSchedulerClient.Schedule(s.globalCtx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids, LocalityFlag: req.LocalityFlag}) + // LocalLog("THE REQ UIDS AT LOCAL SCHEDULER ARE %v", req.Uids) + _, err := s.globalSchedulerClient.Schedule(s.globalCtx, &pb.GlobalScheduleRequest{Uid: uid, Name: req.Name, Args: req.Args, Kwargs: req.Kwargs, Uids: req.Uids, NewObject: true, LocalityFlag: req.LocalityFlag}) if err != nil { - LocalLog("cannot contact global scheduler") + // LocalLog("cannot contact global scheduler") } else { // LocalLog("Just ran it on global!") } @@ -137,7 +150,19 @@ func (s *server) Schedule(ctx context.Context, req *pb.ScheduleRequest) (*pb.Sch } -func SendHeartbeats(ctx context.Context, globalSchedulerClient pb.GlobalSchedulerClient, nodeId uint64) { +func (s *server) KillServer(ctx context.Context, req *pb.StatusResponse) (*pb.StatusResponse, error) { + // LocalLog("GOT KILLED!") + s.alive = false + return &pb.StatusResponse{Success: true}, nil +} + +func (s *server) ReviveServer(ctx context.Context, req *pb.StatusResponse) (*pb.StatusResponse, error) { + s.alive = true + return &pb.StatusResponse{Success: true}, nil +} + +func (s *server) SendHeartbeats(ctx context.Context, globalSchedulerClient pb.GlobalSchedulerClient, nodeId uint64) { + workerAddress := fmt.Sprintf("localhost:%d", cfg.Ports.LocalWorkerStart) workerConn, err := grpc.Dial(workerAddress, util.GetDialOptions()...) if err != nil { @@ -156,60 +181,62 @@ func SendHeartbeats(ctx context.Context, globalSchedulerClient pb.GlobalSchedule workerClient := pb.NewWorkerClient(workerConn) lobsClient := pb.NewLocalObjStoreClient(lobsConn) for { - backoff := 1 - var status *pb.WorkerStatusResponse - - for { - status, err = workerClient.WorkerStatus(ctx, &pb.StatusResponse{}) - if err != nil { - LocalLog("got error from WorkerStatus(): %v", err) - LocalLog("retrying in %v seconds", backoff) - time.Sleep(time.Duration(backoff) * time.Second) - backoff *= 2 - if backoff > 32 { - backoff = 32 // Cap the backoff to 32 seconds + if s.alive { + backoff := 1 + var status *pb.WorkerStatusResponse + + for { + status, err = workerClient.WorkerStatus(ctx, &pb.StatusResponse{}) + if err != nil { + // LocalLog("got error from WorkerStatus(): %v", err) + // LocalLog("retrying in %v seconds", backoff) + time.Sleep(time.Duration(backoff) * time.Second) + backoff *= 2 + if backoff > 32 { + backoff = 32 // Cap the backoff to 32 seconds + } + continue } - continue + break } - break - } - numRunningTasks := status.NumRunningTasks - numQueuedTasks := status.NumQueuedTasks - avgRunningTime := status.AverageRunningTime - var avgBandwidth *pb.BandwidthResponse - backoff = 1 - for { - avgBandwidth, err = lobsClient.AvgBandwidth(ctx, &pb.StatusResponse{}) - if err != nil { - LocalLog("got error from WorkerStatus(): %v", err) - LocalLog("retrying in %v seconds", backoff) - time.Sleep(time.Duration(backoff) * time.Second) - backoff *= 2 - if backoff > 32 { - backoff = 32 // Cap the backoff to 32 seconds + numRunningTasks := status.NumRunningTasks + numQueuedTasks := status.NumQueuedTasks + avgRunningTime := status.AverageRunningTime + var avgBandwidth *pb.BandwidthResponse + backoff = 1 + for { + avgBandwidth, err = lobsClient.AvgBandwidth(ctx, &pb.StatusResponse{}) + if err != nil { + // LocalLog("got error from AvgBand(): %v", err) + // LocalLog("retrying in %v seconds", backoff) + time.Sleep(time.Duration(backoff) * time.Second) + backoff *= 2 + if backoff > 32 { + backoff = 32 // Cap the backoff to 32 seconds + } + continue } - continue + break } - break - } - heartbeatRequest := &pb.HeartbeatRequest{ - RunningTasks: numRunningTasks, - QueuedTasks: numQueuedTasks, - AvgRunningTime: avgRunningTime, - AvgBandwidth: avgBandwidth.AvgBandwidth, - NodeId: nodeId} - - // LocalLog("HeartbeatRequest: RunningTasks=%d, QueuedTasks=%d, AvgRunningTime=%.2f, AvgBandwidth=%.2f, NodeId=%d", - // heartbeatRequest.RunningTasks, - // heartbeatRequest.QueuedTasks, - // heartbeatRequest.AvgRunningTime, - // heartbeatRequest.AvgBandwidth, - // heartbeatRequest.NodeId) - - // LocalLog("SENDING GLOBAL SCHEDULER A HEARTBEAT!!!") - globalSchedulerClient.Heartbeat(ctx, heartbeatRequest) + heartbeatRequest := &pb.HeartbeatRequest{ + RunningTasks: numRunningTasks, + QueuedTasks: numQueuedTasks, + AvgRunningTime: avgRunningTime, + AvgBandwidth: avgBandwidth.AvgBandwidth, + NodeId: nodeId} + + // LocalLog("HeartbeatRequest: RunningTasks=%d, QueuedTasks=%d, AvgRunningTime=%.2f, AvgBandwidth=%.2f, NodeId=%d", + // heartbeatRequest.RunningTasks, + // heartbeatRequest.QueuedTasks, + // heartbeatRequest.AvgRunningTime, + // heartbeatRequest.AvgBandwidth, + // heartbeatRequest.NodeId) + + // LocalLog("SENDING GLOBAL SCHEDULER A HEARTBEAT!!!") + globalSchedulerClient.Heartbeat(ctx, heartbeatRequest) + } time.Sleep(HEARTBEAT_WAIT) } } diff --git a/go/cmd/worker/main.go b/go/cmd/worker/main.go index 9f1730f..a95a95b 100644 --- a/go/cmd/worker/main.go +++ b/go/cmd/worker/main.go @@ -14,6 +14,7 @@ import ( "encoding/base64" "sync" "time" + "runtime" "google.golang.org/grpc" pb "github.com/rodrigo-castellon/babyray/pkg" @@ -75,9 +76,10 @@ func main() { pb.RegisterWorkerServer(s, &workerServer{ funcClient: funcClient, storeClient: storeClient, + alive: true, }) - LocalLog("worker server listening at %v", lis.Addr()) + // LocalLog("worker server listening at %v", lis.Addr()) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } @@ -87,6 +89,7 @@ type workerServer struct { pb.UnimplementedWorkerServer funcClient FuncClient storeClient StoreClient + alive bool } type FuncClient interface { @@ -98,60 +101,61 @@ type StoreClient interface { } func executeFunction(f []byte, args []byte, kwargs []byte) ([]byte, error) { - startTime := time.Now() + // startTime := time.Now() // Prepare the command to run the Python script rootPath := os.Getenv("PROJECT_ROOT") executeFile := filepath.Join(rootPath, "go", "cmd", "worker", "execute.py") cmd := exec.Command("/usr/bin/python3", executeFile) - LocalLog("Time to prepare command: %v\n", time.Since(startTime)) + // LocalLog("Time to prepare command: %v\n", time.Since(startTime)) // Create a buffer to hold the serialized data inputBuffer := bytes.NewBuffer(nil) // Assume data is the serialized data you want to encode in base64 - encodeStartTime := time.Now() + // encodeStartTime := time.Now() fB64 := []byte(base64.StdEncoding.EncodeToString(f)) argsB64 := []byte(base64.StdEncoding.EncodeToString(args)) kwargsB64 := []byte(base64.StdEncoding.EncodeToString(kwargs)) - LocalLog("Time to encode inputs: %v\n", time.Since(encodeStartTime)) + // LocalLog("Time to encode inputs: %v\n", time.Since(encodeStartTime)) // Write the function, args, and kwargs to the buffer - bufferWriteStartTime := time.Now() + // bufferWriteStartTime := time.Now() inputBuffer.Write(fB64) inputBuffer.WriteByte('\n') inputBuffer.Write(argsB64) inputBuffer.WriteByte('\n') inputBuffer.Write(kwargsB64) - LocalLog("Time to write to buffer: %v\n", time.Since(bufferWriteStartTime)) + // LocalLog("Time to write to buffer: %v\n", time.Since(bufferWriteStartTime)) // Set the stdin to our input buffer cmd.Stdin = inputBuffer // Capture the output - cmdExecStartTime := time.Now() + // LocalLog("Running the function here yo") + // cmdExecStartTime := time.Now() output, err := cmd.Output() if err != nil { log.Fatalf("Error executing function: %v", err) } - LocalLog("Time to execute command: %v\n", time.Since(cmdExecStartTime)) + // LocalLog("Time to execute command: %v\n", time.Since(cmdExecStartTime)) // Decode the Base64 output to get the original pickled data // LocalLog("the output from this was: %s", string(output)[:min(len(string(output)), 282)]) // LocalLog("the length of the overall string was: %v", len(string(output))) - decodeStartTime := time.Now() + // decodeStartTime := time.Now() data, err := base64.StdEncoding.DecodeString(string(output)) if err != nil { log.Fatalf("Error decoding Base64: %v", err) } - LocalLog("Time to decode output: %v\n", time.Since(decodeStartTime)) + // LocalLog("Time to decode output: %v\n", time.Since(decodeStartTime)) - totalTime := time.Since(startTime) - log.Printf("Total execution time: %v\n", totalTime) + // totalTime := time.Since(startTime) + // log.Printf("Total execution time: %v\n", totalTime) // Return the output from the Python script return data, nil @@ -159,7 +163,8 @@ func executeFunction(f []byte, args []byte, kwargs []byte) ([]byte, error) { // run executes the function by fetching it, running it, and storing the result. func (s *workerServer) Run(ctx context.Context, req *pb.RunRequest) (*pb.StatusResponse, error) { - LocalLog("in Run() rn") + // LocalLog("in Run() rn") + // defer LocalLog("Finished running") mu.Lock() numQueuedTasks++ mu.Unlock() @@ -180,27 +185,36 @@ func (s *workerServer) Run(ctx context.Context, req *pb.RunRequest) (*pb.StatusR start := time.Now() - LocalLog("....fetching?") + // LocalLog("....fetching?") funcResponse, err := s.funcClient.FetchFunc(ctx, &pb.FetchRequest{Name: req.Name}) - LocalLog("the err was %v", err) + // LocalLog("the err was %v", err) if err != nil { + // LocalLog("Failed to hit func table: %v", err) return nil, err } - LocalLog("gonna execute now") + // LocalLog("gonna execute now") output, err := executeFunction(funcResponse.SerializedFunc, req.Args, req.Kwargs) - LocalLog("Executed!") + // LocalLog("Executed!") if err != nil { + // LocalLog("failed to exec func %v", err) return nil, err } - LocalLog("gonna store now") + // LocalLog("finished executing function") + + // LocalLog("s.alive is %v", s.alive) + + if (!s.alive) { + // LocalLog("Worker is not alive, exiting...") + runtime.Goexit() + } + _, err = s.storeClient.Store(ctx, &pb.StoreRequest{Uid: req.Uid, ObjectBytes: output}) if err != nil { - LocalLog("store client returned an err: %v", err) + // LocalLog("failed to hit gcs %v", err) return nil, err } - LocalLog("stored...") runningTime := float32(time.Since(start).Seconds()) mu.Lock() @@ -210,10 +224,22 @@ func (s *workerServer) Run(ctx context.Context, req *pb.RunRequest) (*pb.StatusR return &pb.StatusResponse{Success: true}, nil } +func (s *workerServer) KillServer(ctx context.Context, req *pb.StatusResponse) (*pb.StatusResponse, error) { + // LocalLog("GOT KILLED!") + s.alive = false + // LocalLog("s.alive is now: %v", s.alive) + return &pb.StatusResponse{Success: true}, nil +} + +func (s *workerServer) ReviveServer(ctx context.Context, req *pb.StatusResponse) (*pb.StatusResponse, error) { + s.alive = true + return &pb.StatusResponse{Success: true}, nil +} + func (s *workerServer) WorkerStatus(ctx context.Context, req *pb.StatusResponse) (*pb.WorkerStatusResponse, error) { mu.Lock() defer mu.Unlock() - // log.Printf("num queued tasks is currently: %v", numQueuedTasks) + //log.Printf("num queued tasks is currently: %v", numQueuedTasks) return &pb.WorkerStatusResponse{ NumRunningTasks: numRunningTasks, NumQueuedTasks: numQueuedTasks, diff --git a/go/cmd/worker/main_test.go b/go/cmd/worker/main_test.go index 9d2c707..8e558a5 100644 --- a/go/cmd/worker/main_test.go +++ b/go/cmd/worker/main_test.go @@ -128,7 +128,7 @@ func TestWorkerRun(t *testing.T) { _ = resp _ = err - log.Printf("%v", resp) + // log.Printf("%v", resp) // if err != nil { // t.Errorf("Run failed: %v", err) // } else if !resp.Success { diff --git a/go/pkg/rayclient.pb.go b/go/pkg/rayclient.pb.go index 0170ad4..ad29be0 100644 --- a/go/pkg/rayclient.pb.go +++ b/go/pkg/rayclient.pb.go @@ -1,15 +1,15 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v3.19.1 +// protoc-gen-go v1.34.1 +// protoc v3.6.1 // source: rayclient.proto package grpc import ( + _ "github.com/golang/protobuf/ptypes/empty" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - _ "google.golang.org/protobuf/types/known/emptypb" reflect "reflect" sync "sync" ) @@ -98,10 +98,12 @@ type GlobalScheduleRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` - Name uint64 `protobuf:"varint,2,opt,name=name,proto3" json:"name,omitempty"` - Args []byte `protobuf:"bytes,3,opt,name=args,proto3" json:"args,omitempty"` - Kwargs []byte `protobuf:"bytes,4,opt,name=kwargs,proto3" json:"kwargs,omitempty"` + Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` + Name uint64 `protobuf:"varint,2,opt,name=name,proto3" json:"name,omitempty"` + Args []byte `protobuf:"bytes,3,opt,name=args,proto3" json:"args,omitempty"` + Kwargs []byte `protobuf:"bytes,4,opt,name=kwargs,proto3" json:"kwargs,omitempty"` + Uids []uint64 `protobuf:"varint,5,rep,packed,name=uids,proto3" json:"uids,omitempty"` + NewObject bool `protobuf:"varint,6,opt,name=newObject,proto3" json:"newObject,omitempty"` } func (x *GlobalScheduleRequest) Reset() { @@ -164,6 +166,20 @@ func (x *GlobalScheduleRequest) GetKwargs() []byte { return nil } +func (x *GlobalScheduleRequest) GetUids() []uint64 { + if x != nil { + return x.Uids + } + return nil +} + +func (x *GlobalScheduleRequest) GetNewObject() bool { + if x != nil { + return x.NewObject + } + return false +} + type HeartbeatRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -248,9 +264,10 @@ type ScheduleRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name uint64 `protobuf:"varint,1,opt,name=name,proto3" json:"name,omitempty"` - Args []byte `protobuf:"bytes,2,opt,name=args,proto3" json:"args,omitempty"` - Kwargs []byte `protobuf:"bytes,3,opt,name=kwargs,proto3" json:"kwargs,omitempty"` + Name uint64 `protobuf:"varint,1,opt,name=name,proto3" json:"name,omitempty"` + Args []byte `protobuf:"bytes,2,opt,name=args,proto3" json:"args,omitempty"` + Kwargs []byte `protobuf:"bytes,3,opt,name=kwargs,proto3" json:"kwargs,omitempty"` + Uids []uint64 `protobuf:"varint,4,rep,packed,name=uids,proto3" json:"uids,omitempty"` } func (x *ScheduleRequest) Reset() { @@ -306,6 +323,13 @@ func (x *ScheduleRequest) GetKwargs() []byte { return nil } +func (x *ScheduleRequest) GetUids() []uint64 { + if x != nil { + return x.Uids + } + return nil +} + // returns a Future, which is just the UID for // the generated object type ScheduleResponse struct { @@ -417,6 +441,13 @@ type GetRequest struct { Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` Testing bool `protobuf:"varint,2,opt,name=testing,proto3" json:"testing,omitempty"` + // normally is true, but is false whenever we want to just block + // for a task to complete + Copy bool `protobuf:"varint,3,opt,name=copy,proto3" json:"copy,omitempty"` + // normally is true, but is false whenever we want to copy the + // object over but not cache it (so we can keep measuring the copy + // speed) + Cache bool `protobuf:"varint,4,opt,name=cache,proto3" json:"cache,omitempty"` } func (x *GetRequest) Reset() { @@ -465,6 +496,20 @@ func (x *GetRequest) GetTesting() bool { return false } +func (x *GetRequest) GetCopy() bool { + if x != nil { + return x.Copy + } + return false +} + +func (x *GetRequest) GetCache() bool { + if x != nil { + return x.Cache + } + return false +} + type GetResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -473,6 +518,7 @@ type GetResponse struct { Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` ObjectBytes []byte `protobuf:"bytes,2,opt,name=objectBytes,proto3" json:"objectBytes,omitempty"` Local bool `protobuf:"varint,3,opt,name=local,proto3" json:"local,omitempty"` + Size uint64 `protobuf:"varint,4,opt,name=size,proto3" json:"size,omitempty"` } func (x *GetResponse) Reset() { @@ -528,6 +574,13 @@ func (x *GetResponse) GetLocal() bool { return false } +func (x *GetResponse) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + type BandwidthResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -895,8 +948,9 @@ type NotifyOwnsRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` - NodeId uint64 `protobuf:"varint,2,opt,name=nodeId,proto3" json:"nodeId,omitempty"` + Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` + NodeId uint64 `protobuf:"varint,2,opt,name=nodeId,proto3" json:"nodeId,omitempty"` + ObjectSize uint64 `protobuf:"varint,3,opt,name=objectSize,proto3" json:"objectSize,omitempty"` } func (x *NotifyOwnsRequest) Reset() { @@ -945,6 +999,13 @@ func (x *NotifyOwnsRequest) GetNodeId() uint64 { return 0 } +func (x *NotifyOwnsRequest) GetObjectSize() uint64 { + if x != nil { + return x.ObjectSize + } + return 0 +} + type RequestLocationRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1052,7 +1113,7 @@ type ObjectLocationsRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Args []byte `protobuf:"bytes,1,opt,name=args,proto3" json:"args,omitempty"` + Args []uint64 `protobuf:"varint,1,rep,packed,name=args,proto3" json:"args,omitempty"` } func (x *ObjectLocationsRequest) Reset() { @@ -1087,7 +1148,7 @@ func (*ObjectLocationsRequest) Descriptor() ([]byte, []int) { return file_rayclient_proto_rawDescGZIP(), []int{17} } -func (x *ObjectLocationsRequest) GetArgs() []byte { +func (x *ObjectLocationsRequest) GetArgs() []uint64 { if x != nil { return x.Args } @@ -1099,8 +1160,8 @@ type LocationByteTuple struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Location uint64 `protobuf:"varint,1,opt,name=location,proto3" json:"location,omitempty"` - Bytes uint64 `protobuf:"varint,2,opt,name=bytes,proto3" json:"bytes,omitempty"` + Locations []uint64 `protobuf:"varint,1,rep,packed,name=locations,proto3" json:"locations,omitempty"` + Bytes uint64 `protobuf:"varint,2,opt,name=bytes,proto3" json:"bytes,omitempty"` } func (x *LocationByteTuple) Reset() { @@ -1135,11 +1196,11 @@ func (*LocationByteTuple) Descriptor() ([]byte, []int) { return file_rayclient_proto_rawDescGZIP(), []int{18} } -func (x *LocationByteTuple) GetLocation() uint64 { +func (x *LocationByteTuple) GetLocations() []uint64 { if x != nil { - return x.Location + return x.Locations } - return 0 + return nil } func (x *LocationByteTuple) GetBytes() uint64 { @@ -1196,6 +1257,108 @@ func (x *ObjectLocationsResponse) GetLocations() map[uint64]*LocationByteTuple { return nil } +type LiveNodesRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + LiveNodes map[uint64]bool `protobuf:"bytes,1,rep,name=liveNodes,proto3" json:"liveNodes,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` +} + +func (x *LiveNodesRequest) Reset() { + *x = LiveNodesRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_rayclient_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LiveNodesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LiveNodesRequest) ProtoMessage() {} + +func (x *LiveNodesRequest) ProtoReflect() protoreflect.Message { + mi := &file_rayclient_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LiveNodesRequest.ProtoReflect.Descriptor instead. +func (*LiveNodesRequest) Descriptor() ([]byte, []int) { + return file_rayclient_proto_rawDescGZIP(), []int{20} +} + +func (x *LiveNodesRequest) GetLiveNodes() map[uint64]bool { + if x != nil { + return x.LiveNodes + } + return nil +} + +type GeneratingRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Uid uint64 `protobuf:"varint,1,opt,name=uid,proto3" json:"uid,omitempty"` + NodeId uint64 `protobuf:"varint,2,opt,name=nodeId,proto3" json:"nodeId,omitempty"` +} + +func (x *GeneratingRequest) Reset() { + *x = GeneratingRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_rayclient_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GeneratingRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GeneratingRequest) ProtoMessage() {} + +func (x *GeneratingRequest) ProtoReflect() protoreflect.Message { + mi := &file_rayclient_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GeneratingRequest.ProtoReflect.Descriptor instead. +func (*GeneratingRequest) Descriptor() ([]byte, []int) { + return file_rayclient_proto_rawDescGZIP(), []int{21} +} + +func (x *GeneratingRequest) GetUid() uint64 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *GeneratingRequest) GetNodeId() uint64 { + if x != nil { + return x.NodeId + } + return 0 +} + type RegisterRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1207,7 +1370,7 @@ type RegisterRequest struct { func (x *RegisterRequest) Reset() { *x = RegisterRequest{} if protoimpl.UnsafeEnabled { - mi := &file_rayclient_proto_msgTypes[20] + mi := &file_rayclient_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1220,7 +1383,7 @@ func (x *RegisterRequest) String() string { func (*RegisterRequest) ProtoMessage() {} func (x *RegisterRequest) ProtoReflect() protoreflect.Message { - mi := &file_rayclient_proto_msgTypes[20] + mi := &file_rayclient_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1233,7 +1396,7 @@ func (x *RegisterRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterRequest.ProtoReflect.Descriptor instead. func (*RegisterRequest) Descriptor() ([]byte, []int) { - return file_rayclient_proto_rawDescGZIP(), []int{20} + return file_rayclient_proto_rawDescGZIP(), []int{22} } func (x *RegisterRequest) GetSerializedFunc() []byte { @@ -1254,7 +1417,7 @@ type RegisterResponse struct { func (x *RegisterResponse) Reset() { *x = RegisterResponse{} if protoimpl.UnsafeEnabled { - mi := &file_rayclient_proto_msgTypes[21] + mi := &file_rayclient_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1267,7 +1430,7 @@ func (x *RegisterResponse) String() string { func (*RegisterResponse) ProtoMessage() {} func (x *RegisterResponse) ProtoReflect() protoreflect.Message { - mi := &file_rayclient_proto_msgTypes[21] + mi := &file_rayclient_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1280,7 +1443,7 @@ func (x *RegisterResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RegisterResponse.ProtoReflect.Descriptor instead. func (*RegisterResponse) Descriptor() ([]byte, []int) { - return file_rayclient_proto_rawDescGZIP(), []int{21} + return file_rayclient_proto_rawDescGZIP(), []int{23} } func (x *RegisterResponse) GetName() uint64 { @@ -1302,7 +1465,7 @@ type FetchRequest struct { func (x *FetchRequest) Reset() { *x = FetchRequest{} if protoimpl.UnsafeEnabled { - mi := &file_rayclient_proto_msgTypes[22] + mi := &file_rayclient_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1315,7 +1478,7 @@ func (x *FetchRequest) String() string { func (*FetchRequest) ProtoMessage() {} func (x *FetchRequest) ProtoReflect() protoreflect.Message { - mi := &file_rayclient_proto_msgTypes[22] + mi := &file_rayclient_proto_msgTypes[24] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1328,7 +1491,7 @@ func (x *FetchRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchRequest.ProtoReflect.Descriptor instead. func (*FetchRequest) Descriptor() ([]byte, []int) { - return file_rayclient_proto_rawDescGZIP(), []int{22} + return file_rayclient_proto_rawDescGZIP(), []int{24} } func (x *FetchRequest) GetName() uint64 { @@ -1356,7 +1519,7 @@ type FetchResponse struct { func (x *FetchResponse) Reset() { *x = FetchResponse{} if protoimpl.UnsafeEnabled { - mi := &file_rayclient_proto_msgTypes[23] + mi := &file_rayclient_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1369,7 +1532,7 @@ func (x *FetchResponse) String() string { func (*FetchResponse) ProtoMessage() {} func (x *FetchResponse) ProtoReflect() protoreflect.Message { - mi := &file_rayclient_proto_msgTypes[23] + mi := &file_rayclient_proto_msgTypes[25] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1382,7 +1545,7 @@ func (x *FetchResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchResponse.ProtoReflect.Descriptor instead. func (*FetchResponse) Descriptor() ([]byte, []int) { - return file_rayclient_proto_rawDescGZIP(), []int{23} + return file_rayclient_proto_rawDescGZIP(), []int{25} } func (x *FetchResponse) GetSerializedFunc() []byte { @@ -1406,192 +1569,232 @@ var file_rayclient_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x69, 0x0a, 0x15, - 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x61, - 0x72, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x12, - 0x16, 0x0a, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x22, 0xbc, 0x01, 0x0a, 0x10, 0x48, 0x65, 0x61, 0x72, - 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, - 0x72, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0c, 0x72, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x73, 0x6b, 0x73, - 0x12, 0x20, 0x0a, 0x0b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x64, 0x54, 0x61, 0x73, - 0x6b, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x76, 0x67, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, - 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0e, 0x61, 0x76, 0x67, 0x52, - 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x61, 0x76, - 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, - 0x52, 0x0c, 0x61, 0x76, 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x12, 0x16, - 0x0a, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, - 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x22, 0x51, 0x0a, 0x0f, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, - 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x61, 0x72, 0x67, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x22, 0x24, 0x0a, 0x10, 0x53, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, - 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x22, - 0x42, 0x0a, 0x0c, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, - 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x79, - 0x74, 0x65, 0x73, 0x22, 0x38, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, - 0x75, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x57, 0x0a, - 0x0b, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x20, - 0x0a, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x05, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x22, 0x37, 0x0a, 0x11, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, - 0x64, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x41, - 0x76, 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x02, 0x52, 0x0c, 0x41, 0x76, 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22, - 0x73, 0x0a, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x75, 0x6e, 0x64, - 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x6f, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6c, 0x6f, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, - 0x70, 0x6f, 0x72, 0x74, 0x22, 0x3d, 0x0a, 0x0b, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x65, 0x72, 0x22, 0x42, 0x0a, 0x0c, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x9b, 0x01, 0x0a, + 0x15, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x61, 0x72, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, + 0x12, 0x16, 0x0a, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x69, 0x64, 0x73, + 0x18, 0x05, 0x20, 0x03, 0x28, 0x04, 0x52, 0x04, 0x75, 0x69, 0x64, 0x73, 0x12, 0x1c, 0x0a, 0x09, + 0x6e, 0x65, 0x77, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x09, 0x6e, 0x65, 0x77, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0xbc, 0x01, 0x0a, 0x10, 0x48, + 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x22, 0x0a, 0x0c, 0x72, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x72, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x61, + 0x73, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x64, 0x54, 0x61, 0x73, + 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x64, + 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x76, 0x67, 0x52, 0x75, 0x6e, 0x6e, + 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0e, 0x61, + 0x76, 0x67, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x22, 0x0a, + 0x0c, 0x61, 0x76, 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x02, 0x52, 0x0c, 0x61, 0x76, 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, + 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x22, 0x65, 0x0a, 0x0f, 0x53, 0x63, 0x68, + 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x61, 0x72, 0x67, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x12, 0x12, 0x0a, 0x04, + 0x75, 0x69, 0x64, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x04, 0x52, 0x04, 0x75, 0x69, 0x64, 0x73, + 0x22, 0x24, 0x0a, 0x10, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x22, 0x42, 0x0a, 0x0c, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x6f, + 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x38, 0x0a, 0x0a, 0x47, 0x65, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x22, 0x57, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x6f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x5e, 0x0a, 0x0a, 0x52, 0x75, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x61, - 0x72, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x12, - 0x16, 0x0a, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x22, 0x98, 0x01, 0x0a, 0x14, 0x57, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x28, 0x0a, 0x0f, 0x6e, 0x75, 0x6d, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x61, - 0x73, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x6e, 0x75, 0x6d, 0x52, 0x75, - 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x6e, 0x75, - 0x6d, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x0e, 0x6e, 0x75, 0x6d, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x54, 0x61, 0x73, - 0x6b, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x61, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x52, 0x75, 0x6e, - 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, 0x12, - 0x61, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x69, - 0x6d, 0x65, 0x22, 0x3d, 0x0a, 0x11, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x4f, 0x77, 0x6e, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x6f, 0x64, - 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, - 0x64, 0x22, 0x48, 0x0a, 0x16, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x1c, 0x0a, - 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x72, 0x22, 0x45, 0x0a, 0x17, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, 0x10, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, - 0x61, 0x74, 0x65, 0x6c, 0x79, 0x46, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x10, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x6c, 0x79, 0x46, 0x6f, 0x75, - 0x6e, 0x64, 0x22, 0x2c, 0x0a, 0x16, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x61, 0x72, 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, - 0x22, 0x45, 0x0a, 0x11, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x74, 0x65, - 0x54, 0x75, 0x70, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, 0x22, 0xba, 0x01, 0x0a, 0x17, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x54, - 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x16, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x42, 0x79, 0x74, 0x65, 0x54, 0x75, 0x70, 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x39, 0x0a, 0x0f, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x69, 0x61, - 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x46, 0x75, 0x6e, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x0e, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x46, 0x75, 0x6e, 0x63, 0x22, - 0x26, 0x0a, 0x10, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x04, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x4a, 0x0a, 0x0c, 0x46, 0x65, 0x74, 0x63, 0x68, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x63, - 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x0e, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x4e, 0x6f, 0x64, - 0x65, 0x49, 0x64, 0x22, 0x37, 0x0a, 0x0d, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, - 0x65, 0x64, 0x46, 0x75, 0x6e, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x73, 0x65, - 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x46, 0x75, 0x6e, 0x63, 0x32, 0x87, 0x01, 0x0a, - 0x0f, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, - 0x12, 0x3b, 0x0a, 0x08, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x72, - 0x61, 0x79, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, - 0x09, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x15, 0x2e, 0x72, 0x61, 0x79, - 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x49, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, + 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x22, 0x37, 0x0a, + 0x11, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x41, 0x76, 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, + 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0c, 0x41, 0x76, 0x67, 0x42, 0x61, 0x6e, + 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x22, 0x73, 0x0a, 0x15, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x46, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, + 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, + 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, + 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x3d, 0x0a, 0x0b, 0x43, + 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x72, 0x22, 0x42, 0x0a, 0x0c, 0x43, 0x6f, + 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x20, 0x0a, 0x0b, + 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x5e, + 0x0a, 0x0a, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6b, 0x77, 0x61, 0x72, 0x67, 0x73, 0x22, 0x98, + 0x01, 0x0a, 0x14, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x0f, 0x6e, 0x75, 0x6d, 0x52, 0x75, + 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x0f, 0x6e, 0x75, 0x6d, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x61, 0x73, 0x6b, + 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x6e, 0x75, 0x6d, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x54, 0x61, + 0x73, 0x6b, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x6e, 0x75, 0x6d, 0x51, 0x75, + 0x65, 0x75, 0x65, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x2e, 0x0a, 0x12, 0x61, 0x76, 0x65, + 0x72, 0x61, 0x67, 0x65, 0x52, 0x75, 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, 0x12, 0x61, 0x76, 0x65, 0x72, 0x61, 0x67, 0x65, 0x52, 0x75, + 0x6e, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x5d, 0x0a, 0x11, 0x4e, 0x6f, 0x74, + 0x69, 0x66, 0x79, 0x4f, 0x77, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, + 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, + 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x6f, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x6f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x48, 0x0a, 0x16, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x03, 0x75, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, + 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x65, 0x72, 0x22, 0x45, 0x0a, 0x17, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, 0x0a, + 0x10, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x6c, 0x79, 0x46, 0x6f, 0x75, 0x6e, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, + 0x74, 0x65, 0x6c, 0x79, 0x46, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0x2c, 0x0a, 0x16, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x04, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x22, 0x47, 0x0a, 0x11, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x54, 0x75, 0x70, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x04, 0x52, + 0x09, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x79, + 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x62, 0x79, 0x74, 0x65, 0x73, + 0x22, 0xba, 0x01, 0x0a, 0x17, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x09, + 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x2b, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x6c, 0x6f, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x54, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x72, 0x61, 0x79, + 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x54, 0x75, 0x70, + 0x6c, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x94, 0x01, + 0x0a, 0x10, 0x4c, 0x69, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x42, 0x0a, 0x09, 0x6c, 0x69, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x76, 0x65, + 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4c, 0x69, 0x76, + 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x6c, 0x69, 0x76, + 0x65, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x1a, 0x3c, 0x0a, 0x0e, 0x4c, 0x69, 0x76, 0x65, 0x4e, 0x6f, + 0x64, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3d, 0x0a, 0x11, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6e, + 0x6f, 0x64, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, + 0x65, 0x49, 0x64, 0x22, 0x39, 0x0a, 0x0f, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, + 0x69, 0x7a, 0x65, 0x64, 0x46, 0x75, 0x6e, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, + 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x46, 0x75, 0x6e, 0x63, 0x22, 0x26, + 0x0a, 0x10, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x4a, 0x0a, 0x0c, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x63, 0x61, + 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0e, 0x63, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x4e, 0x6f, 0x64, 0x65, + 0x49, 0x64, 0x22, 0x37, 0x0a, 0x0d, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, + 0x64, 0x46, 0x75, 0x6e, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x73, 0x65, 0x72, + 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x46, 0x75, 0x6e, 0x63, 0x32, 0x87, 0x01, 0x0a, 0x0f, + 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x12, + 0x3b, 0x0a, 0x08, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x1a, 0x2e, 0x72, 0x61, + 0x79, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x09, + 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x15, 0x2e, 0x72, 0x61, 0x79, 0x2e, + 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xbb, 0x01, 0x0a, 0x0e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x08, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x14, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x32, 0x96, 0x02, 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x62, 0x6a, 0x53, 0x74, - 0x6f, 0x72, 0x65, 0x12, 0x2f, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x11, 0x2e, 0x72, - 0x61, 0x79, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x4b, 0x69, 0x6c, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x0f, 0x2e, 0x72, 0x61, - 0x79, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x72, - 0x61, 0x79, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, - 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x75, 0x6e, 0x64, 0x12, - 0x1a, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, - 0x75, 0x6e, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x1a, 0x13, 0x2e, 0x72, 0x61, - 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x2b, 0x0a, 0x04, 0x43, 0x6f, 0x70, 0x79, 0x12, 0x10, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x43, - 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x72, 0x61, 0x79, - 0x2e, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, - 0x0c, 0x41, 0x76, 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x12, 0x13, 0x2e, + 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x0c, 0x52, 0x65, 0x76, + 0x69, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x13, + 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x32, 0x96, 0x02, 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x4f, 0x62, 0x6a, + 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x2f, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x11, + 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x28, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x0f, 0x2e, + 0x72, 0x61, 0x79, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, + 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x40, 0x0a, 0x0d, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x6f, 0x75, 0x6e, + 0x64, 0x12, 0x1a, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x46, 0x6f, 0x75, 0x6e, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x1a, 0x16, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, - 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x75, 0x0a, 0x06, 0x57, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x12, 0x2b, 0x0a, 0x03, 0x52, 0x75, 0x6e, 0x12, 0x0f, 0x2e, 0x72, 0x61, - 0x79, 0x2e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, - 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x19, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x57, 0x6f, 0x72, - 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x32, 0xe2, 0x01, 0x0a, 0x06, 0x47, 0x43, 0x53, 0x4f, 0x62, 0x6a, 0x12, 0x39, 0x0a, 0x0a, - 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x4f, 0x77, 0x6e, 0x73, 0x12, 0x16, 0x2e, 0x72, 0x61, 0x79, - 0x2e, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x4f, 0x77, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0f, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x2e, 0x72, 0x61, 0x79, - 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1b, 0x2e, 0x72, 0x61, - 0x79, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x7a, 0x0a, 0x07, 0x47, 0x43, 0x53, 0x46, 0x75, 0x6e, - 0x63, 0x12, 0x3b, 0x0a, 0x0c, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x46, 0x75, 0x6e, - 0x63, 0x12, 0x14, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x52, 0x65, - 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, - 0x0a, 0x09, 0x46, 0x65, 0x74, 0x63, 0x68, 0x46, 0x75, 0x6e, 0x63, 0x12, 0x11, 0x2e, 0x72, 0x61, - 0x79, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, - 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x72, 0x6f, 0x64, 0x72, 0x69, 0x67, 0x6f, 0x2d, 0x63, 0x61, 0x73, 0x74, 0x65, 0x6c, 0x6c, - 0x6f, 0x6e, 0x2f, 0x62, 0x61, 0x62, 0x79, 0x72, 0x61, 0x79, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, - 0x72, 0x70, 0x63, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x04, 0x43, 0x6f, 0x70, 0x79, 0x12, 0x10, 0x2e, 0x72, 0x61, 0x79, + 0x2e, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x11, 0x2e, 0x72, + 0x61, 0x79, 0x2e, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x3b, 0x0a, 0x0c, 0x41, 0x76, 0x67, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x12, + 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x16, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x42, 0x61, 0x6e, 0x64, 0x77, + 0x69, 0x64, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x75, 0x0a, 0x06, + 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x2b, 0x0a, 0x03, 0x52, 0x75, 0x6e, 0x12, 0x0f, 0x2e, + 0x72, 0x61, 0x79, 0x2e, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, + 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0c, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x19, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x57, + 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x32, 0xaa, 0x03, 0x0a, 0x06, 0x47, 0x43, 0x53, 0x4f, 0x62, 0x6a, 0x12, 0x39, + 0x0a, 0x0a, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x4f, 0x77, 0x6e, 0x73, 0x12, 0x16, 0x2e, 0x72, + 0x61, 0x79, 0x2e, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x4f, 0x77, 0x6e, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0f, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x2e, 0x72, + 0x61, 0x79, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x72, 0x61, 0x79, 0x2e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x4f, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1b, 0x2e, + 0x72, 0x61, 0x79, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x72, 0x61, 0x79, + 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x0f, 0x52, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x65, 0x72, 0x4c, 0x69, 0x6e, 0x65, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x2e, 0x72, 0x61, + 0x79, 0x2e, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x11, + 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x4c, 0x69, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, + 0x73, 0x12, 0x15, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x76, 0x65, 0x4e, 0x6f, 0x64, 0x65, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, + 0x12, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6e, 0x67, 0x12, 0x16, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x72, 0x61, + 0x79, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x32, 0x7a, 0x0a, 0x07, 0x47, 0x43, 0x53, 0x46, 0x75, 0x6e, 0x63, 0x12, 0x3b, 0x0a, 0x0c, 0x52, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x46, 0x75, 0x6e, 0x63, 0x12, 0x14, 0x2e, 0x72, 0x61, + 0x79, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x15, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x09, 0x46, 0x65, 0x74, 0x63, + 0x68, 0x46, 0x75, 0x6e, 0x63, 0x12, 0x11, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x46, 0x65, 0x74, 0x63, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x72, 0x61, 0x79, 0x2e, 0x46, + 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x34, 0x5a, 0x32, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x6f, 0x64, 0x72, 0x69, + 0x67, 0x6f, 0x2d, 0x63, 0x61, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x6f, 0x6e, 0x2f, 0x62, 0x61, 0x62, + 0x79, 0x72, 0x61, 0x79, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x3b, 0x67, 0x72, + 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1606,7 +1809,7 @@ func file_rayclient_proto_rawDescGZIP() []byte { return file_rayclient_proto_rawDescData } -var file_rayclient_proto_msgTypes = make([]protoimpl.MessageInfo, 25) +var file_rayclient_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_rayclient_proto_goTypes = []interface{}{ (*StatusResponse)(nil), // 0: ray.StatusResponse (*GlobalScheduleRequest)(nil), // 1: ray.GlobalScheduleRequest @@ -1628,50 +1831,64 @@ var file_rayclient_proto_goTypes = []interface{}{ (*ObjectLocationsRequest)(nil), // 17: ray.ObjectLocationsRequest (*LocationByteTuple)(nil), // 18: ray.LocationByteTuple (*ObjectLocationsResponse)(nil), // 19: ray.ObjectLocationsResponse - (*RegisterRequest)(nil), // 20: ray.RegisterRequest - (*RegisterResponse)(nil), // 21: ray.RegisterResponse - (*FetchRequest)(nil), // 22: ray.FetchRequest - (*FetchResponse)(nil), // 23: ray.FetchResponse - nil, // 24: ray.ObjectLocationsResponse.LocationsEntry + (*LiveNodesRequest)(nil), // 20: ray.LiveNodesRequest + (*GeneratingRequest)(nil), // 21: ray.GeneratingRequest + (*RegisterRequest)(nil), // 22: ray.RegisterRequest + (*RegisterResponse)(nil), // 23: ray.RegisterResponse + (*FetchRequest)(nil), // 24: ray.FetchRequest + (*FetchResponse)(nil), // 25: ray.FetchResponse + nil, // 26: ray.ObjectLocationsResponse.LocationsEntry + nil, // 27: ray.LiveNodesRequest.LiveNodesEntry } var file_rayclient_proto_depIdxs = []int32{ - 24, // 0: ray.ObjectLocationsResponse.locations:type_name -> ray.ObjectLocationsResponse.LocationsEntry - 18, // 1: ray.ObjectLocationsResponse.LocationsEntry.value:type_name -> ray.LocationByteTuple - 1, // 2: ray.GlobalScheduler.Schedule:input_type -> ray.GlobalScheduleRequest - 2, // 3: ray.GlobalScheduler.Heartbeat:input_type -> ray.HeartbeatRequest - 3, // 4: ray.LocalScheduler.Schedule:input_type -> ray.ScheduleRequest - 5, // 5: ray.LocalObjStore.Store:input_type -> ray.StoreRequest - 6, // 6: ray.LocalObjStore.Get:input_type -> ray.GetRequest - 9, // 7: ray.LocalObjStore.LocationFound:input_type -> ray.LocationFoundCallback - 10, // 8: ray.LocalObjStore.Copy:input_type -> ray.CopyRequest - 0, // 9: ray.LocalObjStore.AvgBandwidth:input_type -> ray.StatusResponse - 12, // 10: ray.Worker.Run:input_type -> ray.RunRequest - 0, // 11: ray.Worker.WorkerStatus:input_type -> ray.StatusResponse - 14, // 12: ray.GCSObj.NotifyOwns:input_type -> ray.NotifyOwnsRequest - 15, // 13: ray.GCSObj.RequestLocation:input_type -> ray.RequestLocationRequest - 17, // 14: ray.GCSObj.GetObjectLocations:input_type -> ray.ObjectLocationsRequest - 20, // 15: ray.GCSFunc.RegisterFunc:input_type -> ray.RegisterRequest - 22, // 16: ray.GCSFunc.FetchFunc:input_type -> ray.FetchRequest - 0, // 17: ray.GlobalScheduler.Schedule:output_type -> ray.StatusResponse - 0, // 18: ray.GlobalScheduler.Heartbeat:output_type -> ray.StatusResponse - 4, // 19: ray.LocalScheduler.Schedule:output_type -> ray.ScheduleResponse - 0, // 20: ray.LocalObjStore.Store:output_type -> ray.StatusResponse - 7, // 21: ray.LocalObjStore.Get:output_type -> ray.GetResponse - 0, // 22: ray.LocalObjStore.LocationFound:output_type -> ray.StatusResponse - 11, // 23: ray.LocalObjStore.Copy:output_type -> ray.CopyResponse - 8, // 24: ray.LocalObjStore.AvgBandwidth:output_type -> ray.BandwidthResponse - 0, // 25: ray.Worker.Run:output_type -> ray.StatusResponse - 13, // 26: ray.Worker.WorkerStatus:output_type -> ray.WorkerStatusResponse - 0, // 27: ray.GCSObj.NotifyOwns:output_type -> ray.StatusResponse - 16, // 28: ray.GCSObj.RequestLocation:output_type -> ray.RequestLocationResponse - 19, // 29: ray.GCSObj.GetObjectLocations:output_type -> ray.ObjectLocationsResponse - 21, // 30: ray.GCSFunc.RegisterFunc:output_type -> ray.RegisterResponse - 23, // 31: ray.GCSFunc.FetchFunc:output_type -> ray.FetchResponse - 17, // [17:32] is the sub-list for method output_type - 2, // [2:17] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 26, // 0: ray.ObjectLocationsResponse.locations:type_name -> ray.ObjectLocationsResponse.LocationsEntry + 27, // 1: ray.LiveNodesRequest.liveNodes:type_name -> ray.LiveNodesRequest.LiveNodesEntry + 18, // 2: ray.ObjectLocationsResponse.LocationsEntry.value:type_name -> ray.LocationByteTuple + 1, // 3: ray.GlobalScheduler.Schedule:input_type -> ray.GlobalScheduleRequest + 2, // 4: ray.GlobalScheduler.Heartbeat:input_type -> ray.HeartbeatRequest + 3, // 5: ray.LocalScheduler.Schedule:input_type -> ray.ScheduleRequest + 0, // 6: ray.LocalScheduler.KillServer:input_type -> ray.StatusResponse + 0, // 7: ray.LocalScheduler.ReviveServer:input_type -> ray.StatusResponse + 5, // 8: ray.LocalObjStore.Store:input_type -> ray.StoreRequest + 6, // 9: ray.LocalObjStore.Get:input_type -> ray.GetRequest + 9, // 10: ray.LocalObjStore.LocationFound:input_type -> ray.LocationFoundCallback + 10, // 11: ray.LocalObjStore.Copy:input_type -> ray.CopyRequest + 0, // 12: ray.LocalObjStore.AvgBandwidth:input_type -> ray.StatusResponse + 12, // 13: ray.Worker.Run:input_type -> ray.RunRequest + 0, // 14: ray.Worker.WorkerStatus:input_type -> ray.StatusResponse + 14, // 15: ray.GCSObj.NotifyOwns:input_type -> ray.NotifyOwnsRequest + 15, // 16: ray.GCSObj.RequestLocation:input_type -> ray.RequestLocationRequest + 17, // 17: ray.GCSObj.GetObjectLocations:input_type -> ray.ObjectLocationsRequest + 1, // 18: ray.GCSObj.RegisterLineage:input_type -> ray.GlobalScheduleRequest + 20, // 19: ray.GCSObj.RegisterLiveNodes:input_type -> ray.LiveNodesRequest + 21, // 20: ray.GCSObj.RegisterGenerating:input_type -> ray.GeneratingRequest + 22, // 21: ray.GCSFunc.RegisterFunc:input_type -> ray.RegisterRequest + 24, // 22: ray.GCSFunc.FetchFunc:input_type -> ray.FetchRequest + 0, // 23: ray.GlobalScheduler.Schedule:output_type -> ray.StatusResponse + 0, // 24: ray.GlobalScheduler.Heartbeat:output_type -> ray.StatusResponse + 4, // 25: ray.LocalScheduler.Schedule:output_type -> ray.ScheduleResponse + 0, // 26: ray.LocalScheduler.KillServer:output_type -> ray.StatusResponse + 0, // 27: ray.LocalScheduler.ReviveServer:output_type -> ray.StatusResponse + 0, // 28: ray.LocalObjStore.Store:output_type -> ray.StatusResponse + 7, // 29: ray.LocalObjStore.Get:output_type -> ray.GetResponse + 0, // 30: ray.LocalObjStore.LocationFound:output_type -> ray.StatusResponse + 11, // 31: ray.LocalObjStore.Copy:output_type -> ray.CopyResponse + 8, // 32: ray.LocalObjStore.AvgBandwidth:output_type -> ray.BandwidthResponse + 0, // 33: ray.Worker.Run:output_type -> ray.StatusResponse + 13, // 34: ray.Worker.WorkerStatus:output_type -> ray.WorkerStatusResponse + 0, // 35: ray.GCSObj.NotifyOwns:output_type -> ray.StatusResponse + 16, // 36: ray.GCSObj.RequestLocation:output_type -> ray.RequestLocationResponse + 19, // 37: ray.GCSObj.GetObjectLocations:output_type -> ray.ObjectLocationsResponse + 0, // 38: ray.GCSObj.RegisterLineage:output_type -> ray.StatusResponse + 0, // 39: ray.GCSObj.RegisterLiveNodes:output_type -> ray.StatusResponse + 0, // 40: ray.GCSObj.RegisterGenerating:output_type -> ray.StatusResponse + 23, // 41: ray.GCSFunc.RegisterFunc:output_type -> ray.RegisterResponse + 25, // 42: ray.GCSFunc.FetchFunc:output_type -> ray.FetchResponse + 23, // [23:43] is the sub-list for method output_type + 3, // [3:23] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_rayclient_proto_init() } @@ -1921,7 +2138,7 @@ func file_rayclient_proto_init() { } } file_rayclient_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RegisterRequest); i { + switch v := v.(*LiveNodesRequest); i { case 0: return &v.state case 1: @@ -1933,7 +2150,7 @@ func file_rayclient_proto_init() { } } file_rayclient_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RegisterResponse); i { + switch v := v.(*GeneratingRequest); i { case 0: return &v.state case 1: @@ -1945,7 +2162,7 @@ func file_rayclient_proto_init() { } } file_rayclient_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FetchRequest); i { + switch v := v.(*RegisterRequest); i { case 0: return &v.state case 1: @@ -1957,6 +2174,30 @@ func file_rayclient_proto_init() { } } file_rayclient_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RegisterResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_rayclient_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FetchRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_rayclient_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*FetchResponse); i { case 0: return &v.state @@ -1975,7 +2216,7 @@ func file_rayclient_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_rayclient_proto_rawDesc, NumEnums: 0, - NumMessages: 25, + NumMessages: 28, NumExtensions: 0, NumServices: 6, }, diff --git a/go/pkg/rayclient_grpc.pb.go b/go/pkg/rayclient_grpc.pb.go index 7171cd6..b4b691f 100644 --- a/go/pkg/rayclient_grpc.pb.go +++ b/go/pkg/rayclient_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 -// - protoc v3.19.1 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v3.6.1 // source: rayclient.proto package grpc @@ -15,8 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 const ( GlobalScheduler_Schedule_FullMethodName = "/ray.GlobalScheduler/Schedule" @@ -40,8 +40,9 @@ func NewGlobalSchedulerClient(cc grpc.ClientConnInterface) GlobalSchedulerClient } func (c *globalSchedulerClient) Schedule(ctx context.Context, in *GlobalScheduleRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(StatusResponse) - err := c.cc.Invoke(ctx, GlobalScheduler_Schedule_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, GlobalScheduler_Schedule_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -49,8 +50,9 @@ func (c *globalSchedulerClient) Schedule(ctx context.Context, in *GlobalSchedule } func (c *globalSchedulerClient) Heartbeat(ctx context.Context, in *HeartbeatRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(StatusResponse) - err := c.cc.Invoke(ctx, GlobalScheduler_Heartbeat_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, GlobalScheduler_Heartbeat_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -146,14 +148,20 @@ var GlobalScheduler_ServiceDesc = grpc.ServiceDesc{ } const ( - LocalScheduler_Schedule_FullMethodName = "/ray.LocalScheduler/Schedule" + LocalScheduler_Schedule_FullMethodName = "/ray.LocalScheduler/Schedule" + LocalScheduler_KillServer_FullMethodName = "/ray.LocalScheduler/KillServer" + LocalScheduler_ReviveServer_FullMethodName = "/ray.LocalScheduler/ReviveServer" ) // LocalSchedulerClient is the client API for LocalScheduler service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// START WORKER NODE LOCAL SCHEDULER type LocalSchedulerClient interface { Schedule(ctx context.Context, in *ScheduleRequest, opts ...grpc.CallOption) (*ScheduleResponse, error) + KillServer(ctx context.Context, in *StatusResponse, opts ...grpc.CallOption) (*StatusResponse, error) + ReviveServer(ctx context.Context, in *StatusResponse, opts ...grpc.CallOption) (*StatusResponse, error) } type localSchedulerClient struct { @@ -165,8 +173,29 @@ func NewLocalSchedulerClient(cc grpc.ClientConnInterface) LocalSchedulerClient { } func (c *localSchedulerClient) Schedule(ctx context.Context, in *ScheduleRequest, opts ...grpc.CallOption) (*ScheduleResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ScheduleResponse) - err := c.cc.Invoke(ctx, LocalScheduler_Schedule_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, LocalScheduler_Schedule_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *localSchedulerClient) KillServer(ctx context.Context, in *StatusResponse, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(StatusResponse) + err := c.cc.Invoke(ctx, LocalScheduler_KillServer_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *localSchedulerClient) ReviveServer(ctx context.Context, in *StatusResponse, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(StatusResponse) + err := c.cc.Invoke(ctx, LocalScheduler_ReviveServer_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -176,8 +205,12 @@ func (c *localSchedulerClient) Schedule(ctx context.Context, in *ScheduleRequest // LocalSchedulerServer is the server API for LocalScheduler service. // All implementations must embed UnimplementedLocalSchedulerServer // for forward compatibility +// +// START WORKER NODE LOCAL SCHEDULER type LocalSchedulerServer interface { Schedule(context.Context, *ScheduleRequest) (*ScheduleResponse, error) + KillServer(context.Context, *StatusResponse) (*StatusResponse, error) + ReviveServer(context.Context, *StatusResponse) (*StatusResponse, error) mustEmbedUnimplementedLocalSchedulerServer() } @@ -188,6 +221,12 @@ type UnimplementedLocalSchedulerServer struct { func (UnimplementedLocalSchedulerServer) Schedule(context.Context, *ScheduleRequest) (*ScheduleResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Schedule not implemented") } +func (UnimplementedLocalSchedulerServer) KillServer(context.Context, *StatusResponse) (*StatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method KillServer not implemented") +} +func (UnimplementedLocalSchedulerServer) ReviveServer(context.Context, *StatusResponse) (*StatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReviveServer not implemented") +} func (UnimplementedLocalSchedulerServer) mustEmbedUnimplementedLocalSchedulerServer() {} // UnsafeLocalSchedulerServer may be embedded to opt out of forward compatibility for this service. @@ -219,6 +258,42 @@ func _LocalScheduler_Schedule_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _LocalScheduler_KillServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StatusResponse) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LocalSchedulerServer).KillServer(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: LocalScheduler_KillServer_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LocalSchedulerServer).KillServer(ctx, req.(*StatusResponse)) + } + return interceptor(ctx, in, info, handler) +} + +func _LocalScheduler_ReviveServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StatusResponse) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LocalSchedulerServer).ReviveServer(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: LocalScheduler_ReviveServer_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LocalSchedulerServer).ReviveServer(ctx, req.(*StatusResponse)) + } + return interceptor(ctx, in, info, handler) +} + // LocalScheduler_ServiceDesc is the grpc.ServiceDesc for LocalScheduler service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -230,6 +305,14 @@ var LocalScheduler_ServiceDesc = grpc.ServiceDesc{ MethodName: "Schedule", Handler: _LocalScheduler_Schedule_Handler, }, + { + MethodName: "KillServer", + Handler: _LocalScheduler_KillServer_Handler, + }, + { + MethodName: "ReviveServer", + Handler: _LocalScheduler_ReviveServer_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "rayclient.proto", @@ -246,6 +329,8 @@ const ( // LocalObjStoreClient is the client API for LocalObjStore service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// START WORKER NODE LOCAL OBJECT STORE type LocalObjStoreClient interface { Store(ctx context.Context, in *StoreRequest, opts ...grpc.CallOption) (*StatusResponse, error) Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) @@ -263,8 +348,9 @@ func NewLocalObjStoreClient(cc grpc.ClientConnInterface) LocalObjStoreClient { } func (c *localObjStoreClient) Store(ctx context.Context, in *StoreRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(StatusResponse) - err := c.cc.Invoke(ctx, LocalObjStore_Store_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, LocalObjStore_Store_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -272,8 +358,9 @@ func (c *localObjStoreClient) Store(ctx context.Context, in *StoreRequest, opts } func (c *localObjStoreClient) Get(ctx context.Context, in *GetRequest, opts ...grpc.CallOption) (*GetResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetResponse) - err := c.cc.Invoke(ctx, LocalObjStore_Get_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, LocalObjStore_Get_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -281,8 +368,9 @@ func (c *localObjStoreClient) Get(ctx context.Context, in *GetRequest, opts ...g } func (c *localObjStoreClient) LocationFound(ctx context.Context, in *LocationFoundCallback, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(StatusResponse) - err := c.cc.Invoke(ctx, LocalObjStore_LocationFound_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, LocalObjStore_LocationFound_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -290,8 +378,9 @@ func (c *localObjStoreClient) LocationFound(ctx context.Context, in *LocationFou } func (c *localObjStoreClient) Copy(ctx context.Context, in *CopyRequest, opts ...grpc.CallOption) (*CopyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CopyResponse) - err := c.cc.Invoke(ctx, LocalObjStore_Copy_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, LocalObjStore_Copy_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -299,8 +388,9 @@ func (c *localObjStoreClient) Copy(ctx context.Context, in *CopyRequest, opts .. } func (c *localObjStoreClient) AvgBandwidth(ctx context.Context, in *StatusResponse, opts ...grpc.CallOption) (*BandwidthResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(BandwidthResponse) - err := c.cc.Invoke(ctx, LocalObjStore_AvgBandwidth_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, LocalObjStore_AvgBandwidth_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -310,6 +400,8 @@ func (c *localObjStoreClient) AvgBandwidth(ctx context.Context, in *StatusRespon // LocalObjStoreServer is the server API for LocalObjStore service. // All implementations must embed UnimplementedLocalObjStoreServer // for forward compatibility +// +// START WORKER NODE LOCAL OBJECT STORE type LocalObjStoreServer interface { Store(context.Context, *StoreRequest) (*StatusResponse, error) Get(context.Context, *GetRequest) (*GetResponse, error) @@ -481,6 +573,8 @@ const ( // WorkerClient is the client API for Worker service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// START WORKER SERVICE type WorkerClient interface { Run(ctx context.Context, in *RunRequest, opts ...grpc.CallOption) (*StatusResponse, error) WorkerStatus(ctx context.Context, in *StatusResponse, opts ...grpc.CallOption) (*WorkerStatusResponse, error) @@ -495,8 +589,9 @@ func NewWorkerClient(cc grpc.ClientConnInterface) WorkerClient { } func (c *workerClient) Run(ctx context.Context, in *RunRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(StatusResponse) - err := c.cc.Invoke(ctx, Worker_Run_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, Worker_Run_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -504,8 +599,9 @@ func (c *workerClient) Run(ctx context.Context, in *RunRequest, opts ...grpc.Cal } func (c *workerClient) WorkerStatus(ctx context.Context, in *StatusResponse, opts ...grpc.CallOption) (*WorkerStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(WorkerStatusResponse) - err := c.cc.Invoke(ctx, Worker_WorkerStatus_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, Worker_WorkerStatus_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -515,6 +611,8 @@ func (c *workerClient) WorkerStatus(ctx context.Context, in *StatusResponse, opt // WorkerServer is the server API for Worker service. // All implementations must embed UnimplementedWorkerServer // for forward compatibility +// +// START WORKER SERVICE type WorkerServer interface { Run(context.Context, *RunRequest) (*StatusResponse, error) WorkerStatus(context.Context, *StatusResponse) (*WorkerStatusResponse, error) @@ -604,15 +702,23 @@ const ( GCSObj_NotifyOwns_FullMethodName = "/ray.GCSObj/NotifyOwns" GCSObj_RequestLocation_FullMethodName = "/ray.GCSObj/RequestLocation" GCSObj_GetObjectLocations_FullMethodName = "/ray.GCSObj/GetObjectLocations" + GCSObj_RegisterLineage_FullMethodName = "/ray.GCSObj/RegisterLineage" + GCSObj_RegisterLiveNodes_FullMethodName = "/ray.GCSObj/RegisterLiveNodes" + GCSObj_RegisterGenerating_FullMethodName = "/ray.GCSObj/RegisterGenerating" ) // GCSObjClient is the client API for GCSObj service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// START GCS OBJECT TABLE type GCSObjClient interface { NotifyOwns(ctx context.Context, in *NotifyOwnsRequest, opts ...grpc.CallOption) (*StatusResponse, error) RequestLocation(ctx context.Context, in *RequestLocationRequest, opts ...grpc.CallOption) (*RequestLocationResponse, error) GetObjectLocations(ctx context.Context, in *ObjectLocationsRequest, opts ...grpc.CallOption) (*ObjectLocationsResponse, error) + RegisterLineage(ctx context.Context, in *GlobalScheduleRequest, opts ...grpc.CallOption) (*StatusResponse, error) + RegisterLiveNodes(ctx context.Context, in *LiveNodesRequest, opts ...grpc.CallOption) (*StatusResponse, error) + RegisterGenerating(ctx context.Context, in *GeneratingRequest, opts ...grpc.CallOption) (*StatusResponse, error) } type gCSObjClient struct { @@ -624,8 +730,9 @@ func NewGCSObjClient(cc grpc.ClientConnInterface) GCSObjClient { } func (c *gCSObjClient) NotifyOwns(ctx context.Context, in *NotifyOwnsRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(StatusResponse) - err := c.cc.Invoke(ctx, GCSObj_NotifyOwns_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, GCSObj_NotifyOwns_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -633,8 +740,9 @@ func (c *gCSObjClient) NotifyOwns(ctx context.Context, in *NotifyOwnsRequest, op } func (c *gCSObjClient) RequestLocation(ctx context.Context, in *RequestLocationRequest, opts ...grpc.CallOption) (*RequestLocationResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RequestLocationResponse) - err := c.cc.Invoke(ctx, GCSObj_RequestLocation_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, GCSObj_RequestLocation_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -642,8 +750,39 @@ func (c *gCSObjClient) RequestLocation(ctx context.Context, in *RequestLocationR } func (c *gCSObjClient) GetObjectLocations(ctx context.Context, in *ObjectLocationsRequest, opts ...grpc.CallOption) (*ObjectLocationsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ObjectLocationsResponse) - err := c.cc.Invoke(ctx, GCSObj_GetObjectLocations_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, GCSObj_GetObjectLocations_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *gCSObjClient) RegisterLineage(ctx context.Context, in *GlobalScheduleRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(StatusResponse) + err := c.cc.Invoke(ctx, GCSObj_RegisterLineage_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *gCSObjClient) RegisterLiveNodes(ctx context.Context, in *LiveNodesRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(StatusResponse) + err := c.cc.Invoke(ctx, GCSObj_RegisterLiveNodes_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *gCSObjClient) RegisterGenerating(ctx context.Context, in *GeneratingRequest, opts ...grpc.CallOption) (*StatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(StatusResponse) + err := c.cc.Invoke(ctx, GCSObj_RegisterGenerating_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -653,10 +792,15 @@ func (c *gCSObjClient) GetObjectLocations(ctx context.Context, in *ObjectLocatio // GCSObjServer is the server API for GCSObj service. // All implementations must embed UnimplementedGCSObjServer // for forward compatibility +// +// START GCS OBJECT TABLE type GCSObjServer interface { NotifyOwns(context.Context, *NotifyOwnsRequest) (*StatusResponse, error) RequestLocation(context.Context, *RequestLocationRequest) (*RequestLocationResponse, error) GetObjectLocations(context.Context, *ObjectLocationsRequest) (*ObjectLocationsResponse, error) + RegisterLineage(context.Context, *GlobalScheduleRequest) (*StatusResponse, error) + RegisterLiveNodes(context.Context, *LiveNodesRequest) (*StatusResponse, error) + RegisterGenerating(context.Context, *GeneratingRequest) (*StatusResponse, error) mustEmbedUnimplementedGCSObjServer() } @@ -673,6 +817,15 @@ func (UnimplementedGCSObjServer) RequestLocation(context.Context, *RequestLocati func (UnimplementedGCSObjServer) GetObjectLocations(context.Context, *ObjectLocationsRequest) (*ObjectLocationsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetObjectLocations not implemented") } +func (UnimplementedGCSObjServer) RegisterLineage(context.Context, *GlobalScheduleRequest) (*StatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RegisterLineage not implemented") +} +func (UnimplementedGCSObjServer) RegisterLiveNodes(context.Context, *LiveNodesRequest) (*StatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RegisterLiveNodes not implemented") +} +func (UnimplementedGCSObjServer) RegisterGenerating(context.Context, *GeneratingRequest) (*StatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RegisterGenerating not implemented") +} func (UnimplementedGCSObjServer) mustEmbedUnimplementedGCSObjServer() {} // UnsafeGCSObjServer may be embedded to opt out of forward compatibility for this service. @@ -740,6 +893,60 @@ func _GCSObj_GetObjectLocations_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _GCSObj_RegisterLineage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GlobalScheduleRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GCSObjServer).RegisterLineage(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GCSObj_RegisterLineage_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GCSObjServer).RegisterLineage(ctx, req.(*GlobalScheduleRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _GCSObj_RegisterLiveNodes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LiveNodesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GCSObjServer).RegisterLiveNodes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GCSObj_RegisterLiveNodes_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GCSObjServer).RegisterLiveNodes(ctx, req.(*LiveNodesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _GCSObj_RegisterGenerating_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GeneratingRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GCSObjServer).RegisterGenerating(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GCSObj_RegisterGenerating_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GCSObjServer).RegisterGenerating(ctx, req.(*GeneratingRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GCSObj_ServiceDesc is the grpc.ServiceDesc for GCSObj service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -759,6 +966,18 @@ var GCSObj_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetObjectLocations", Handler: _GCSObj_GetObjectLocations_Handler, }, + { + MethodName: "RegisterLineage", + Handler: _GCSObj_RegisterLineage_Handler, + }, + { + MethodName: "RegisterLiveNodes", + Handler: _GCSObj_RegisterLiveNodes_Handler, + }, + { + MethodName: "RegisterGenerating", + Handler: _GCSObj_RegisterGenerating_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "rayclient.proto", @@ -772,6 +991,8 @@ const ( // GCSFuncClient is the client API for GCSFunc service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// START GCS FUNCTION TABLE type GCSFuncClient interface { RegisterFunc(ctx context.Context, in *RegisterRequest, opts ...grpc.CallOption) (*RegisterResponse, error) FetchFunc(ctx context.Context, in *FetchRequest, opts ...grpc.CallOption) (*FetchResponse, error) @@ -786,8 +1007,9 @@ func NewGCSFuncClient(cc grpc.ClientConnInterface) GCSFuncClient { } func (c *gCSFuncClient) RegisterFunc(ctx context.Context, in *RegisterRequest, opts ...grpc.CallOption) (*RegisterResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RegisterResponse) - err := c.cc.Invoke(ctx, GCSFunc_RegisterFunc_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, GCSFunc_RegisterFunc_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -795,8 +1017,9 @@ func (c *gCSFuncClient) RegisterFunc(ctx context.Context, in *RegisterRequest, o } func (c *gCSFuncClient) FetchFunc(ctx context.Context, in *FetchRequest, opts ...grpc.CallOption) (*FetchResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(FetchResponse) - err := c.cc.Invoke(ctx, GCSFunc_FetchFunc_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, GCSFunc_FetchFunc_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -806,6 +1029,8 @@ func (c *gCSFuncClient) FetchFunc(ctx context.Context, in *FetchRequest, opts .. // GCSFuncServer is the server API for GCSFunc service. // All implementations must embed UnimplementedGCSFuncServer // for forward compatibility +// +// START GCS FUNCTION TABLE type GCSFuncServer interface { RegisterFunc(context.Context, *RegisterRequest) (*RegisterResponse, error) FetchFunc(context.Context, *FetchRequest) (*FetchResponse, error) diff --git a/proto/rayclient.proto b/proto/rayclient.proto index a343e70..c089b9f 100644 --- a/proto/rayclient.proto +++ b/proto/rayclient.proto @@ -22,6 +22,7 @@ message StatusResponse { service GlobalScheduler { rpc Schedule(GlobalScheduleRequest) returns (StatusResponse); rpc Heartbeat(HeartbeatRequest) returns (StatusResponse); + } message GlobalScheduleRequest { @@ -29,11 +30,12 @@ message GlobalScheduleRequest { uint64 name = 2; bytes args = 3; bytes kwargs = 4; - repeated uint64 uids = 5; + repeated uint64 uids = 5; + bool newObject = 6; // optional but in case the client wants to specify a node ID // to send a computation to - uint64 nodeId = 6; - bool localityFlag = 7; + uint64 nodeId = 7; + bool localityFlag = 8; } message HeartbeatRequest { @@ -45,11 +47,14 @@ message HeartbeatRequest { } + /* END GLOBAL SCHEDULER */ /* START WORKER NODE LOCAL SCHEDULER */ service LocalScheduler { rpc Schedule(ScheduleRequest) returns (ScheduleResponse); + rpc KillServer(StatusResponse) returns (StatusResponse); + rpc ReviveServer(StatusResponse) returns (StatusResponse); } message ScheduleRequest { @@ -130,6 +135,8 @@ message CopyResponse { service Worker { rpc Run(RunRequest) returns (StatusResponse); rpc WorkerStatus(StatusResponse) returns (WorkerStatusResponse); + rpc KillServer(StatusResponse) returns (StatusResponse); + rpc ReviveServer(StatusResponse) returns (StatusResponse); } message RunRequest { @@ -152,6 +159,9 @@ service GCSObj { rpc NotifyOwns(NotifyOwnsRequest) returns (StatusResponse); rpc RequestLocation(RequestLocationRequest) returns (RequestLocationResponse); rpc GetObjectLocations(ObjectLocationsRequest) returns (ObjectLocationsResponse); + rpc RegisterLineage(GlobalScheduleRequest) returns (StatusResponse); + rpc RegisterLiveNodes(LiveNodesRequest) returns (StatusResponse); + rpc RegisterGenerating(GeneratingRequest) returns (StatusResponse); } message NotifyOwnsRequest { @@ -178,12 +188,22 @@ message LocationByteTuple { message ObjectLocationsResponse { map locations = 1; } +message LiveNodesRequest { + map liveNodes = 1; +} + +message GeneratingRequest { + uint64 uid = 1; + uint64 nodeId = 2; +} + /* END GCS OBJECT TABLE */ /* START GCS FUNCTION TABLE */ service GCSFunc { rpc RegisterFunc(RegisterRequest) returns (RegisterResponse); rpc FetchFunc(FetchRequest) returns (FetchResponse); + } message RegisterRequest { diff --git a/python/babyray/__init__.py b/python/babyray/__init__.py index a8a00ff..b48e028 100644 --- a/python/babyray/__init__.py +++ b/python/babyray/__init__.py @@ -87,7 +87,7 @@ def get(self, copy=True, cache=True, pickle_load=True): rayclient_pb2.GetRequest(uid=self.uid, copy=True, cache=cache) ) - print(f"reading from shared memory: {self.uid}, {response.size}") + # print(f"reading from shared memory: {self.uid}, {response.size}") bytes_ = read_shared_memory(self.uid, response.size) @@ -146,6 +146,15 @@ def remote(self, *args, **kwargs): localityFlag=self.locality_flag, ) ).uid + + # uid = local_scheduler_gRPC.Schedule( + # rayclient_pb2.ScheduleRequest( + # name=self.name, + # args=pickle.dumps(args), + # kwargs=pickle.dumps(kwargs), + # uids=arg_uids, + # ) + # ).uid return Future(uid) def register(self): @@ -181,6 +190,11 @@ def get(futures, copy=True, cache=True, pickle_load=True): get(future, copy=copy, cache=cache, pickle_load=pickle_load) for future in futures ] + elif isinstance(futures, tuple): + return tuple( + get(future, copy=copy, cache=cache, pickle_load=pickle_load) + for future in futures + ) elif isinstance(futures, dict): return { k: get(future, copy=copy, cache=cache, pickle_load=pickle_load) @@ -194,6 +208,46 @@ def get(futures, copy=True, cache=True, pickle_load=True): ) +def kill_node(node_id): + # kill node node_id + + local_scheduler_channel = grpc.insecure_channel( + f"node{node_id}:{str(LOCAL_SCHEDULER_PORT)}" + ) + node_local_scheduler_gRPC = rayclient_pb2_grpc.LocalSchedulerStub( + local_scheduler_channel + ) + + worker_channel = grpc.insecure_channel( + f"node{node_id}:{str(LOCAL_WORKER_PORT_START)}" + ) + + node_worker_gRPC = rayclient_pb2_grpc.WorkerStub(worker_channel) + + node_local_scheduler_gRPC.KillServer(rayclient_pb2.StatusResponse()) + node_worker_gRPC.KillServer(rayclient_pb2.StatusResponse()) + + +def revive_node(node_id): + # revive node node_id + + local_scheduler_channel = grpc.insecure_channel( + f"node{node_id}:{str(LOCAL_SCHEDULER_PORT)}" + ) + node_local_scheduler_gRPC = rayclient_pb2_grpc.LocalSchedulerStub( + local_scheduler_channel + ) + + worker_channel = grpc.insecure_channel( + f"node{node_id}:{str(LOCAL_WORKER_PORT_START)}" + ) + + node_worker_gRPC = rayclient_pb2_grpc.WorkerStub(worker_channel) + + node_local_scheduler_gRPC.ReviveServer(rayclient_pb2.StatusResponse()) + node_worker_gRPC.ReviveServer(rayclient_pb2.StatusResponse()) + + def demo(): # Example function to simulate Ray's behavior @remote diff --git a/pythonserver/worker.py b/pythonserver/worker.py index e483518..858b7ef 100644 --- a/pythonserver/worker.py +++ b/pythonserver/worker.py @@ -1,26 +1,46 @@ -# server-side implementation for Python-based worker server - import grpc from concurrent import futures import time -import base64 -import subprocess import threading -import os from datetime import datetime import cloudpickle as pickle +import os from babyray import init, get, remote, Future from .constants import * - from . import rayclient_pb2 from . import rayclient_pb2_grpc MAX_CONCURRENT_TASKS = 10 EMA_PARAM = 0.9 + +class DebugSemaphore(threading.Semaphore): + def __init__(self, value=1): + super().__init__(value) + self._value = value + self._value_lock = threading.Lock() + + def acquire(self, blocking=True, timeout=None): + with self._value_lock: + if super().acquire(blocking, timeout): + self._value -= 1 + return True + return False + + def release(self): + with self._value_lock: + self._value += 1 + super().release() + + def get_value(self): + with self._value_lock: + return self._value + + # Global variables -semaphore = threading.Semaphore(MAX_CONCURRENT_TASKS) +# semaphore = threading.Semaphore(MAX_CONCURRENT_TASKS) +semaphore = DebugSemaphore(MAX_CONCURRENT_TASKS) num_running_tasks = 0 num_queued_tasks = 0 average_running_time = 0.1 @@ -28,8 +48,15 @@ MAX_MESSAGE_SIZE = 1024 * 1024 * 1024 # 1GB + +def block_forever(): + while True: + time.sleep(100) + + gcs_func_gRPC = None lobs_gRPC = None +server = None def local_log(*s): @@ -49,7 +76,6 @@ def init_all_stubs(): ("grpc.max_receive_message_length", MAX_MESSAGE_SIZE), ] - # so boilerplate-y gcs_func_channel = grpc.insecure_channel( f"node{GCS_NODE_ID}:{GCS_FUNCTION_TABLE_PORT}", options=channel_options, @@ -67,16 +93,24 @@ def init_all_stubs(): class WorkerServer(rayclient_pb2_grpc.WorkerServicer): def __init__(self): init() + self.alive = True def Run(self, request, context): global num_running_tasks, num_queued_tasks, average_running_time - local_log("in Run() rn") + if not self.alive: + block_forever() + + # local_log("in Run() rn") with mu: + # local_log("queueing up") num_queued_tasks += 1 + # local_log("the number of queued tasks is", num_queued_tasks) + # local_log("now waiting for the sema, whose value is:", semaphore.get_value()) with semaphore: + # local_log("got through the semaphore") with mu: num_queued_tasks -= 1 num_running_tasks += 1 @@ -91,21 +125,31 @@ def Run(self, request, context): args_obj = pickle.loads(request.args) kwargs_obj = pickle.loads(request.kwargs) + try: + local_log( + f"executing function \"{func_obj.__name__}\"on worker #{int(os.environ['NODE_ID']) - 1}" + ) + except Exception as e: + local_log("got exception:", e) + output = func_obj(*args_obj, **kwargs_obj) + if not self.alive: + block_forever() + output_pickled = pickle.dumps(output) - local_log("Executed!") + # local_log("Executed!") - local_log("gonna store now") + # local_log("gonna store now") lobs_gRPC.Store( rayclient_pb2.StoreRequest( uid=request.uid, objectBytes=output_pickled ) ) - local_log("stored...") + # local_log("stored...") running_time = time.time() - start - local_log(f"took this many seconds: {running_time:.3g}") + # local_log(f"took this many seconds: {running_time:.3g}") with mu: average_running_time = ( @@ -120,26 +164,35 @@ def Run(self, request, context): def WorkerStatus(self, request, context): global num_running_tasks, num_queued_tasks, average_running_time - # num_running_tasks, num_queued_tasks, average_running_time = 0, 0, 0.1 + if not self.alive: + block_forever() - # local_log("HERES THE WORKER STATUS......... GONNA TRY TO ACQUIRE THE THING") with mu: - # local_log("returning......>!!!!") return rayclient_pb2.WorkerStatusResponse( numRunningTasks=num_running_tasks, numQueuedTasks=num_queued_tasks, averageRunningTime=average_running_time, ) + def KillServer(self, request, context): + # global task_threads + local_log("GOT KILLED") + self.alive = False + return rayclient_pb2.StatusResponse(success=True) + + def ReviveServer(self, request, context): + # local_log("GOT REVIVED!") + self.alive = True + return rayclient_pb2.StatusResponse(success=True) + def serve(): + global server init_all_stubs() server = grpc.server( - # let's only start dropping requests 10x in futures.ThreadPoolExecutor(max_workers=10 * MAX_CONCURRENT_TASKS) ) - # server = grpc.server() rayclient_pb2_grpc.add_WorkerServicer_to_server(WorkerServer(), server) server_address = "0.0.0.0:50002" diff --git a/scripts/demo.py b/scripts/demo.py new file mode 100644 index 0000000..da13661 --- /dev/null +++ b/scripts/demo.py @@ -0,0 +1,94 @@ +from babyray import init, remote, get, kill_node +import time + +max_concurrent_tasks = 10 + +init() + + +@remote +def short_sleeper(): + time.sleep(10) + return 0 + + +@remote +def generate_big_object(n): + return bytearray(n) + + +@remote +def use_big_object(x): + return get(x) + + +@remote +def task_A(): + time.sleep(10) + return 0 + + +task_A.set_node(3) + + +@remote +def task_B(): + return get(task_A.remote()) + 1 + + +def bottom_up(): + # bottom up scheduling + + sleepers = [] + # submit sleeper tasks + _ = input("submit sleeper tasks [ENTER]:") + for _ in range(max_concurrent_tasks): + sleepers.append(short_sleeper.remote()) + + # submit sleeper tasks to other nodes + for _ in range(5): + _ = input("submit an extra sleeper task [ENTER]:") + sleepers.append(short_sleeper.remote()) + + get(sleepers) # wait on all of these to complete + + +def task_locality(): + # task locality + _ = input("Generate a big object on Worker 2 [ENTER]:") + + generate_big_object.set_node(3) + + x = generate_big_object.remote(2048) + + _ = input("Submit sleeper tasks to Worker 1 [ENTER]:") + + sleepers = [] + for _ in range(max_concurrent_tasks): + sleepers.append(short_sleeper.remote()) + + _ = input("Submit task that uses the big object [ENTER]") + use_big_object.remote(x) + + get(sleepers) + + +def fault_tolerance(): + # Fault Tolerance + _ = input("Run task B, which depends on task A [ENTER]:") + out_B = task_B.remote() + + _ = input("Kill task A's node [ENTER]:") + kill_node(3) + + print("output: ", get(out_B)) + + +def scripting(): + bottom_up() + task_locality() + fault_tolerance() + + +if __name__ == "__main__": + scripting() diff --git a/scripts/docker-compose.test.yml b/scripts/docker-compose.test.yml new file mode 100644 index 0000000..e69de29 diff --git a/scripts/fig10b.py b/scripts/fig10b.py index dfe3833..43a5100 100644 --- a/scripts/fig10b.py +++ b/scripts/fig10b.py @@ -55,26 +55,31 @@ def get_memory_usage(): i = 0 -while elapsed_time < total_duration and i < num_tasks: - # Submit a single task - f.remote() - i += 1 - - # Measure memory usage - current_time = time.time() - elapsed_time = current_time - start_time - - usage = 0 - if current_time >= next_log_time: - usage = get_memory_usage() - log(f"FIG10b: Memory usage without flush after {i} tasks: {usage} MB") - - # Schedule next log time - if elapsed_time < initial_period: - current_interval = initial_interval - else: - current_interval = min(second_interval, max_interval) - next_log_time = current_time + current_interval - - if usage > 2000: # Stop if memory usage exceeds 2 GB - break +old_way = True +if old_way: + while elapsed_time < total_duration and i < num_tasks: + # Submit a single task + f.remote() + i += 1 + + # Measure memory usage + current_time = time.time() + elapsed_time = current_time - start_time + + usage = 0 + if current_time >= next_log_time: + usage = get_memory_usage() + log(f"FIG10b: Memory usage without flush after {i} tasks: {usage} MB") + + # Schedule next log time + if elapsed_time < initial_period: + current_interval = initial_interval + else: + current_interval = min(second_interval, max_interval) + next_log_time = current_time + current_interval + + if usage > 2000: # Stop if memory usage exceeds 2 GB + break +else: + while True: + f.remote() \ No newline at end of file diff --git a/scripts/fig11.py b/scripts/fig11.py new file mode 100644 index 0000000..c80d7d3 --- /dev/null +++ b/scripts/fig11.py @@ -0,0 +1,61 @@ +import sys +from babyray import init, remote, get, Future, kill_node +import random +import time + +from utils import * + +# (pseudo)code for figure 11a + +init() + +DELAY = 0.1 # 100ms, per the original paper +BEGIN_INTERVAL = 50 # 50s, per the original paper +SECOND_INTERVAL = 40 # ~40s, per the original paper +THIRD_INTERVAL = 120 # ~120s, per the original paper + + +@remote +def firstfunc(): + # time.sleep(DELAY) + return 0 + + +def create_chain(n): + + funcs = [firstfunc] + for i in range(80): + + @remote + def func(): + time.sleep(DELAY) + out = get(funcs[-1].remote()) + print("out", out) + return out + 1 + + funcs.append(func) + + return funcs + + +funcs = create_chain(30) +funcs2 = create_chain(30) + +# probably need to manually define all of these because Python issues + + +out = funcs[-1].remote() +out2 = funcs2[-1].remote() + +output = get((out, out2)) + +print("the output was", output) +# out = func1.remote() +# keep eye on logs from here on out +# time.sleep(BEGIN_INTERVAL) +# kill_node(3) +# time.sleep(SECOND_INTERVAL) +# kill_node(4) +# time.sleep(THIRD_INTERVAL) +# revive_node(3) +# revive_node(4) diff --git a/scripts/fig8a.py b/scripts/fig8a.py index 74e480e..4e7bb7e 100644 --- a/scripts/fig8a.py +++ b/scripts/fig8a.py @@ -12,7 +12,7 @@ # worker 1: # - create object #1 -# - schedule 10 sleeper tasks +# - schedule MAX_CONCURRENT_TASKS sleeper tasks # - while True: time doing get(task(object #1)) or get(task(object #2)) # worker 2: # - create object #2 @@ -23,6 +23,7 @@ # 100KB to start # BYTEARR_SIZE = 100_000 BYTEARR_SIZE = 200_000_000 # 100_000_000 # 100 MB +MAX_CONCURRENT_TASKS = 50 # just create a local bytearray of size n @@ -47,7 +48,7 @@ def sleeper_func(): # sleeper functions are to ensure that any task we try to run will go to the # global scheduler first (instead of routed only locally) log("running sleeper functions") -for i in range(11): +for i in range(MAX_CONCURRENT_TASKS + 1): sleeper_func.remote() log("wtf", i) diff --git a/scripts/fig9.py b/scripts/fig9.py index 2bd9db8..1a3f5c6 100644 --- a/scripts/fig9.py +++ b/scripts/fig9.py @@ -58,6 +58,6 @@ def get_iops(size, max_time): iops = 1 / elapsed thpt = iops * size - log(f"RES:{size},{iops},{thpt}") + log(f"RES:{size},{elapsed},{iops},{thpt}") log("DONE!") diff --git a/scripts/run_fig11.sh b/scripts/run_fig11.sh new file mode 100755 index 0000000..e222490 --- /dev/null +++ b/scripts/run_fig11.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# scripts/run_tests.sh + +# Build and run the tests +# docker-compose build +docker-compose -f docker-compose.yml -f docker-compose.fig11.yml up + +# Wait for the services to be up and running +sleep 20 + +# Check the logs for the worker1 container to see if the tests passed +docker-compose logs worker1 | tee output.log + +# Check if "All tests passed" is in the logs +if grep -q "1 passed" output.log; then + echo "All tests passed!" + docker-compose down + exit 0 +else + echo "Some tests failed." + docker-compose down + exit 1 +fi \ No newline at end of file