rodrigo-castellon · Russell-Tran · Jun 8, 2024 · May 30, 2024 · May 30, 2024 · May 30, 2024
diff --git a/.DS_Store b/.DS_Store
diff --git a/.gitignore b/.gitignore
@@ -198,3 +198,5 @@ go/cmd/gcsobjtable/gcsobjtable
 python/babyray/rayclient_pb2_grpc.py-e
 go/cmd/localobjstore/localobjstore
 go/cmd/localscheduler/localscheduler
+gcsobjtable.db
+.DS_Store
diff --git a/Dockerfile b/Dockerfile
@@ -4,8 +4,15 @@ ARG CONFIG=base
 # Base stage for common setup
 FROM golang as base
 
-RUN apt update
-RUN apt install -y protobuf-compiler
+# Install gcc and other necessary tools
+# we need gcc for Golang SQLite bc uses C code
+RUN apt-get update && \
+    apt-get install -y gcc && \ 
+    apt-get install -y protobuf-compiler && \
+    apt-get clean
+
+# Set CGO_ENABLED=1 , we need this for Golang SQLite bc uses C code
+ENV CGO_ENABLED=1
 
 # Set the Current Working Directory inside the container
 WORKDIR /app
@@ -28,6 +35,10 @@ RUN python3 -m pip install grpcio --break-system-packages
 
 RUN python3 -m pip install grpcio-tools --break-system-packages
 
+# For figure10b
+# RUN python3 -m pip install matplotlib
+
+# 
 RUN go install google.golang.org/protobuf/cmd/[email protected]
 RUN go install google.golang.org/grpc/cmd/[email protected]
 

diff --git a/Makefile b/Makefile
@@ -25,27 +25,27 @@ servers: gcsfunctable gcsobjtable globalscheduler localobjstore localscheduler w
 
 gcsfunctable:
 	@echo "Building GCS Function Table Server..."
-	cd go && go build -o bin/gcsfunctable cmd/gcsfunctable/main.go
+	cd go && go build -o bin/gcsfunctable cmd/gcsfunctable/*.go
 
 gcsobjtable:
 	@echo "Building GCS Object Table Server..."
-	cd go && go build -o bin/gcsobjtable cmd/gcsobjtable/main.go
+	cd go && go build -o bin/gcsobjtable cmd/gcsobjtable/*.go
 
 globalscheduler:
 	@echo "Building Global Scheduler Server..."
-	cd go && go build -o bin/globalscheduler cmd/globalscheduler/main.go
+	cd go && go build -o bin/globalscheduler cmd/globalscheduler/*.go
 
 localobjstore:
 	@echo "Building Local Object Store Server..."
-	cd go && go build -o bin/localobjstore cmd/localobjstore/main.go
+	cd go && go build -o bin/localobjstore cmd/localobjstore/*.go
 
 localscheduler:
 	@echo "Building Local Scheduler Server..."
-	cd go && go build -o bin/localscheduler cmd/localscheduler/main.go
+	cd go && go build -o bin/localscheduler cmd/localscheduler/*.go
 
 worker:
 	@echo "Building Worker Server..."
-	cd go && go build -o bin/worker cmd/worker/main.go
+	cd go && go build -o bin/worker cmd/worker/*.go
 
 docker:
 	docker build --build-arg CONFIG=base -t ray-node:base .

diff --git a/analysis/README.md b/analysis/README.md
@@ -0,0 +1,9 @@
+# analysis
+`output_10b_run1.txt`: Results of 10b run1, aka no caching, 2GB memory limit
+
+* After runs 1-5, realized we've been measuring memory on worker and not the GCS. need to change that!!
+
+* Run 6 is flush to disk every 1s using AOF and garbage collection
+* Run 7 is the new baseline, don't flush to disk.
+
+run 6 vs run 8 https://chatgpt.com/c/76bd6aef-f711-4067-b626-31446db56773
diff --git a/analysis/fig10b_graphs/attempt2_blurry.png b/analysis/fig10b_graphs/attempt2_blurry.png
diff --git a/analysis/fig10b_graphs/attempt2_sharp.png b/analysis/fig10b_graphs/attempt2_sharp.png
diff --git a/analysis/fig10b_graphs/attempt_Figure_1.png b/analysis/fig10b_graphs/attempt_Figure_1.png
diff --git a/analysis/fig10b_graphs/output a(1).png b/analysis/fig10b_graphs/output a(1).png
diff --git a/analysis/fig10b_graphs/outputa(2).png b/analysis/fig10b_graphs/outputa(2).png
diff --git a/analysis/fig10b_graphs/outputa.png b/analysis/fig10b_graphs/outputa.png
diff --git a/analysis/fig10b_graphs/run_6_vs_run8_1.png b/analysis/fig10b_graphs/run_6_vs_run8_1.png
diff --git a/analysis/fig10b_graphs/run_6_vs_run8_2.png b/analysis/fig10b_graphs/run_6_vs_run8_2.png
diff --git a/analysis/fig10b_graphs/run_6_vs_run8_3.png b/analysis/fig10b_graphs/run_6_vs_run8_3.png
diff --git a/analysis/output_10b_run1.txt b/analysis/output_10b_run1.txt
diff --git a/analysis/output_10b_run1_cleaned.csv b/analysis/output_10b_run1_cleaned.csv
@@ -0,0 +1,225 @@
+Timestamp,Memory Usage (MB)
+2024-06-04 01:44:08.529000,970.90234375
+2024-06-04 01:44:09.530000,977.0
+2024-06-04 01:44:10.533000,986.859375
+2024-06-04 01:44:11.533000,987.625
+2024-06-04 01:44:12.538000,989.30078125
+2024-06-04 01:44:14.551000,991.5859375
+2024-06-04 01:44:15.558000,991.37109375
+2024-06-04 01:44:18.565000,999.69140625
+2024-06-04 01:44:19.571000,999.6328125
+2024-06-04 01:44:21.582000,1001.9453125
+2024-06-04 01:44:23.603000,1002.921875
+2024-06-04 01:44:26.620000,1003.6015625
+2024-06-04 01:44:27.620000,1005.21875
+2024-06-04 01:44:28.620000,1004.91015625
+2024-06-04 01:44:32.635000,1010.0390625
+2024-06-04 01:44:34.655000,1013.94140625
+2024-06-04 01:44:38.734000,1018.734375
+2024-06-04 01:44:39.971000,1018.75390625
+2024-06-04 01:44:40.893000,1018.98828125
+2024-06-04 01:44:41.889000,1019.6015625
+2024-06-04 01:44:42.891000,1023.62109375
+2024-06-04 01:44:43.896000,1023.48046875
+2024-06-04 01:44:44.901000,1023.890625
+2024-06-04 01:44:45.907000,1024.66015625
+2024-06-04 01:44:46.908000,1025.2265625
+2024-06-04 01:44:47.913000,1024.94140625
+2024-06-04 01:44:49.917000,1026.1484375
+2024-06-04 01:44:50.929000,1025.8984375
+2024-06-04 01:44:53.946000,1022.25
+2024-06-04 01:44:54.955000,1021.984375
+2024-06-04 01:44:55.959000,1021.9375
+2024-06-04 01:44:56.961000,1022.0546875
+2024-06-04 01:44:57.967000,1022.19140625
+2024-06-04 01:45:00.974000,1021.69921875
+2024-06-04 01:45:02.982000,1021.86328125
+2024-06-04 01:45:03.990000,1021.4609375
+2024-06-04 01:45:06.003000,1022.73828125
+2024-06-04 01:45:10.023000,1026.3515625
+2024-06-04 01:45:11.028000,1025.765625
+2024-06-04 01:45:12.038000,1025.53125
+2024-06-04 01:45:14.047000,1026.4296875
+2024-06-04 01:45:15.051000,1026.5234375
+2024-06-04 01:45:17.055000,1026.36328125
+2024-06-04 01:45:18.063000,1026.046875
+2024-06-04 01:45:22.081000,1026.33203125
+2024-06-04 01:45:23.084000,1026.7421875
+2024-06-04 01:45:27.102000,1028.72265625
+2024-06-04 01:45:30.134000,1028.61328125
+2024-06-04 01:45:31.138000,1029.0859375
+2024-06-04 01:45:32.145000,1029.4609375
+2024-06-04 01:45:33.153000,1029.68359375
+2024-06-04 01:45:34.154000,1030.1015625
+2024-06-04 01:45:35.163000,1030.99609375
+2024-06-04 01:45:36.171000,1029.703125
+2024-06-04 01:45:37.176000,1029.75
+2024-06-04 01:45:39.180000,1029.24609375
+2024-06-04 01:45:42.189000,1030.2890625
+2024-06-04 01:45:45.206000,1031.359375
+2024-06-04 01:45:46.207000,1031.94921875
+2024-06-04 01:45:47.208000,1032.20703125
+2024-06-04 01:45:48.214000,1033.19921875
+2024-06-04 01:45:51.221000,1032.45703125
+2024-06-04 01:45:53.224000,1033.6171875
+2024-06-04 01:45:54.225000,1033.1796875
+2024-06-04 01:45:57.233000,1034.3515625
+2024-06-04 01:45:59.245000,1035.3203125
+2024-06-04 01:46:00.247000,1035.94140625
+2024-06-04 01:46:01.255000,1037.515625
+2024-06-04 01:46:10.290000,1048.42578125
+2024-06-04 01:46:12.298000,1049.0859375
+2024-06-04 01:46:14.308000,1045.19140625
+2024-06-04 01:46:17.324000,1049.62890625
+2024-06-04 01:46:18.332000,1050.609375
+2024-06-04 01:46:21.344000,1051.9765625
+2024-06-04 01:46:23.352000,1053.4765625
+2024-06-04 01:46:28.372000,1054.71484375
+2024-06-04 01:46:29.373000,1053.77734375
+2024-06-04 01:46:32.387000,1055.52734375
+2024-06-04 01:46:33.396000,1054.6484375
+2024-06-04 01:46:34.397000,1054.453125
+2024-06-04 01:46:37.404000,1056.26171875
+2024-06-04 01:46:38.409000,1056.53125
+2024-06-04 01:46:39.409000,1056.0234375
+2024-06-04 01:46:41.423000,1056.2890625
+2024-06-04 01:46:42.432000,1054.0234375
+2024-06-04 01:46:44.463000,1054.8125
+2024-06-04 01:46:46.468000,1057.4609375
+2024-06-04 01:46:47.471000,1057.5390625
+2024-06-04 01:46:50.486000,1058.125
+2024-06-04 01:46:52.494000,1057.8671875
+2024-06-04 01:46:54.505000,1058.33984375
+2024-06-04 01:46:55.507000,1058.59375
+2024-06-04 01:46:57.516000,1059.72265625
+2024-06-04 01:46:58.538000,1059.53515625
+2024-06-04 01:46:59.542000,1059.1328125
+2024-06-04 01:47:00.544000,1059.484375
+2024-06-04 01:47:03.555000,1059.3359375
+2024-06-04 01:47:05.560000,1059.09375
+2024-06-04 01:47:06.562000,1059.08984375
+2024-06-04 01:47:09.579000,1062.77734375
+2024-06-04 01:47:10.580000,1062.42578125
+2024-06-04 01:47:11.581000,1062.2421875
+2024-06-04 01:47:16.607000,1064.97265625
+2024-06-04 01:47:19.613000,1066.9296875
+2024-06-04 01:47:20.620000,1066.99609375
+2024-06-04 01:47:23.625000,1066.984375
+2024-06-04 01:47:25.638000,1068.578125
+2024-06-04 01:47:26.643000,1068.1796875
+2024-06-04 01:47:27.648000,1068.94921875
+2024-06-04 01:47:32.671000,1069.390625
+2024-06-04 01:47:34.678000,1069.671875
+2024-06-04 01:47:35.685000,1070.22265625
+2024-06-04 01:47:37.694000,1069.1328125
+2024-06-04 01:47:38.700000,1069.8671875
+2024-06-04 01:47:39.702000,1069.93359375
+2024-06-04 01:47:40.704000,1069.94140625
+2024-06-04 01:47:42.711000,1071.3828125
+2024-06-04 01:47:43.715000,1071.140625
+2024-06-04 01:47:44.721000,1071.86328125
+2024-06-04 01:47:47.732000,1077.11328125
+2024-06-04 01:47:49.743000,1078.8984375
+2024-06-04 01:47:51.755000,1083.46484375
+2024-06-04 01:47:52.759000,1083.05078125
+2024-06-04 01:47:53.762000,1084.11328125
+2024-06-04 01:47:54.767000,1084.171875
+2024-06-04 01:47:57.784000,1088.45703125
+2024-06-04 01:47:58.790000,1089.6484375
+2024-06-04 01:47:59.797000,1090.2265625
+2024-06-04 01:48:00.801000,1090.1796875
+2024-06-04 01:48:02.816000,1087.9765625
+2024-06-04 01:48:03.824000,1087.44921875
+2024-06-04 01:48:05.835000,1092.90625
+2024-06-04 01:48:06.837000,1091.5703125
+2024-06-04 01:48:10.853000,1094.234375
+2024-06-04 01:48:14.870000,1094.109375
+2024-06-04 01:48:18.895000,1095.3984375
+2024-06-04 01:48:19.895000,1095.86328125
+2024-06-04 01:48:21.905000,1096.32421875
+2024-06-04 01:48:22.909000,1096.24609375
+2024-06-04 01:48:23.915000,1096.22265625
+2024-06-04 01:48:24.923000,1096.70703125
+2024-06-04 01:48:25.930000,1096.33203125
+2024-06-04 01:48:26.931000,1097.38671875
+2024-06-04 01:48:28.934000,1097.703125
+2024-06-04 01:48:32.952000,1098.62890625
+2024-06-04 01:48:34.958000,1098.76953125
+2024-06-04 01:48:35.964000,1098.85546875
+2024-06-04 01:48:36.969000,1099.4921875
+2024-06-04 01:48:37.973000,1099.6171875
+2024-06-04 01:48:40.982000,1098.63671875
+2024-06-04 01:48:45.997000,1100.421875
+2024-06-04 01:48:48.006000,1101.375
+2024-06-04 01:48:52.017000,1102.9296875
+2024-06-04 01:48:54.028000,1103.86328125
+2024-06-04 01:48:55.033000,1103.33984375
+2024-06-04 01:48:58.043000,1101.92578125
+2024-06-04 01:49:01.050000,1103.71484375
+2024-06-04 01:49:03.060000,1110.6015625
+2024-06-04 01:49:04.067000,1110.89453125
+2024-06-04 01:50:38.096000,1130.8515625
+2024-06-04 01:51:08.099000,1138.8671875
+2024-06-04 01:51:38.103000,1166.4609375
+2024-06-04 01:52:08.104000,1178.38671875
+2024-06-04 01:52:38.107000,1180.4375
+2024-06-04 01:53:08.107000,1186.28125
+2024-06-04 01:54:38.123000,1221.234375
+2024-06-04 01:56:08.139000,1239.5703125
+2024-06-04 01:56:38.149000,1245.96875
+2024-06-04 01:57:08.156000,1254.7734375
+2024-06-04 02:00:08.183000,1348.3046875
+2024-06-04 02:00:38.186000,1345.56640625
+2024-06-04 02:01:08.189000,1353.29296875
+2024-06-04 02:03:38.214000,1388.85546875
+2024-06-04 02:04:08.223000,1432.5390625
+2024-06-04 02:06:08.241000,1430.30859375
+2024-06-04 02:06:38.246000,1432.46875
+2024-06-04 02:07:08.251000,1441.54296875
+2024-06-04 02:08:08.260000,1461.6875
+2024-06-04 02:09:08.269000,1540.4609375
+2024-06-04 02:10:38.280000,1518.2578125
+2024-06-04 02:11:08.283000,1508.953125
+2024-06-04 02:12:08.290000,1563.53515625
+2024-06-04 02:12:38.302000,1554.7265625
+2024-06-04 02:13:08.310000,1594.203125
+2024-06-04 02:14:38.328000,1715.3515625
+2024-06-04 02:15:08.334000,1722.95703125
+2024-06-04 02:15:38.336000,1720.2890625
+2024-06-04 02:17:08.358000,1663.44140625
+2024-06-04 02:17:38.359000,1677.34765625
+2024-06-04 02:18:08.360000,1678.1796875
+2024-06-04 02:19:08.373000,1754.8046875
+2024-06-04 02:19:38.376000,1681.375
+2024-06-04 02:20:08.378000,1684.84375
+2024-06-04 02:20:38.381000,1696.22265625
+2024-06-04 02:22:08.395000,1719.63671875
+2024-06-04 02:22:38.395000,1711.984375
+2024-06-04 02:23:08.396000,1719.796875
+2024-06-04 02:23:38.397000,1730.06640625
+2024-06-04 02:24:08.400000,1766.78125
+2024-06-04 02:24:38.405000,1775.64453125
+2024-06-04 02:25:38.411000,1788.42578125
+2024-06-04 02:26:08.413000,1801.94140625
+2024-06-04 02:26:38.417000,1809.4609375
+2024-06-04 02:27:08.419000,1816.6640625
+2024-06-04 02:28:08.422000,1795.0859375
+2024-06-04 02:28:38.427000,1802.6015625
+2024-06-04 02:29:38.432000,1847.20703125
+2024-06-04 02:30:38.438000,1859.59375
+2024-06-04 02:31:38.448000,1868.171875
+2024-06-04 02:33:08.458000,1830.6640625
+2024-06-04 02:34:08.465000,1883.7109375
+2024-06-04 02:34:38.467000,1893.63671875
+2024-06-04 02:35:08.474000,1901.78515625
+2024-06-04 02:35:38.479000,1912.9296875
+2024-06-04 02:36:08.481000,1932.51171875
+2024-06-04 02:36:38.487000,1939.3203125
+2024-06-04 02:37:08.491000,1898.3125
+2024-06-04 02:37:38.494000,1906.39453125
+2024-06-04 02:38:08.498000,1913.25
+2024-06-04 02:38:38.498000,1922.63671875
+2024-06-04 02:39:38.503000,1979.796875
+2024-06-04 02:40:08.505000,1974.71484375
+2024-06-04 02:41:08.514000,1992.9921875
+2024-06-04 02:41:38.516000,2002.17578125