From 6244f96d80700e6e4d232989c5e979430fbf1f5f Mon Sep 17 00:00:00 2001 From: Michael Maurer Date: Thu, 22 Jun 2023 13:43:51 -0400 Subject: [PATCH 1/3] Update 3pc -> parsec Modify string literals to contain parsec where 3pc is currently expected. Signed-off-by: Michael Maurer --- coordinator/sources/sources.go | 6 +++--- coordinator/testruns/commands.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/coordinator/sources/sources.go b/coordinator/sources/sources.go index 91b2149..5aeba15 100644 --- a/coordinator/sources/sources.go +++ b/coordinator/sources/sources.go @@ -249,7 +249,7 @@ func (s *SourcesManager) Compile( proxy_path := filepath.Join( sourcesDir(), "src", - "3pc", + "parsec", "agent", "runners", "evm", @@ -257,14 +257,14 @@ func (s *SourcesManager) Compile( ) if _, err := os.Stat(proxy_path); !os.IsNotExist(err) { logging.Infof( - "[Compile %s-%t]: Copying 3PC/EVM RPC proxy", + "[Compile %s-%t]: Copying parsec/EVM RPC proxy", hash, profilingOrDebugging, ) dest_proxy_path := filepath.Join( binariesPath, "src", - "3pc", + "parsec", "agent", "runners", "evm", diff --git a/coordinator/testruns/commands.go b/coordinator/testruns/commands.go index f59bb2f..0c05eff 100644 --- a/coordinator/testruns/commands.go +++ b/coordinator/testruns/commands.go @@ -30,10 +30,10 @@ var roleBinaries = map[common.SystemRole]string{ common.SystemRoleSentinelTwoPhase: "sources/build/src/uhs/twophase/sentinel_2pc/sentineld-2pc", common.SystemRoleAtomizerCliWatchtower: "sources/build/tools/bench/atomizer-cli-watchtower", common.SystemRoleTwoPhaseGen: "sources/build/tools/bench/twophase-gen", - common.SystemRoleAgent: "sources/build/src/3pc/agent/agentd", - common.SystemRoleRuntimeLockingShard: "sources/build/src/3pc/runtime_locking_shard/runtime_locking_shardd", - common.SystemRoleTicketMachine: "sources/build/src/3pc/ticket_machine/ticket_machined", - common.SystemRolePhaseTwoGen: "sources/build/tools/bench/3pc/evm/evm_bench", + common.SystemRoleAgent: "sources/build/src/parsec/agent/agentd", + common.SystemRoleRuntimeLockingShard: "sources/build/src/parsec/runtime_locking_shard/runtime_locking_shardd", + common.SystemRoleTicketMachine: "sources/build/src/parsec/ticket_machine/ticket_machined", + common.SystemRolePhaseTwoGen: "sources/build/tools/bench/parsec/evm/evm_bench", } // roleParameters is a map from the system role to the parameters we have to From a3e639af43f1d19342fc84ca8fb3d3dded621a7d Mon Sep 17 00:00:00 2001 From: Michael Maurer Date: Thu, 22 Jun 2023 14:02:39 -0400 Subject: [PATCH 2/3] Rename Phase Two -> Parsec Update variable names, comments where "Phase Two" is mentioned to reference PArSEC instead Signed-off-by: Michael Maurer --- common/normalizedtestrun.go | 6 ++--- common/system.go | 10 +++---- .../testruns/architecture_phase_two.go | 26 +++++++++---------- coordinator/testruns/commands.go | 8 +++--- coordinator/testruns/execute.go | 2 +- coordinator/testruns/parameters.go | 2 +- coordinator/testruns/roleconfig.go | 4 +-- coordinator/testruns/roles.go | 2 +- coordinator/testruns/transfer.go | 2 +- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/common/normalizedtestrun.go b/common/normalizedtestrun.go index 9fd838a..c8ff645 100644 --- a/common/normalizedtestrun.go +++ b/common/normalizedtestrun.go @@ -165,7 +165,7 @@ func (tr *TestRun) NormalizedConfigWithAgentData( cpu, SystemRoleAtomizerCliWatchtower, SystemRoleTwoPhaseGen, - SystemRolePhaseTwoGen, + SystemRoleParsecGen, ), ) trc.SentinelCPU = int( @@ -198,7 +198,7 @@ func (tr *TestRun) NormalizedConfigWithAgentData( ram, SystemRoleAtomizerCliWatchtower, SystemRoleTwoPhaseGen, - SystemRolePhaseTwoGen, + SystemRoleParsecGen, ) / 1024, ) trc.SentinelRAM = int( @@ -236,7 +236,7 @@ func (tr *TestRun) NormalizedConfigWithAgentData( count, SystemRoleAtomizerCliWatchtower, SystemRoleTwoPhaseGen, - SystemRolePhaseTwoGen, + SystemRoleParsecGen, ) trc.SentinelCount = getIntValueForAnyKey( count, diff --git a/common/system.go b/common/system.go index be67ea1..26c3f77 100644 --- a/common/system.go +++ b/common/system.go @@ -20,7 +20,7 @@ const SystemRoleTwoPhaseGen SystemRole = "twophase-gen" const SystemRoleAgent SystemRole = "agent" const SystemRoleRuntimeLockingShard SystemRole = "runtime_locking_shard" const SystemRoleTicketMachine SystemRole = "ticket_machine" -const SystemRolePhaseTwoGen SystemRole = "phasetwo_bench" +const SystemRoleParsecGen SystemRole = "parsec_bench" const SystemRoleLoadGen SystemRole = "loadgen" type SystemArchitectureRole struct { @@ -223,8 +223,8 @@ var AvailableArchitectures = []SystemArchitecture{ }, }, { - ID: "phase-two", - Name: "Phase Two Programmability", + ID: "parsec", + Name: "PArSEC", Roles: []SystemArchitectureRole{ { Role: SystemRoleAgent, @@ -242,7 +242,7 @@ var AvailableArchitectures = []SystemArchitecture{ ShortTitle: "Ticketer", }, { - Role: SystemRolePhaseTwoGen, + Role: SystemRoleParsecGen, Title: "Generator", ShortTitle: "Gen", }, @@ -261,7 +261,7 @@ var AvailableArchitectures = []SystemArchitecture{ RaftMaxBatch: 100000, SnapshotDistance: 0, ShardReplicationFactor: 3, - Architecture: "phase-two", + Architecture: "parsec", SampleCount: 315, LoadGenOutputCount: 2, LoadGenInputCount: 2, diff --git a/coordinator/testruns/architecture_phase_two.go b/coordinator/testruns/architecture_phase_two.go index 5734557..b573352 100644 --- a/coordinator/testruns/architecture_phase_two.go +++ b/coordinator/testruns/architecture_phase_two.go @@ -8,20 +8,20 @@ import ( "github.com/mit-dci/opencbdc-tctl/common" ) -func (t *TestRunManager) IsPhaseTwo(architectureID string) bool { - return strings.HasPrefix(architectureID, "phase-two") +func (t *TestRunManager) IsParsec(architectureID string) bool { + return strings.HasPrefix(architectureID, "parsec") } -// RunBinariesPhaseTwo will orchestrate the running of all roles for a full -// cycle test with the phase two architecture -func (t *TestRunManager) RunBinariesPhaseTwo( +// RunBinariesParsec will orchestrate the running of all roles for a full +// cycle test with the PArSEC architecture +func (t *TestRunManager) RunBinariesParsec( tr *common.TestRun, envs map[int32][]byte, cmd chan *common.ExecutedCommand, failures chan *common.ExecutedCommand, ) error { // Build the sequence of commands to start - startSequence := t.CreateStartSequencePhaseTwo(tr) + startSequence := t.CreateStartSequenceParsec(tr) // Execute the sequence of commands to start allCmds, terminated, err := t.executeStartSequence( @@ -76,7 +76,7 @@ func (t *TestRunManager) RunBinariesPhaseTwo( case <-time.After(timeout): } - err = t.CleanupCommandsPhaseTwo(tr, allCmds, envs) + err = t.CleanupCommandsParsec(tr, allCmds, envs) if err != nil { return err } @@ -87,7 +87,7 @@ func (t *TestRunManager) RunBinariesPhaseTwo( return nil } -func (t *TestRunManager) CleanupCommandsPhaseTwo( +func (t *TestRunManager) CleanupCommandsParsec( tr *common.TestRun, allCmds []runningCommand, envs map[int32][]byte, @@ -109,7 +109,7 @@ func (t *TestRunManager) CleanupCommandsPhaseTwo( t.WriteLog(tr, "Interrupting all loadgens") err := t.BreakAllCmds( tr, - t.FilterCommandsByRole(tr, allCmds, common.SystemRolePhaseTwoGen), + t.FilterCommandsByRole(tr, allCmds, common.SystemRoleParsecGen), ) if err != nil { return err @@ -132,7 +132,7 @@ func (t *TestRunManager) CleanupCommandsPhaseTwo( t.WriteLog(tr, "Terminating all loadgens") err = t.TerminateAllCmds( tr, - t.FilterCommandsByRole(tr, allCmds, common.SystemRolePhaseTwoGen), + t.FilterCommandsByRole(tr, allCmds, common.SystemRoleParsecGen), ) if err != nil { return err @@ -183,11 +183,11 @@ func (t *TestRunManager) CleanupCommandsPhaseTwo( return nil } -// CreateStartSequencePhaseTwo uses the test run configuration to determine in +// CreateStartSequenceParsec uses the test run configuration to determine in // which sequence the agent roles should be started, and returns an array of // startSequenceEntry elements that are ordered in the sequence in which they // should be started up. -func (t *TestRunManager) CreateStartSequencePhaseTwo( +func (t *TestRunManager) CreateStartSequenceParsec( tr *common.TestRun, ) []startSequenceEntry { // Determine the start sequence @@ -236,7 +236,7 @@ func (t *TestRunManager) CreateStartSequencePhaseTwo( // Start all load generators startSequence = append(startSequence, startSequenceEntry{ - roles: t.GetAllRolesSorted(tr, common.SystemRolePhaseTwoGen), + roles: t.GetAllRolesSorted(tr, common.SystemRoleParsecGen), timeout: roleStartTimeout, waitForPort: []PortIncrement{}, // Don't wait for anything - loadgens don't accept incoming }) diff --git a/coordinator/testruns/commands.go b/coordinator/testruns/commands.go index 0c05eff..c387e40 100644 --- a/coordinator/testruns/commands.go +++ b/coordinator/testruns/commands.go @@ -33,7 +33,7 @@ var roleBinaries = map[common.SystemRole]string{ common.SystemRoleAgent: "sources/build/src/parsec/agent/agentd", common.SystemRoleRuntimeLockingShard: "sources/build/src/parsec/runtime_locking_shard/runtime_locking_shardd", common.SystemRoleTicketMachine: "sources/build/src/parsec/ticket_machine/ticket_machined", - common.SystemRolePhaseTwoGen: "sources/build/tools/bench/parsec/evm/evm_bench", + common.SystemRoleParsecGen: "sources/build/tools/bench/parsec/evm/evm_bench", } // roleParameters is a map from the system role to the parameters we have to @@ -78,7 +78,7 @@ var roleParameters = map[common.SystemRole][]string{ "--loglevel=%LOGLEVEL%", "--component_id=%IDX%", }, - common.SystemRolePhaseTwoGen: []string{ + common.SystemRoleParsecGen: []string{ "--component_id=%IDX%", "--loadgen_accounts=%ACCOUNTS%", "--loadgen_agent_affinity=%LGAFFINITY%", @@ -373,8 +373,8 @@ func (t *TestRunManager) RunBinaries( return t.RunBinariesAtomizer(tr, envs, cmd, failures) } else if t.Is2PC(tr.Architecture) { return t.RunBinariesTwoPhase(tr, envs, cmd, failures) - } else if t.IsPhaseTwo(tr.Architecture) { - return t.RunBinariesPhaseTwo(tr, envs, cmd, failures) + } else if t.IsParsec(tr.Architecture) { + return t.RunBinariesParsec(tr, envs, cmd, failures) } return fmt.Errorf("unknown architecture: [%s]", tr.Architecture) } diff --git a/coordinator/testruns/execute.go b/coordinator/testruns/execute.go index 5a38457..3c2ecc4 100644 --- a/coordinator/testruns/execute.go +++ b/coordinator/testruns/execute.go @@ -85,7 +85,7 @@ func (t *TestRunManager) ExecuteTestRun(tr *common.TestRun) { } } - if !t.IsPhaseTwo(tr.Architecture) { + if !t.IsParsec(tr.Architecture) { // Generate the configuration file the system needs based on the // configured // parameters in the UI diff --git a/coordinator/testruns/parameters.go b/coordinator/testruns/parameters.go index f1e0ccb..feac43b 100644 --- a/coordinator/testruns/parameters.go +++ b/coordinator/testruns/parameters.go @@ -55,7 +55,7 @@ func (t *TestRunManager) SubstituteParameters( fmt.Sprintf("%d", tr.LoadGenAccounts), ) p = strings.ReplaceAll(p, "%LOGLEVEL%", t.RoleLogLevel(tr, r)) - if r.Role == common.SystemRolePhaseTwoGen { + if r.Role == common.SystemRoleParsecGen { p = strings.ReplaceAll(p, "%LGAFFINITY%", t.LoadGenAffinity(tr, r)) } newParams = append(newParams, p) diff --git a/coordinator/testruns/roleconfig.go b/coordinator/testruns/roleconfig.go index 6b77cf9..2648a5f 100644 --- a/coordinator/testruns/roleconfig.go +++ b/coordinator/testruns/roleconfig.go @@ -205,7 +205,7 @@ func (t *TestRunManager) writeTestRunConfigVariables( numRoles := t.countRoles(tr) numClis := 0 numClis += numRoles[common.SystemRoleAtomizerCliWatchtower] - numClis += numRoles[common.SystemRolePhaseTwoGen] + numClis += numRoles[common.SystemRoleParsecGen] numClis += numRoles[common.SystemRoleTwoPhaseGen] // Calculate target per role @@ -355,7 +355,7 @@ func (t *TestRunManager) RoleTelLevel( tellevel = tr.WatchtowerTelemetryLevel case common.SystemRoleCoordinator: tellevel = tr.CoordinatorTelemetryLevel - case common.SystemRolePhaseTwoGen: + case common.SystemRoleParsecGen: fallthrough case common.SystemRoleTwoPhaseGen: fallthrough diff --git a/coordinator/testruns/roles.go b/coordinator/testruns/roles.go index d8c734b..b879c27 100644 --- a/coordinator/testruns/roles.go +++ b/coordinator/testruns/roles.go @@ -40,7 +40,7 @@ func (t *TestRunManager) NormalizeRole( role = common.SystemRoleShard } else if role == common.SystemRoleSentinelTwoPhase { role = common.SystemRoleSentinel - } else if role == common.SystemRoleAtomizerCliWatchtower || role == common.SystemRolePhaseTwoGen || role == common.SystemRoleTwoPhaseGen { + } else if role == common.SystemRoleAtomizerCliWatchtower || role == common.SystemRoleParsecGen || role == common.SystemRoleTwoPhaseGen { role = common.SystemRoleLoadGen } return role diff --git a/coordinator/testruns/transfer.go b/coordinator/testruns/transfer.go index 4f245de..24496db 100644 --- a/coordinator/testruns/transfer.go +++ b/coordinator/testruns/transfer.go @@ -61,7 +61,7 @@ var copyFiles = map[common.SystemRole][]string{ "tps_target_%IDX%.txt%%OPT", "telemetry.bin%%OPT", }, - common.SystemRolePhaseTwoGen: { + common.SystemRoleParsecGen: { "tx_samples_%IDX%.txt", "telemetry.bin%%OPT", }, From 17f62b79bba645bae0a462ab5dbd4ad3055c9d2f Mon Sep 17 00:00:00 2001 From: Michael Maurer Date: Thu, 29 Jun 2023 16:47:20 -0400 Subject: [PATCH 3/3] Trimmed samples no longer affect statistics Calculation of statistics now omits the first TRIM_SAMPLES samples Signed-off-by: Michael Maurer --- coordinator/scripts/calculate_results.py | 45 +++++++++++++----------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/coordinator/scripts/calculate_results.py b/coordinator/scripts/calculate_results.py index 186ad08..5e25d06 100644 --- a/coordinator/scripts/calculate_results.py +++ b/coordinator/scripts/calculate_results.py @@ -209,6 +209,10 @@ def process_lats(lats): df['latsS'] = df.lats / 10**3 df['pDate'] = df.time.values.astype('datetime64[ns]') df = df[df.time > 1609459200000] # Filter out (corrupt) times before 2021 + # trim first samples from dataframe + if 'TRIM_SAMPLES' in environ: + trim_samples = int(environ['TRIM_SAMPLES']) + df = df[df.time > df[0][0] + trim_samples*(block_time_ms / 1000)*one_sec] dat = df.groupby(by=MyBinnerTime(expression=df.pDate, resolution='s', df=df, label='pDate'), agg={'count': 'count', 'lats': vaex.agg.list('lats')}) dat['lats'] = dat['lats'].apply(process_lats) @@ -236,7 +240,7 @@ def process_lats(lats): lat_99999.append(tps_its[2][1][idx][2]) idx += 1 current += datetime.timedelta(seconds=1) - + dat = df.groupby(df.latsS, agg='count') lats = dat.values lat_max = df.max(df.latsS) @@ -262,11 +266,11 @@ def process_lats(lats): lat_lines.append({"lats":lat_99999, "title":"99.999%", "freq": 1}) periods = [] - + idx = 0 tps_target_files = [join('outputs',x) for x in listdir('outputs') \ if 'tps_target_' in x and 'hdf5' not in x] - if len(tps_target_files) > 0: + if len(tps_target_files) > 0: t_index = pandas.date_range(start=begin- datetime.timedelta(seconds=5), end=end, freq='1s') exports = 0 for f in tps_target_files: @@ -285,7 +289,7 @@ def process_lats(lats): exports = exports + 1 else: print('{} has no rows', f) - + if exports > 0: df2 = vaex.open('outputs/*-tps_target_*.txt.hdf5') df2['pDate'] = df2['index'] @@ -296,11 +300,11 @@ def process_lats(lats): dat3.drop('tps_target', inplace=True) dat3.drop('pDate', inplace=True) dat2.join(dat3, inplace=True) - + its = dat2.to_items() tps_target = make_tps_target_series_line(its, begin, end, (lambda its,idx: its[0][1][idx].astype(datetime.datetime)), (lambda its,idx: its[1][1][idx])) periods = extract_tps_target_periods(its, begin, end, (lambda its,idx: its[0][1][idx].astype(datetime.datetime)), (lambda its,idx: its[1][1][idx]), (lambda its,idx: its[3][1][idx])) - + tps_lines.append({"tps":tps_target, "title":"Loadgen target", "freq": 1, "ma": False}) prev_lat99 = 0 @@ -326,6 +330,13 @@ def process_lats(lats): elbow_latmean.append(prev_latmean) elbow_lat99.append(prev_lat99) elbow_lat99999.append(prev_lat99999) + elif 'TRIM_SAMPLES' in environ : + ## Lob off (configurable) more "warm up" samples + trim_samples = int(environ['TRIM_SAMPLES']) + for i in range(len(tps_lines)): + tps_lines[i]["tps"] = tps_lines[i]["tps"][trim_samples:] + for i in range(len(lat_lines)): + lat_lines[i]["lats"] = lat_lines[i]["lats"][trim_samples:] if archiver_based: for output_file in output_files: @@ -356,13 +367,7 @@ def process_lats(lats): while len(lat_lines[i]["lats"]) > 0 and int(lat_lines[i]["lats"][-1]) == 0: lat_lines[i]["lats"].pop() -## Lob off (configurable) more "warm up" samples -if 'TRIM_SAMPLES' in environ: - trim_samples = int(environ['TRIM_SAMPLES']) - for i in range(len(tps_lines)): - tps_lines[i]["tps"] = tps_lines[i]["tps"][trim_samples:] - for i in range(len(lat_lines)): - lat_lines[i]["lats"] = lat_lines[i]["lats"][trim_samples:] + ## Create throughput histogram @@ -480,7 +485,7 @@ def dev_to_val(dev): if len(colors) > j: color = colors[j] ax.plot(tps_time, tps_ma, label='{} ({}ms MA)'.format(tps_line["title"],tps_ma_ms), color=color) - + max = max * 1.02 @@ -572,7 +577,7 @@ def dev_to_val(dev): if len(markers) > i: marker = markers[i] ax.plot(elbow_tps, yy, label=titles[i], color=color, marker=marker) - + max = max * 1.02 ax.set_ylabel('Latency (ms)') @@ -584,13 +589,13 @@ def dev_to_val(dev): # TODO: Find proper way of finding peak TPS range. None of this is working # accurately # for yy in y: - # delta_ma_tmp = [] + # delta_ma_tmp = [] # pf_x = x # pf_y = yy # while math.isnan(pf_y[-1]): # pf_y = pf_y[:-1] # pf_x = pf_x[:-1] - + # while math.isnan(pf_y[1]): # pf_y = pf_y[1:] # pf_x = pf_x[1:] @@ -625,16 +630,16 @@ def dev_to_val(dev): # peak_lb_idx = 0 # if peak_ub_idx < 0: # peak_ub_idx = 0 - + # peak_lb = pf_x[peak_lb_idx] # peak_ub = pf_x[peak_ub_idx] # peak_found = True # if delta_ma_above < 8: # peak_found = False - + # if peak_found: # break - + # if peak_ub > 0: # ax.set_title('Latency/Throughput Elbow\nDetected peak {}-{} TX/s'.format(peak_lb, peak_ub))