diff --git a/docs/docs/.vitepress/config.ts b/docs/docs/.vitepress/config.ts index 8b90eeb1..5005a50b 100644 --- a/docs/docs/.vitepress/config.ts +++ b/docs/docs/.vitepress/config.ts @@ -126,6 +126,10 @@ export default defineConfig({ text: 'Profiles', link: '/guide/profiles' }, + { + text: 'Experiments', + link: '/guide/experiments' + }, { text: 'Safety', link: '/guide/safety' diff --git a/docs/docs/guide/experiments.md b/docs/docs/guide/experiments.md new file mode 100644 index 00000000..237271ea --- /dev/null +++ b/docs/docs/guide/experiments.md @@ -0,0 +1,23 @@ +--- +title: Experiments +--- + +# Experiments + +Experiments are new experimental features of Regolith to be released in the future versions, once proven to be stable and useful. The experiments can be enabled with the `--experiments` flag. + +## Currently Available Experiments + +### `size_time_check` + +The `size_time_check` is an experiment that aims to speed up `regolith run` and `regolith watch` commands. It achieves this by checking the size and modification time of the files before moving them between working and output directories. If the source file is the same size and has the same modification time as the destination file, the target file will remain untouched (Regolith assumes that the files are the same). + +The `size_time_check` should greatly speed up the exports of large projects. + +The downside of this approach is that on the first run, the export will be slower, but on subsequent runs, the export will be much faster. This means that the `size_time_check` is not recommended for CI where Regolith is run only once. + +Usage: +``` +regolith run --experiments size_time_check +regolith watch --experiments size_time_check +``` \ No newline at end of file diff --git a/main.go b/main.go index 21590b0b..254ed3f0 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "strings" "github.com/Bedrock-OSS/go-burrito/burrito" "github.com/stirante/go-simple-eval/eval" @@ -358,11 +359,22 @@ func main() { } subcommands = append(subcommands, cmdUpdateResolvers) - // add --debug and --timings flag to every command + // Generate the description for the experiments + experimentDescs := make([]string, len(regolith.AvailableExperiments)) + for i, experiment := range regolith.AvailableExperiments { + experimentDescs[i] = "- " + experiment.Name + " - " + strings.Trim(experiment.Description, "\n") + } + + // add --debug, --timings and --experiment flag to every command for _, cmd := range subcommands { cmd.Flags().BoolVarP(&burrito.PrintStackTrace, "debug", "", false, "Enables debugging") cmd.Flags().BoolVarP(®olith.EnableTimings, "timings", "", false, "Enables timing information") + cmd.Flags().StringSliceVar( + ®olith.EnabledExperiments, "experiments", nil, + "Enables experimental features. Currently supported experiments:\n"+ + strings.Join(experimentDescs, "\n")) } + // Build and run CLI rootCmd.AddCommand(subcommands...) rootCmd.Execute() diff --git a/regolith/experiments.go b/regolith/experiments.go new file mode 100644 index 00000000..5384ff84 --- /dev/null +++ b/regolith/experiments.go @@ -0,0 +1,40 @@ +package regolith + +type Experiment int + +const ( + // SizeTimeCheck is an experiment that checks the size and modification time when exporting + SizeTimeCheck Experiment = iota +) + +// The descriptions shouldn't be too wide, the text with their description is +// indented a lot. +const sizeTimeCheckDesc = ` +Activates optimization for file exporting by checking the size and +modification time of files before exporting, and only exporting if +the file has changed. This experiment applies to 'run' and 'watch' +commands. +` + +type ExperimentInfo struct { + Name string + Description string +} + +var AvailableExperiments = map[Experiment]ExperimentInfo{ + SizeTimeCheck: {"size_time_check", sizeTimeCheckDesc}, +} + +var EnabledExperiments []string + +func IsExperimentEnabled(exp Experiment) bool { + if EnabledExperiments == nil { + return false + } + for _, e := range EnabledExperiments { + if e == AvailableExperiments[exp].Name { + return true + } + } + return false +} diff --git a/regolith/export.go b/regolith/export.go index 36cdc5f8..593f0d61 100644 --- a/regolith/export.go +++ b/regolith/export.go @@ -141,6 +141,7 @@ func GetExportNames(exportTarget ExportTarget, ctx RunContext) (bpName string, r // ExportProject copies files from the tmp paths (tmp/BP and tmp/RP) into // the project's export target. The paths are generated with GetExportPaths. func ExportProject(ctx RunContext) error { + MeasureStart("Export - GetExportPaths") profile, err := ctx.GetProfile() if err != nil { return burrito.WrapError(err, runContextGetProfileError) @@ -159,6 +160,7 @@ func ExportProject(ctx RunContext) error { err, "Failed to get generate export paths.") } + MeasureStart("Export - LoadEditedFiles") // Loading edited_files.json or creating empty object editedFiles := LoadEditedFiles(dotRegolithPath) err = editedFiles.CheckDeletionSafety(rpPath, bpPath) @@ -174,22 +176,28 @@ func ExportProject(ctx RunContext) error { rpPath, bpPath) } - // Clearing output locations - // Spooky, I hope file protection works, and it won't do any damage - err = os.RemoveAll(bpPath) - if err != nil { - return burrito.WrapErrorf( - err, "Failed to clear behavior pack from build path %q.\n"+ - "Are user permissions correct?", bpPath) - } - err = os.RemoveAll(rpPath) - if err != nil { - return burrito.WrapErrorf( - err, "Failed to clear resource pack from build path %q.\n"+ - "Are user permissions correct?", rpPath) + MeasureStart("Export - Clean") + // When comparing the size and modification time of the files, we need to + // keep the files in target paths. + if !IsExperimentEnabled(SizeTimeCheck) { + // Clearing output locations + // Spooky, I hope file protection works, and it won't do any damage + err = os.RemoveAll(bpPath) + if err != nil { + return burrito.WrapErrorf( + err, "Failed to clear behavior pack from build path %q.\n"+ + "Are user permissions correct?", bpPath) + } + err = os.RemoveAll(rpPath) + if err != nil { + return burrito.WrapErrorf( + err, "Failed to clear resource pack from build path %q.\n"+ + "Are user permissions correct?", rpPath) + } } + MeasureEnd() // List the names of the filters that opt-in to the data export process - exportedFilterNames := []string{} + var exportedFilterNames []string for filter := range profile.Filters { filter := profile.Filters[filter] usingDataPath, err := filter.IsUsingDataExport(dotRegolithPath) @@ -228,6 +236,7 @@ func ExportProject(ctx RunContext) error { return burrito.WrapErrorf(err, osReadDirError, dataPath) } } + MeasureStart("Export - RevertibleOps") // Create revertible operations object backupPath := filepath.Join(dotRegolithPath, ".dataBackup") revertibleOps, err := NewRevertibleFsOperations(backupPath) @@ -235,6 +244,7 @@ func ExportProject(ctx RunContext) error { return burrito.WrapErrorf(err, newRevertibleFsOperationsError, backupPath) } // Export data + MeasureStart("Export - ExportData") for _, exportedFilterName := range exportedFilterNames { // Clear export target targetPath := filepath.Join(dataPath, exportedFilterName) @@ -278,18 +288,35 @@ func ExportProject(ctx RunContext) error { return mainError } } - // Export BP - Logger.Infof("Exporting behavior pack to \"%s\".", bpPath) - err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true) - if err != nil { - return burrito.WrapError(err, "Failed to export behavior pack.") - } - // Export RP - Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath)) - err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true) - if err != nil { - return burrito.WrapError(err, "Failed to export resource pack.") + MeasureStart("Export - MoveOrCopy") + if IsExperimentEnabled(SizeTimeCheck) { + // Export BP + Logger.Infof("Exporting behavior pack to \"%s\".", bpPath) + err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly) + if err != nil { + return burrito.WrapError(err, "Failed to export behavior pack.") + } + // Export RP + Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath)) + err = SyncDirectories(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly) + if err != nil { + return burrito.WrapError(err, "Failed to export resource pack.") + } + } else { + // Export BP + Logger.Infof("Exporting behavior pack to \"%s\".", bpPath) + err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/BP"), bpPath, exportTarget.ReadOnly, true) + if err != nil { + return burrito.WrapError(err, "Failed to export behavior pack.") + } + // Export RP + Logger.Infof("Exporting project to \"%s\".", filepath.Clean(rpPath)) + err = MoveOrCopy(filepath.Join(dotRegolithPath, "tmp/RP"), rpPath, exportTarget.ReadOnly, true) + if err != nil { + return burrito.WrapError(err, "Failed to export resource pack.") + } } + MeasureStart("Export - UpdateFromPaths") // Update or create edited_files.json err = editedFiles.UpdateFromPaths(rpPath, bpPath) if err != nil { @@ -306,6 +333,7 @@ func ExportProject(ctx RunContext) error { if err := revertibleOps.Close(); err != nil { return burrito.PassError(err) } + MeasureEnd() return nil } diff --git a/regolith/file_system.go b/regolith/file_system.go index 2c962771..765b5fca 100644 --- a/regolith/file_system.go +++ b/regolith/file_system.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/Bedrock-OSS/go-burrito/burrito" @@ -872,3 +873,130 @@ func MoveOrCopy( } return nil } + +// SyncDirectories copies the source to destination while checking size and modification time. +// If the file in the destination is different than the one in the source, it's overwritten, +// otherwise it's skipped (the destination file is not modified). +func SyncDirectories( + source string, destination string, makeReadOnly bool, +) error { + // Make destination parent if not exists + destinationParent := filepath.Dir(destination) + if err := os.MkdirAll(destinationParent, 0755); err != nil { + return burrito.WrapErrorf(err, osMkdirError, destinationParent) + } + err := filepath.Walk(source, func(srcPath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(source, srcPath) + if err != nil { + return burrito.WrapErrorf(err, filepathRelError, source, srcPath) + } + destPath := filepath.Join(destination, relPath) + + destInfo, err := os.Stat(destPath) + if err != nil && !os.IsNotExist(err) { + return burrito.WrapErrorf(err, osStatErrorAny, destPath) + } + if (err != nil && os.IsNotExist(err)) || info.ModTime() != destInfo.ModTime() || info.Size() != destInfo.Size() { + if info.IsDir() { + return os.MkdirAll(destPath, info.Mode()) + } + Logger.Debugf("SYNC: Copying file %s to %s", srcPath, destPath) + // If file exists, we need to remove it first to avoid permission issues when it's + // read-only + if destInfo != nil { + err = os.Remove(destPath) + if err != nil { + return burrito.WrapErrorf(err, osRemoveError, destPath) + } + } + return copyFile(srcPath, destPath, info) + } else { + Logger.Debugf("SYNC: Skipping file %s", srcPath) + } + return nil + }) + if err != nil { + return burrito.WrapErrorf(err, osCopyError, source, destination) + } + + // Remove files/folders in destination that are not in source + toRemoveList := make([]string, 0) + err = filepath.Walk(destination, func(destPath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + relPath, err := filepath.Rel(destination, destPath) + if err != nil { + return burrito.WrapErrorf(err, filepathRelError, destination, destPath) + } + srcPath := filepath.Join(source, relPath) + if _, err := os.Stat(srcPath); os.IsNotExist(err) { + // TODO: Not sure if this is the best way to do this + // The toRemoveList might get pretty big + if !SliceAny[string](toRemoveList, func(s string) bool { + return strings.HasPrefix(destPath, s) + }) { + Logger.Debugf("SYNC: Removing file %s", destPath) + // Add to list of files to remove, because otherwise walk function might fail + // when trying to walk a directory that doesn't exist anymore + toRemoveList = append(toRemoveList, destPath) + } + } + return nil + }) + + if err != nil { + return burrito.PassError(err) + } + + for _, path := range toRemoveList { + err = os.RemoveAll(path) + if err != nil { + return burrito.WrapErrorf(err, osRemoveError, path) + } + } + + // Make files read only if this option is selected + if makeReadOnly { + Logger.Infof("Changing the access for output path to "+ + "read-only.\n\tPath: %s", destination) + err := filepath.WalkDir(destination, + func(s string, d fs.DirEntry, e error) error { + + if e != nil { + // Error message isn't important as it's not passed further + // in the code + return e + } + if !d.IsDir() { + os.Chmod(s, 0444) + } + return nil + }) + if err != nil { + Logger.Warnf( + "Failed to change access of the output path to read-only.\n"+ + "\tPath: %s", + destination) + } + } + return nil +} + +func copyFile(src, dest string, info os.FileInfo) error { + data, err := os.ReadFile(src) + if err != nil { + return burrito.WrapErrorf(err, fileReadError, src) + } + if err = os.WriteFile(dest, data, info.Mode()); err != nil { + return burrito.WrapErrorf(err, fileWriteError, dest) + } + err = os.Chtimes(dest, time.Now(), info.ModTime()) + if err != nil { + return burrito.WrapErrorf(err, osChtimesError, dest) + } + return nil +} diff --git a/regolith/profile.go b/regolith/profile.go index 2a203075..d22b00f8 100644 --- a/regolith/profile.go +++ b/regolith/profile.go @@ -15,15 +15,18 @@ import ( // SetupTmpFiles set up the workspace for the filters. func SetupTmpFiles(config Config, dotRegolithPath string) error { start := time.Now() + useSizeTimeCheck := IsExperimentEnabled(SizeTimeCheck) // Setup Directories tmpPath := filepath.Join(dotRegolithPath, "tmp") - Logger.Debugf("Cleaning \"%s\"", tmpPath) - err := os.RemoveAll(tmpPath) - if err != nil { - return burrito.WrapErrorf(err, osRemoveError, tmpPath) + if !useSizeTimeCheck { + Logger.Debugf("Cleaning \"%s\"", tmpPath) + err := os.RemoveAll(tmpPath) + if err != nil { + return burrito.WrapErrorf(err, osRemoveError, tmpPath) + } } - err = os.MkdirAll(tmpPath, 0755) + err := os.MkdirAll(tmpPath, 0755) if err != nil { return burrito.WrapErrorf(err, osMkdirError, tmpPath) } @@ -36,6 +39,7 @@ func SetupTmpFiles(config Config, dotRegolithPath string) error { path, shortName, descriptiveName string, ) error { p := filepath.Join(tmpPath, shortName) + // A project don't have to have a RP or BP so path can be "" if path != "" { stats, err := os.Stat(path) if err != nil { @@ -48,12 +52,19 @@ func SetupTmpFiles(config Config, dotRegolithPath string) error { } } } else if stats.IsDir() { - err = copy.Copy( - path, - p, - copy.Options{PreserveTimes: false, Sync: false}) - if err != nil { - return burrito.WrapErrorf(err, osCopyError, path, p) + if useSizeTimeCheck { + err = SyncDirectories(path, p, false) + if err != nil { + return burrito.WrapError(err, "Failed to export behavior pack.") + } + } else { + err = copy.Copy( + path, + p, + copy.Options{PreserveTimes: false, Sync: false}) + if err != nil { + return burrito.WrapErrorf(err, osCopyError, path, p) + } } } else { // The folder paths leads to a file return burrito.WrappedErrorf(isDirNotADirError, path) diff --git a/regolith/utils.go b/regolith/utils.go index c07fc7fd..24a5dd11 100644 --- a/regolith/utils.go +++ b/regolith/utils.go @@ -430,3 +430,13 @@ func EscapePathPart(s string) string { } return sb.String() } + +// SliceAny returns true if any of the elements in the slice satisfy the predicate. +func SliceAny[T interface{}](slice []T, predicate func(T) bool) bool { + for _, item := range slice { + if predicate(item) { + return true + } + } + return false +} diff --git a/test/common.go b/test/common.go index af250bab..a83b44ff 100644 --- a/test/common.go +++ b/test/common.go @@ -77,6 +77,13 @@ const ( customPackNamePath = "testdata/custom_pack_name" dataModifyRemoteFilter = "testdata/data_modify_remote_filter" + + // sizeTimeCheckOptimizationPath contains two subdirectories 'project' and + // 'project_after_run'. The project is a Regolith project with a simple + // Python filter that generates some additional files.The + // 'project_after_run' is the same project but after running Regolith with + // the size_time_check experiment enabled. + sizeTimeCheckOptimizationPath = "testdata/size_time_check_optimization" ) // firstErr returns the first error in a list of errors. If the list is empty diff --git a/test/size_time_check_optimization_method_test.go b/test/size_time_check_optimization_method_test.go new file mode 100644 index 00000000..f5a74be8 --- /dev/null +++ b/test/size_time_check_optimization_method_test.go @@ -0,0 +1,112 @@ +package test + +import ( + "bytes" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Bedrock-OSS/regolith/regolith" +) + +// createBigFile creates a text file filled with a's with the given size in MB. +func createBigFile(sizeMB int, path string) error { + // Create the file + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(bytes.Repeat([]byte("a"), sizeMB*1024*1024)) + if err != nil { + return err + } + return nil +} + +// TestSizeTimeCheckOptimizationCorectness tests if running Regolith with the +// size_time_check experiment enabled exports the files correctly. +func TestSizeTimeCheckOptimizationCorectness(t *testing.T) { + // Switch to current working directory at the end of the test + defer os.Chdir(getWdOrFatal(t)) + + // TEST PREPARATION + t.Log("Clearing the testing directory...") + tmpDir := prepareTestDirectory("TestSizeTimeCheckOptimizationCorectness", t) + + t.Log("Copying the project files into the testing directory...") + project := absOrFatal(filepath.Join(sizeTimeCheckOptimizationPath, "project"), t) + copyFilesOrFatal(project, tmpDir, t) + + // Load abs path of the expected result and switch to the working directory + expectedBuildResult := absOrFatal( + filepath.Join(sizeTimeCheckOptimizationPath, "project_after_run"), t) + os.Chdir(tmpDir) + + // THE TEST + // Enable the experiment + regolith.EnabledExperiments = append(regolith.EnabledExperiments, "size_time_check") + + // Run the project + t.Log("Running Regolith...") + + if err := regolith.Run("default", true); err != nil { + t.Fatal("'regolith run' failed:", err.Error()) + } + // TEST EVALUATION + t.Log("Evaluating the result...") + comparePaths(expectedBuildResult, tmpDir, t) +} + +// TestSizeTimeCheckOptimizationSpeed tests if running Regolith with the +// size_time_check experiment enabled is faster on the second run (when the +// files are not changed in relation to the first run). +func TestSizeTimeCheckOptimizationSpeed(t *testing.T) { + // Switch to current working directory at the end of the test + defer os.Chdir(getWdOrFatal(t)) + + // TEST PREPARATION + t.Log("Clearing the testing directory...") + tmpDir := prepareTestDirectory("TestSizeTimeCheckOptimizationSpeed", t) + + t.Log("Copying the project files into the testing directory...") + project := absOrFatal(filepath.Join(sizeTimeCheckOptimizationPath, "project"), t) + copyFilesOrFatal(project, tmpDir, t) + + // Add a big file to the project 10 MB + t.Log("Creating a big file in the test project...") + bigFilePath := filepath.Join(tmpDir, "packs/BP/big_file.txt") + if err := createBigFile(10, bigFilePath); err != nil { + t.Fatalf("Creating a big file failed: %v", err) + } + os.Chdir(tmpDir) + + // THE TEST + // Enable the experiment + regolith.EnabledExperiments = append(regolith.EnabledExperiments, "size_time_check") + + // Run the project twice, the second run should be faster + runtimes := make([]time.Duration, 0) + for i := 0; i < 2; i++ { + // Run the project + t.Logf("Running Regolith for the %d. time...", i+1) + + // Start the timer + start := time.Now() + if err := regolith.Run("default", true); err != nil { + t.Fatal("'regolith run' failed:", err.Error()) + } + // Stop the timer + runtimes = append(runtimes, time.Since(start)) + } + // Check if the second run was faster. It should be because in the second + // run files are not copied (because they don't change). + if runtimes[0] < runtimes[1] { + t.Fatalf("The second run was slower than the first one: %v < %v", + runtimes[0], runtimes[1]) + } else { + t.Logf("The second run was faster than the first one: %v > %v", + runtimes[0], runtimes[1]) + } +} diff --git a/test/testdata/size_time_check_optimization/project/.gitignore b/test/testdata/size_time_check_optimization/project/.gitignore new file mode 100644 index 00000000..3f195ca9 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/.gitignore @@ -0,0 +1,2 @@ +/build +/.regolith \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project/.regolith/cache/venvs/.ignoreme b/test/testdata/size_time_check_optimization/project/.regolith/cache/venvs/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/.regolith/cache/venvs/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project/config.json b/test/testdata/size_time_check_optimization/project/config.json new file mode 100644 index 00000000..5ed12edc --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/config.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://raw.githubusercontent.com/Bedrock-OSS/regolith-schemas/main/config/v1.2.json", + "author": "Your name", + "name": "Project name", + "packs": { + "behaviorPack": "./packs/BP", + "resourcePack": "./packs/RP" + }, + "regolith": { + "dataPath": "./packs/data", + "filterDefinitions": { + "fill_with_data": { + "runWith": "python", + "script": "local_filters/fill_with_data.py" + } + }, + "profiles": { + "default": { + "export": { + "readOnly": true, + "target": "local", + "bpName": "'BP'", + "rpName": "'RP'" + }, + "filters": [ + { + "filter": "fill_with_data" + } + ] + } + } + } +} diff --git a/test/testdata/size_time_check_optimization/project/local_filters/fill_with_data.py b/test/testdata/size_time_check_optimization/project/local_filters/fill_with_data.py new file mode 100644 index 00000000..52774f24 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/local_filters/fill_with_data.py @@ -0,0 +1,16 @@ +''' +Fills the BP with data for testing. +''' + +def main(): + for i in range(1, 6): + with open( + f'BP/filter_generated_data_{i}.txt', + 'w', + encoding='utf8' + ) as f: + f.write( + f'This is file number {i} generated by fill_with_data.py\n') + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_1.txt b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_1.txt new file mode 100644 index 00000000..1938d622 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_1.txt @@ -0,0 +1 @@ +This is file number 1 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_2.txt b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_2.txt new file mode 100644 index 00000000..e380e385 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_2.txt @@ -0,0 +1 @@ +This is file number 2 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_3.txt b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_3.txt new file mode 100644 index 00000000..610d3d57 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_3.txt @@ -0,0 +1 @@ +This is file number 3 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_4.txt b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_4.txt new file mode 100644 index 00000000..2a4e1aeb --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_4.txt @@ -0,0 +1 @@ +This is file number 4 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_5.txt b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_5.txt new file mode 100644 index 00000000..837f1449 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/packs/BP/bp_file_5.txt @@ -0,0 +1 @@ +This is file number 5 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project/packs/RP/.ignoreme b/test/testdata/size_time_check_optimization/project/packs/RP/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/packs/RP/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project/packs/data/.ignoreme b/test/testdata/size_time_check_optimization/project/packs/data/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project/packs/data/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/.gitignore b/test/testdata/size_time_check_optimization/project_after_run/.gitignore new file mode 100644 index 00000000..3f195ca9 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.gitignore @@ -0,0 +1,2 @@ +/build +/.regolith \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/cache/edited_files.json b/test/testdata/size_time_check_optimization/project_after_run/.regolith/cache/edited_files.json new file mode 100644 index 00000000..7ab7ba53 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/cache/edited_files.json @@ -0,0 +1,21 @@ +{ + "rp": { + "build/RP/": [ + ".ignoreme" + ] + }, + "bp": { + "build/BP/": [ + "bp_file_1.txt", + "bp_file_2.txt", + "bp_file_3.txt", + "bp_file_4.txt", + "bp_file_5.txt", + "filter_generated_data_1.txt", + "filter_generated_data_2.txt", + "filter_generated_data_3.txt", + "filter_generated_data_4.txt", + "filter_generated_data_5.txt" + ] + } +} \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/cache/venvs/.ignoreme b/test/testdata/size_time_check_optimization/project_after_run/.regolith/cache/venvs/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/cache/venvs/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_1.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_1.txt new file mode 100644 index 00000000..1938d622 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_1.txt @@ -0,0 +1 @@ +This is file number 1 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_2.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_2.txt new file mode 100644 index 00000000..e380e385 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_2.txt @@ -0,0 +1 @@ +This is file number 2 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_3.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_3.txt new file mode 100644 index 00000000..610d3d57 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_3.txt @@ -0,0 +1 @@ +This is file number 3 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_4.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_4.txt new file mode 100644 index 00000000..2a4e1aeb --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_4.txt @@ -0,0 +1 @@ +This is file number 4 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_5.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_5.txt new file mode 100644 index 00000000..837f1449 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/bp_file_5.txt @@ -0,0 +1 @@ +This is file number 5 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_1.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_1.txt new file mode 100644 index 00000000..faa9f477 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_1.txt @@ -0,0 +1 @@ +This is file number 1 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_2.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_2.txt new file mode 100644 index 00000000..24a4b495 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_2.txt @@ -0,0 +1 @@ +This is file number 2 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_3.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_3.txt new file mode 100644 index 00000000..be2d60dc --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_3.txt @@ -0,0 +1 @@ +This is file number 3 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_4.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_4.txt new file mode 100644 index 00000000..0862fd62 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_4.txt @@ -0,0 +1 @@ +This is file number 4 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_5.txt b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_5.txt new file mode 100644 index 00000000..0c584edc --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/BP/filter_generated_data_5.txt @@ -0,0 +1 @@ +This is file number 5 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/RP/.ignoreme b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/RP/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/RP/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/data/.ignoreme b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/data/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/.regolith/tmp/data/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_1.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_1.txt new file mode 100644 index 00000000..1938d622 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_1.txt @@ -0,0 +1 @@ +This is file number 1 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_2.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_2.txt new file mode 100644 index 00000000..e380e385 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_2.txt @@ -0,0 +1 @@ +This is file number 2 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_3.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_3.txt new file mode 100644 index 00000000..610d3d57 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_3.txt @@ -0,0 +1 @@ +This is file number 3 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_4.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_4.txt new file mode 100644 index 00000000..2a4e1aeb --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_4.txt @@ -0,0 +1 @@ +This is file number 4 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_5.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_5.txt new file mode 100644 index 00000000..837f1449 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/bp_file_5.txt @@ -0,0 +1 @@ +This is file number 5 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_1.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_1.txt new file mode 100644 index 00000000..faa9f477 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_1.txt @@ -0,0 +1 @@ +This is file number 1 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_2.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_2.txt new file mode 100644 index 00000000..24a4b495 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_2.txt @@ -0,0 +1 @@ +This is file number 2 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_3.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_3.txt new file mode 100644 index 00000000..be2d60dc --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_3.txt @@ -0,0 +1 @@ +This is file number 3 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_4.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_4.txt new file mode 100644 index 00000000..0862fd62 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_4.txt @@ -0,0 +1 @@ +This is file number 4 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_5.txt b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_5.txt new file mode 100644 index 00000000..0c584edc --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/BP/filter_generated_data_5.txt @@ -0,0 +1 @@ +This is file number 5 generated by fill_with_data.py diff --git a/test/testdata/size_time_check_optimization/project_after_run/build/RP/.ignoreme b/test/testdata/size_time_check_optimization/project_after_run/build/RP/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/build/RP/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/config.json b/test/testdata/size_time_check_optimization/project_after_run/config.json new file mode 100644 index 00000000..5ed12edc --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/config.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://raw.githubusercontent.com/Bedrock-OSS/regolith-schemas/main/config/v1.2.json", + "author": "Your name", + "name": "Project name", + "packs": { + "behaviorPack": "./packs/BP", + "resourcePack": "./packs/RP" + }, + "regolith": { + "dataPath": "./packs/data", + "filterDefinitions": { + "fill_with_data": { + "runWith": "python", + "script": "local_filters/fill_with_data.py" + } + }, + "profiles": { + "default": { + "export": { + "readOnly": true, + "target": "local", + "bpName": "'BP'", + "rpName": "'RP'" + }, + "filters": [ + { + "filter": "fill_with_data" + } + ] + } + } + } +} diff --git a/test/testdata/size_time_check_optimization/project_after_run/local_filters/fill_with_data.py b/test/testdata/size_time_check_optimization/project_after_run/local_filters/fill_with_data.py new file mode 100644 index 00000000..52774f24 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/local_filters/fill_with_data.py @@ -0,0 +1,16 @@ +''' +Fills the BP with data for testing. +''' + +def main(): + for i in range(1, 6): + with open( + f'BP/filter_generated_data_{i}.txt', + 'w', + encoding='utf8' + ) as f: + f.write( + f'This is file number {i} generated by fill_with_data.py\n') + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_1.txt b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_1.txt new file mode 100644 index 00000000..1938d622 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_1.txt @@ -0,0 +1 @@ +This is file number 1 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_2.txt b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_2.txt new file mode 100644 index 00000000..e380e385 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_2.txt @@ -0,0 +1 @@ +This is file number 2 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_3.txt b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_3.txt new file mode 100644 index 00000000..610d3d57 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_3.txt @@ -0,0 +1 @@ +This is file number 3 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_4.txt b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_4.txt new file mode 100644 index 00000000..2a4e1aeb --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_4.txt @@ -0,0 +1 @@ +This is file number 4 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_5.txt b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_5.txt new file mode 100644 index 00000000..837f1449 --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/packs/BP/bp_file_5.txt @@ -0,0 +1 @@ +This is file number 5 from the source BP. diff --git a/test/testdata/size_time_check_optimization/project_after_run/packs/RP/.ignoreme b/test/testdata/size_time_check_optimization/project_after_run/packs/RP/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/packs/RP/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file diff --git a/test/testdata/size_time_check_optimization/project_after_run/packs/data/.ignoreme b/test/testdata/size_time_check_optimization/project_after_run/packs/data/.ignoreme new file mode 100644 index 00000000..de58919d --- /dev/null +++ b/test/testdata/size_time_check_optimization/project_after_run/packs/data/.ignoreme @@ -0,0 +1 @@ +This file is used for testing to simulate an empty directory because git doesn't allow saving empty directories. \ No newline at end of file