Merge branch 'main' of https://github.com/Bedrock-OSS/regolith

src/filters.go

@@ -16,8 +16,8 @@ import (
 )
 
 type filterDefinition struct {
-	filter  func(filter Filter, settings map[string]interface{}, absoluteLocation string)
-	install func(filter Filter, path string)
+	filter  func(filter Filter, settings map[string]interface{}, absoluteLocation string, profile Profile)
+	install func(filter Filter, path string, profile Profile)
 	check   func()
 }
 
@@ -94,7 +94,7 @@ func RunProfile(profileName string) {
 	//now, we go through the filters!
 	for _, filter := range profile.Filters {
 		path, _ := filepath.Abs(".")
-		filter.RunFilter(path)
+		filter.RunFilter(path, profile)
 	}
 
 	//copy contents of .regolith/tmp to build
@@ -160,27 +160,27 @@ func ExportProject(profile Profile, name string) {
 }
 
 // RunFilter Runs the filter by selecting the correct filter type and running it
-func (filter *Filter) RunFilter(absoluteLocation string) {
+func (filter *Filter) RunFilter(absoluteLocation string, profile Profile) {
 	Logger.Infof("Running filter '%s'", filter.Name)
 	start := time.Now()
 
 	if filter.Url != "" {
-		RunRemoteFilter(filter.Url, filter.Settings, filter.Arguments)
+		RunRemoteFilter(filter.Url, filter.Settings, filter.Arguments, profile)
 	} else if filter.Filter != "" {
-		RunStandardFilter(*filter)
+		RunStandardFilter(*filter, profile)
 	} else {
 		if f, ok := FilterTypes[filter.RunWith]; ok {
-			f.filter(*filter, filter.Settings, absoluteLocation)
+			f.filter(*filter, filter.Settings, absoluteLocation, profile)
 		} else {
 			Logger.Warnf("Filter type '%s' not supported", filter.RunWith)
 		}
 		Logger.Info("Executed in ", time.Since(start))
 	}
 }
 
-func RunStandardFilter(filter Filter) {
+func RunStandardFilter(filter Filter, profile Profile) {
 	Logger.Infof("RunStandardFilter '%s'", filter.Filter)
-	RunRemoteFilter(FilterNameToUrl(filter.Filter), filter.Settings, filter.Arguments)
+	RunRemoteFilter(FilterNameToUrl(filter.Filter), filter.Settings, filter.Arguments, profile)
 }
 
 func LoadFiltersFromPath(path string) Profile {
@@ -199,7 +199,7 @@ func LoadFiltersFromPath(path string) Profile {
 	return result
 }
 
-func RunRemoteFilter(url string, settings map[string]interface{}, arguments []string) {
+func RunRemoteFilter(url string, settings map[string]interface{}, arguments []string, profile Profile) {
 	Logger.Infof("RunRemoteFilter '%s'", url)
 	if !IsRemoteFilterCached(url) {
 		Logger.Error("Filter is not downloaded! Please run 'regolith install'.")
@@ -212,7 +212,7 @@ func RunRemoteFilter(url string, settings map[string]interface{}, arguments []st
 		for k, v := range settings {
 			filter.Settings[k] = v
 		}
-		filter.RunFilter(absolutePath)
+		filter.RunFilter(absolutePath, profile)
 	}
 }
 

src/nodejs_filter.go

@@ -18,7 +18,7 @@ func RegisterNodeJSFilter(filters map[string]filterDefinition) {
 	}
 }
 
-func runNodeJSFilter(filter Filter, settings map[string]interface{}, absoluteLocation string) {
+func runNodeJSFilter(filter Filter, settings map[string]interface{}, absoluteLocation string, profile Profile) {
 	if len(settings) == 0 {
 		RunSubProcess("node", append([]string{absoluteLocation + string(os.PathSeparator) + filter.Location}, filter.Arguments...), GetAbsoluteWorkingDirectory())
 	} else {
@@ -27,7 +27,7 @@ func runNodeJSFilter(filter Filter, settings map[string]interface{}, absoluteLoc
 	}
 }
 
-func installNodeJSFilter(filter Filter, filterPath string) {
+func installNodeJSFilter(filter Filter, filterPath string, profile Profile) {
 	if hasPackageJson(filterPath) {
 		Logger.Info("Installing npm dependencies...")
 		RunSubProcess("npm", []string{"i", "--no-fund", "--no-audit"}, filterPath)

src/project.go

@@ -33,6 +33,7 @@ type Profile struct {
 	Unsafe       bool         `json:"unsafe"`
 	Filters      []Filter     `json:"filters"`
 	ExportTarget ExportTarget `json:"export"`
+	VenvPath     string       `json:"venv_path"`
 }
 
 type Filter struct {
@@ -91,7 +92,7 @@ func InitializeRegolithProject(isForced bool) bool {
 		}
 
 		// Delete old configuration
-		err := os.Remove(ManifestName)
+		os.Remove(ManifestName)
 
 		// Create new configuration
 		file, err := os.Create(ManifestName)

src/python_filter.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"runtime"
 	"strings"
 )
@@ -19,15 +20,15 @@ func RegisterPythonFilter(filters map[string]filterDefinition) {
 	}
 }
 
-func runPythonFilter(filter Filter, settings map[string]interface{}, absoluteLocation string) {
+func runPythonFilter(filter Filter, settings map[string]interface{}, absoluteLocation string, profile Profile) {
 	command := "python"
-	dir := path.Dir(absoluteLocation)
-	if needsVenv(dir) {
+	if needsVenv(absoluteLocation) {
+		venvPath := resolveVenvPath(profile, absoluteLocation)
 		suffix := ""
 		if runtime.GOOS == "windows" {
 			suffix = ".exe"
 		}
-		command = dir + "/venv/Scripts/python" + suffix
+		command = path.Join(venvPath, "Scripts/python"+suffix)
 	}
 	if len(settings) == 0 {
 		RunSubProcess(command, append([]string{"-u", absoluteLocation + string(os.PathSeparator) + filter.Location}, filter.Arguments...), GetAbsoluteWorkingDirectory())
@@ -37,16 +38,19 @@ func runPythonFilter(filter Filter, settings map[string]interface{}, absoluteLoc
 	}
 }
 
-func installPythonFilter(filter Filter, filterPath string) {
+func installPythonFilter(filter Filter, filterPath string, profile Profile) {
 	if needsVenv(filterPath) {
+		venvPath := resolveVenvPath(profile, filterPath)
 		Logger.Info("Creating venv...")
-		RunSubProcess("python", []string{"-m", "venv", "venv"}, filterPath)
+		RunSubProcess("python", []string{"-m", "venv", venvPath}, "")
 		suffix := ""
 		if runtime.GOOS == "windows" {
 			suffix = ".exe"
 		}
 		Logger.Info("Installing pip dependencies...")
-		RunSubProcess("venv/Scripts/pip"+suffix, []string{"install", "-r", "requirements.txt"}, filterPath)
+		RunSubProcess(
+			path.Join(venvPath, "Scripts/pip"+suffix),
+			[]string{"install", "-r", "requirements.txt"}, filterPath)
 	}
 }
 
@@ -58,6 +62,32 @@ func needsVenv(filterPath string) bool {
 	return false
 }
 
+func resolveVenvPath(profile Profile, filterPath string) string {
+	var resolvedPath string
+	var err error
+	if profile.VenvPath == "" {
+		// No path defined use filterPath
+		resolvedPath, err = filepath.Abs(path.Join(filterPath, "venv"))
+		if err != nil {
+			Logger.Fatal("Unable to resolve filter path: ", filterPath)
+		}
+		Logger.Debug("Using default venv_path: ", resolvedPath)
+	} else if filepath.IsAbs(profile.VenvPath) {
+		// path is absolute (don't change)
+		Logger.Debug("Using absolute venv_path: ", profile.VenvPath)
+		return profile.VenvPath
+	} else {
+		// non-absolute path put it into .regolith/venvs
+		resolvedPath, err = filepath.Abs(
+			path.Join(".regolith/venvs", profile.VenvPath))
+		if err != nil {
+			Logger.Fatal("Unable to resolve venv_path: ", profile.VenvPath)
+		}
+		Logger.Debug("Using resolved venv_path: ", profile.VenvPath)
+	}
+	return resolvedPath
+}
+
 func checkPythonRequirements() {
 	_, err := exec.LookPath("python")
 	if err != nil {

src/remote_filters.go

@@ -18,12 +18,8 @@ func FilterNameToUrl(name string) string {
 }
 
 func IsRemoteFilterCached(url string) bool {
-
 	_, err := os.Stat(UrlToPath(url))
-	if err != nil {
-		return false
-	}
-	return true
+	return err == nil
 }
 
 func DownloadFileTest() {
@@ -33,18 +29,16 @@ func DownloadFileTest() {
 	getter.Get("./.regolith/cache/test", fileUrl)
 }
 
-func GatherDependencies() []string {
-	project := LoadConfig()
+func GatherDependencies(profile Profile) []string {
 	var dependencies []string
-	for _, profile := range project.Profiles {
-		for _, filter := range profile.Filters {
-			if filter.Url != "" {
-				dependencies = append(dependencies, filter.Url)
-			}
 
-			if filter.Filter != "" {
-				dependencies = append(dependencies, FilterNameToUrl(filter.Filter))
-			}
+	for _, filter := range profile.Filters {
+		if filter.Url != "" {
+			dependencies = append(dependencies, filter.Url)
+		}
+
+		if filter.Filter != "" {
+			dependencies = append(dependencies, FilterNameToUrl(filter.Filter))
 		}
 	}
 	return dependencies
@@ -56,21 +50,29 @@ func InstallDependencies() {
 
 	err := os.MkdirAll(".regolith/cache", 0777)
 	if err != nil {
-		Logger.Fatal(fmt.Sprintf("Could not create .regolith/cache: "), err)
+		Logger.Fatal("Could not create .regolith/cache: ", err)
+	}
+	// Special path for virtual environments for python
+	err = os.MkdirAll(".regolith/venvs", 0777)
+	if err != nil {
+		Logger.Fatal("Could not create .regolith/venvs: ", err)
 	}
 
-	dependencies := GatherDependencies()
-	for _, dependency := range dependencies {
-		err := InstallDependency(dependency)
-		if err != nil {
-			Logger.Fatal(fmt.Sprintf("Could not install dependency %s: ", dependency), err)
+	project := LoadConfig()
+	for _, profile := range project.Profiles {
+		dependencies := GatherDependencies(profile)
+		for _, dependency := range dependencies {
+			err := InstallDependency(dependency, profile)
+			if err != nil {
+				Logger.Fatal(fmt.Sprintf("Could not install dependency %s: ", dependency), err)
+			}
 		}
 	}
 
 	Logger.Infof("Dependencies installed.")
 }
 
-func InstallDependency(url string) error {
+func InstallDependency(url string, profile Profile) error {
 	Logger.Infof("Installing dependency %s...", url)
 
 	// Download the filter into the cache folder
@@ -97,7 +99,7 @@ func InstallDependency(url string) error {
 	for _, filter := range result.Filters {
 		if filter.RunWith != "" {
 			if f, ok := FilterTypes[filter.RunWith]; ok {
-				f.install(filter, path)
+				f.install(filter, path, profile)
 			} else {
 				Logger.Warnf("Filter type '%s' not supported", filter.RunWith)
 			}

src/shell_filter.go

@@ -2,11 +2,12 @@ package src
 
 import (
 	"encoding/json"
-	"go.uber.org/zap"
 	"os"
 	"os/exec"
 	"strconv"
 	"strings"
+
+	"go.uber.org/zap"
 )
 
 const shellFilterName = "shell"
@@ -15,15 +16,13 @@ var shells = [][]string{{"powershell", "-command"}, {"cmd", "/k"}, {"bash", "-c"
 
 func RegisterShellFilter(filters map[string]filterDefinition) {
 	filters[shellFilterName] = filterDefinition{
-		filter: runShellFilter,
-		install: func(filter Filter, path string) {
-
-		},
-		check: checkShellRequirements,
+		filter:  runShellFilter,
+		install: func(filter Filter, path string, profile Profile) {},
+		check:   checkShellRequirements,
 	}
 }
 
-func runShellFilter(filter Filter, settings map[string]interface{}, absoluteLocation string) {
+func runShellFilter(filter Filter, settings map[string]interface{}, absoluteLocation string, profile Profile) {
 	if len(settings) == 0 {
 		executeCommand(filter.Command, filter.Arguments, absoluteLocation, GetAbsoluteWorkingDirectory())
 	} else {