diff --git a/cmd/flags.go b/cmd/flags.go index 7c9e702d6..a30c728a7 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -52,6 +52,7 @@ const ( qadisablecliFlag = "qa-disable-cli" qaportFlag = "qa-port" planProgressPortFlag = "plan-progress-port" + maxCloneSizeBytesFlag = "max-clone-size" transformerSelectorFlag = "transformer-selector" qaEnabledCategoriesFlag = "qa-enable" qaDisabledCategoriesFlag = "qa-disable" diff --git a/cmd/plan.go b/cmd/plan.go index f7eea7920..85d637344 100644 --- a/cmd/plan.go +++ b/cmd/plan.go @@ -36,6 +36,7 @@ import ( ) type planFlags struct { + maxVCSRepoCloneSize int64 progressServerPort int planfile string srcpath string @@ -65,6 +66,8 @@ func planHandler(cmd *cobra.Command, flags planFlags) { }() defer lib.Destroy() + vcs.SetMaxRepoCloneSize(flags.maxVCSRepoCloneSize) + var err error planfile := flags.planfile srcpath := flags.srcpath @@ -182,6 +185,7 @@ func GetPlanCommand() *cobra.Command { planCmd.Flags().StringSliceVar(&flags.preSets, preSetFlag, []string{}, "Specify preset config to use.") planCmd.Flags().StringArrayVar(&flags.setconfigs, setConfigFlag, []string{}, "Specify config key-value pairs.") planCmd.Flags().IntVar(&flags.progressServerPort, planProgressPortFlag, 0, "Port for the plan progress server. If not provided, the server won't be started.") + planCmd.Flags().Int64Var(&flags.maxVCSRepoCloneSize, maxCloneSizeBytesFlag, -1, "Max size in bytes when cloning a git repo. Default -1 is infinite") planCmd.Flags().BoolVar(&flags.disableLocalExecution, common.DisableLocalExecutionFlag, false, "Allow files to be executed locally.") planCmd.Flags().BoolVar(&flags.failOnEmptyPlan, common.FailOnEmptyPlan, false, "If true, planning will exit with a failure exit code if no services are detected (and no default transformers are found).") diff --git a/cmd/transform.go b/cmd/transform.go index 11448af2b..152f266d2 100644 --- a/cmd/transform.go +++ b/cmd/transform.go @@ -38,6 +38,8 @@ import ( type transformFlags struct { qaflags + // maxVCSRepoCloneSize is the maximum size in bytes for cloning repos + maxVCSRepoCloneSize int64 // ignoreEnv tells us whether to use data collected from the local machine ignoreEnv bool // disableLocalExecution disables execution of executables locally @@ -72,6 +74,7 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) { } defer pprof.StopCPUProfile() } + vcs.SetMaxRepoCloneSize(flags.maxVCSRepoCloneSize) ctx, cancel := context.WithCancel(cmd.Context()) logrus.AddHook(common.NewCleanupHook(cancel)) @@ -250,7 +253,14 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) { } startQA(flags.qaflags) } - if err := lib.Transform(ctx, transformationPlan, preExistingPlan, flags.outpath, flags.transformerSelector, flags.maxIterations); err != nil { + if err := lib.Transform( + ctx, + transformationPlan, + preExistingPlan, + flags.outpath, + flags.transformerSelector, + flags.maxIterations, + ); err != nil { logrus.Fatalf("failed to transform. Error: %q", err) } logrus.Infof("Transformed target artifacts can be found at [%s].", flags.outpath) @@ -290,6 +300,7 @@ func GetTransformCommand() *cobra.Command { transformCmd.Flags().StringVarP(&flags.customizationsPath, customizationsFlag, "c", "", "Specify directory or a git url (see https://move2kube.konveyor.io/concepts/git-support) where customizations are stored. By default we look for "+common.DefaultCustomizationDir) transformCmd.Flags().StringVarP(&flags.transformerSelector, transformerSelectorFlag, "t", "", "Specify the transformer selector.") transformCmd.Flags().BoolVar(&flags.qaskip, qaSkipFlag, false, "Enable/disable the default answers to questions posed in QA Cli sub-system. If disabled, you will have to answer the questions posed by QA during interaction.") + transformCmd.Flags().Int64Var(&flags.maxVCSRepoCloneSize, maxCloneSizeBytesFlag, -1, "Max size in bytes when cloning a git repo. Default -1 is infinite") // QA options transformCmd.Flags().StringSliceVar(&flags.qaEnabledCategories, qaEnabledCategoriesFlag, []string{}, "Specify the QA categories to enable (cannot be used in conjunction with qa-disable)") diff --git a/common/vcs/git.go b/common/vcs/git.go index 3d8c9fd03..4e8fb4240 100644 --- a/common/vcs/git.go +++ b/common/vcs/git.go @@ -24,12 +24,16 @@ import ( "strings" "time" + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/cache" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/go-git/go-git/v5/storage/filesystem" "github.com/konveyor/move2kube/common" "github.com/konveyor/move2kube/qaengine" "github.com/sirupsen/logrus" @@ -47,6 +51,11 @@ type GitVCSRepo struct { GitRepoPath string } +var ( + // for https or ssh git repo urls + gitVCSRegex = regexp.MustCompile(`^git\+(https|ssh)://[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,5}(:[0-9]{1,5})?(\/.*)?$`) +) + func isGitCommitHash(commithash string) bool { gitCommitHashRegex := regexp.MustCompile(`^[a-fA-F0-9]{40}$`) return gitCommitHashRegex.MatchString(commithash) @@ -112,18 +121,12 @@ func getGitRepoStruct(vcsurl string) (*GitVCSRepo, error) { } -// isGitVCS checks if the given vcs url is git +// isGitVCS checks if the given vcs url is a git repo url func isGitVCS(vcsurl string) bool { - // for https or ssh - gitVCSRegex := `^git\+(https|ssh)://[a-zA-Z0-9]+([\-\.]{1}[a-zA-Z0-9]+)*\.[a-zA-Z]{2,5}(:[0-9]{1,5})?(\/.*)?$` - matched, err := regexp.MatchString(gitVCSRegex, vcsurl) - if err != nil { - logrus.Fatalf("failed to match the given vcsurl %v with the git vcs regex expression %v. Error : %v", vcsurl, gitVCSRegex, err) - } - return matched + return gitVCSRegex.MatchString(vcsurl) } -func pushGitVCS(remotePath, folderName string) error { +func pushGitVCS(remotePath, folderName string, maxSize int64) error { if !common.IgnoreEnvironment { logrus.Warnf("push to remote git repositories using credentials from the environment is not yet supported.") } @@ -131,7 +134,10 @@ func pushGitVCS(remotePath, folderName string) error { remotePathSplitByColon := strings.Split(remotePathSplitByAt[0], ":") isSSH := strings.HasPrefix(remotePath, "git+ssh") isHTTPS := strings.HasPrefix(remotePath, "git+https") - gitFSPath := GetClonedPath(remotePath, folderName, false) + gitFSPath, err := GetClonedPath(remotePath, folderName, false) + if err != nil { + return fmt.Errorf("failed to clone the repo. Error: %w", err) + } if (isHTTPS && len(remotePathSplitByColon) > 2) || (isSSH && len(remotePathSplitByColon) > 2) { gitFSPath = strings.TrimSuffix(gitFSPath, remotePathSplitByColon[len(remotePathSplitByColon)-1]) } @@ -202,60 +208,73 @@ func pushGitVCS(remotePath, folderName string) error { return nil } -// Clone Clones a git repository with the given commit depth and path where to be cloned and returns final path -func (gvcsrepo *GitVCSRepo) Clone(gitCloneOptions VCSCloneOptions) (string, error) { - - if gitCloneOptions.CloneDestinationPath == "" { - return "", fmt.Errorf("the path where the repository has to be clone is empty - %s", gitCloneOptions.CloneDestinationPath) +// Clone clones a git repository with the given commit depth +// and path where it is to be cloned and returns the final path inside the repo +func (gvcsrepo *GitVCSRepo) Clone(cloneOptions VCSCloneOptions) (string, error) { + if cloneOptions.CloneDestinationPath == "" { + return "", fmt.Errorf("the path where the repository has to be cloned cannot be empty") } - repoPath := filepath.Join(gitCloneOptions.CloneDestinationPath, gvcsrepo.GitRepoPath) - _, err := os.Stat(repoPath) - if os.IsNotExist(err) { - logrus.Debugf("cloned output would be available at '%s'", repoPath) - } else if gitCloneOptions.Overwrite { - logrus.Infof("git repository might get overwritten at %s", repoPath) - err = os.RemoveAll(repoPath) - if err != nil { - return "", fmt.Errorf("failed to remove the directory at the given path - %s", repoPath) + repoPath := filepath.Join(cloneOptions.CloneDestinationPath, gvcsrepo.GitRepoPath) + repoDirInfo, err := os.Stat(repoPath) + if err != nil { + if !os.IsNotExist(err) { + return "", fmt.Errorf("failed to stat the git repo clone destination path '%s'. error: %w", repoPath, err) } + logrus.Debugf("the cloned git repo will be available at '%s'", repoPath) } else { - return filepath.Join(repoPath, gvcsrepo.PathWithinRepo), nil + if !cloneOptions.Overwrite { + if !repoDirInfo.IsDir() { + return "", fmt.Errorf("a file already exists at the git repo clone destination path '%s'", repoPath) + } + logrus.Infof("Assuming that the directory at '%s' is the cloned repo", repoPath) + return filepath.Join(repoPath, gvcsrepo.PathWithinRepo), nil + } + logrus.Infof("git repository clone will overwrite the files/directories at '%s'", repoPath) + if err := os.RemoveAll(repoPath); err != nil { + return "", fmt.Errorf("failed to remove the files/directories at '%s' . error: %w", repoPath, err) + } + } + logrus.Infof("Cloning the repository using git into '%s' . This might take some time.", cloneOptions.CloneDestinationPath) + + // ------------ + var repoDirWt, dotGitDir billy.Filesystem + repoDirWt = osfs.New(repoPath) + dotGitDir, _ = repoDirWt.Chroot(git.GitDirName) + fStorer := filesystem.NewStorage(dotGitDir, cache.NewObjectLRUDefault()) + limitStorer := Limit(fStorer, cloneOptions.MaxSize) + // ------------ + + commitDepth := 1 + if cloneOptions.CommitDepth != 0 { + commitDepth = cloneOptions.CommitDepth } - logrus.Infof("Cloning the repository using git into %s. This might take some time.", gitCloneOptions.CloneDestinationPath) if gvcsrepo.Branch != "" { - commitDepth := 1 - if gitCloneOptions.CommitDepth != 0 { - commitDepth = gitCloneOptions.CommitDepth - } cloneOpts := git.CloneOptions{ URL: gvcsrepo.URL, Depth: commitDepth, SingleBranch: true, ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", gvcsrepo.Branch)), } - gvcsrepo.GitRepository, err = git.PlainClone(repoPath, false, &cloneOpts) + gvcsrepo.GitRepository, err = git.Clone(limitStorer, repoDirWt, &cloneOpts) if err != nil { - logrus.Debugf("provided branch %+v does not exist in the remote, therefore creating one.", gvcsrepo.Branch) + logrus.Debugf("failed to clone the given branch '%s' . Will clone the entire repo and try again.", gvcsrepo.Branch) cloneOpts := git.CloneOptions{ URL: gvcsrepo.URL, Depth: commitDepth, } - gvcsrepo.GitRepository, err = git.PlainClone(repoPath, false, &cloneOpts) + gvcsrepo.GitRepository, err = git.Clone(limitStorer, repoDirWt, &cloneOpts) if err != nil { - return "", fmt.Errorf("failed to perform clone operation using git with options. Error : %+v", err) + return "", fmt.Errorf("failed to perform clone operation using git. Error: %w", err) } branch := fmt.Sprintf("refs/heads/%s", gvcsrepo.Branch) b := plumbing.ReferenceName(branch) w, err := gvcsrepo.GitRepository.Worktree() if err != nil { - return "", fmt.Errorf("failed return a worktree for the repostiory. Error : %+v", err) + return "", fmt.Errorf("failed return a worktree for the repostiory. Error: %w", err) } - - err = w.Checkout(&git.CheckoutOptions{Create: false, Force: false, Branch: b}) - - if err != nil { - err := w.Checkout(&git.CheckoutOptions{Create: true, Force: false, Branch: b}) - if err != nil { + if err := w.Checkout(&git.CheckoutOptions{Create: false, Force: false, Branch: b}); err != nil { + logrus.Debugf("failed to checkout the branch '%s', creating it...", b) + if err := w.Checkout(&git.CheckoutOptions{Create: true, Force: false, Branch: b}); err != nil { return "", fmt.Errorf("failed checkout a new branch. Error : %+v", err) } } @@ -265,45 +284,41 @@ func (gvcsrepo *GitVCSRepo) Clone(gitCloneOptions VCSCloneOptions) (string, erro cloneOpts := git.CloneOptions{ URL: gvcsrepo.URL, } - gvcsrepo.GitRepository, err = git.PlainClone(repoPath, false, &cloneOpts) + gvcsrepo.GitRepository, err = git.Clone(limitStorer, repoDirWt, &cloneOpts) if err != nil { - return "", fmt.Errorf("failed to perform clone operation using git with options %+v. Error : %+v", cloneOpts, err) + return "", fmt.Errorf("failed to perform clone operation using git with options %+v. Error: %w", cloneOpts, err) } r, err := git.PlainOpen(repoPath) if err != nil { - return "", fmt.Errorf("failed to open the git repository at the given path %+v. Error : %+v", repoPath, err) + return "", fmt.Errorf("failed to open the git repository at the given path '%s' . Error: %w", repoPath, err) } w, err := r.Worktree() if err != nil { - return "", fmt.Errorf("failed return a worktree for the repostiory %+v. Error : %+v", r, err) + return "", fmt.Errorf("failed return a worktree for the repostiory %+v. Error: %w", r, err) } - checkoutOpts := git.CheckoutOptions{ - Hash: commitHash, - } - err = w.Checkout(&checkoutOpts) - if err != nil { - return "", fmt.Errorf("failed to checkout commit hash : %s on work tree. Error : %+v", commitHash, w) + checkoutOpts := git.CheckoutOptions{Hash: commitHash} + if err := w.Checkout(&checkoutOpts); err != nil { + return "", fmt.Errorf("failed to checkout commit hash '%s' on work tree. Error: %w", commitHash, err) } } else if gvcsrepo.Tag != "" { cloneOpts := git.CloneOptions{ URL: gvcsrepo.URL, ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/tags/%s", gvcsrepo.Tag)), } - gvcsrepo.GitRepository, err = git.PlainClone(repoPath, false, &cloneOpts) + gvcsrepo.GitRepository, err = git.Clone(limitStorer, repoDirWt, &cloneOpts) if err != nil { - return "", fmt.Errorf("failed to perform clone operation using git with options %+v. Error : %+v", cloneOpts, err) + return "", fmt.Errorf("failed to perform clone operation using git with options %+v. Error: %w", cloneOpts, err) } } else { - commitDepth := 1 cloneOpts := git.CloneOptions{ URL: gvcsrepo.URL, Depth: commitDepth, SingleBranch: true, ReferenceName: "refs/heads/main", } - gvcsrepo.GitRepository, err = git.PlainClone(repoPath, false, &cloneOpts) + gvcsrepo.GitRepository, err = git.Clone(limitStorer, repoDirWt, &cloneOpts) if err != nil { - return "", fmt.Errorf("failed to perform clone operation using git with options %+v. Error : %+v", cloneOpts, err) + return "", fmt.Errorf("failed to perform clone operation using git with options %+v and %+v. Error: %w", cloneOpts, cloneOptions, err) } } return filepath.Join(repoPath, gvcsrepo.PathWithinRepo), nil diff --git a/common/vcs/git_test.go b/common/vcs/git_test.go index c579bfd4e..a550bbd37 100644 --- a/common/vcs/git_test.go +++ b/common/vcs/git_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/google/go-cmp/cmp" - "github.com/konveyor/move2kube/common" ) func TestIsGitCommitHash(t *testing.T) { @@ -125,45 +124,47 @@ func TestIsGitVCS(t *testing.T) { } func TestClone(t *testing.T) { - // Test case - clone a valid vcs url with overwrite true + t.Log("Test case - clone a valid vcs url with overwrite true") gitURL := "git+https://github.com/konveyor/move2kube.git" repo, err := getGitRepoStruct(gitURL) if err != nil { - t.Errorf("failed to get git repo struct for the given git URL %s. Error : %+v", gitURL, err) + t.Fatalf("failed to get git repo struct for the given git URL %s. Error : %+v", gitURL, err) } overwrite := true - tempPath, err := filepath.Abs(common.RemoteTempPath) - if err != nil { - t.Errorf("failed to get absolute path of %s. Error : %+v", common.RemoteTempPath, err) + tempPath := t.TempDir() + cloneDestPath := filepath.Join(tempPath, "test-clone") + var infiniteSize int64 = -1 + cloneOpts := VCSCloneOptions{ + CommitDepth: 1, + Overwrite: overwrite, + CloneDestinationPath: cloneDestPath, + MaxSize: infiniteSize, } - folderName := "test-clone" - cloneOpts := VCSCloneOptions{CommitDepth: 1, Overwrite: overwrite, CloneDestinationPath: filepath.Join(tempPath, folderName)} clonedPath, err := repo.Clone(cloneOpts) if err != nil { - t.Errorf("failed to clone the git repo. Error : %+v", err) + t.Fatalf("failed to clone the git repo. Error : %+v", err) } - // Test case 2 - Repository already exists with overwrite true + t.Log("Test case 2 - Repository already exists with overwrite false") gitURL = "git+https://github.com/konveyor/move2kube.git" repo, err = getGitRepoStruct(gitURL) if err != nil { - t.Errorf("failed to get git repo struct for the given git URL %s. Error : %+v", gitURL, err) + t.Fatalf("failed to get git repo struct for the given git URL '%s' . Error : %+v", gitURL, err) } overwrite = false - tempPath, err = filepath.Abs(common.RemoteTempPath) - if err != nil { - t.Errorf("failed to get absolute path of %s. Error : %+v", common.RemoteTempPath, err) + cloneOpts = VCSCloneOptions{ + CommitDepth: 1, + Overwrite: overwrite, + CloneDestinationPath: cloneDestPath, + MaxSize: infiniteSize, } - folderName = "test-clone" - cloneOpts = VCSCloneOptions{CommitDepth: 1, Overwrite: overwrite, CloneDestinationPath: filepath.Join(tempPath, folderName)} clonedPathWithoutOverwrite, err := repo.Clone(cloneOpts) if err != nil { - t.Errorf("failed to clone the git repo. Error : %+v", err) + t.Fatalf("failed to clone the git repo. Error : %+v", err) } if clonedPath != clonedPathWithoutOverwrite { - t.Errorf("cloned paths did not match with overwrite false. cloned path %s, cloned path without overwrite: %s", clonedPath, clonedPathWithoutOverwrite) + t.Fatalf("cloned paths did not match with overwrite false. cloned path '%s', cloned path without overwrite: '%s'", clonedPath, clonedPathWithoutOverwrite) } - } func TestIsGitBranch(t *testing.T) { diff --git a/common/vcs/limit.go b/common/vcs/limit.go new file mode 100644 index 000000000..77a039a84 --- /dev/null +++ b/common/vcs/limit.go @@ -0,0 +1,70 @@ +/* + * Copyright IBM Corporation 2023 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vcs + +import ( + "errors" + "sync/atomic" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage" +) + +// ErrLimitExceeded is the error returned when the storage limit has been exceeded (usually during git clone) +var ErrLimitExceeded = errors.New("repo size limit exceeded") + +// Limited wraps git.Storer to limit the number of bytes that can be stored. +type Limited struct { + storage.Storer + N atomic.Int64 +} + +// Limit returns a git.Storer limited to the specified number of bytes. +func Limit(s storage.Storer, n int64) storage.Storer { + if n < 0 { + return s + } + l := &Limited{Storer: s} + l.N.Store(n) + return l +} + +// SetEncodedObject is a Storer interface method that is used to store an object +func (s *Limited) SetEncodedObject(obj plumbing.EncodedObject) (plumbing.Hash, error) { + objSize := obj.Size() + n := s.N.Load() + if n-objSize < 0 { + return plumbing.ZeroHash, ErrLimitExceeded + } + for !s.N.CompareAndSwap(n, n-objSize) { + n = s.N.Load() + if n-objSize < 0 { + return plumbing.ZeroHash, ErrLimitExceeded + } + } + return s.Storer.SetEncodedObject(obj) +} + +// Module is a Storer interface method that is used to get the working tree for a repo sub-module +func (s *Limited) Module(name string) (storage.Storer, error) { + m, err := s.Storer.Module(name) + if err != nil { + return nil, err + } + n := s.N.Load() + return Limit(m, n), nil +} diff --git a/common/vcs/vcs.go b/common/vcs/vcs.go index 2ce65ff87..c0de11f6b 100644 --- a/common/vcs/vcs.go +++ b/common/vcs/vcs.go @@ -27,8 +27,9 @@ import ( // VCSCloneOptions stores version control system clone options type VCSCloneOptions struct { CommitDepth int - CloneDestinationPath string Overwrite bool + MaxSize int64 + CloneDestinationPath string } // VCS defines interface for version control system @@ -47,36 +48,15 @@ type FailedVCSPush struct { Err error } -// GetClonedPath takes input and folder name performs clone with the appropriate VCS and then return the file system and remote paths -func GetClonedPath(input, folderName string, overwrite bool) string { - vcsRepo, err := GetVCSRepo(input) - var vcsSrcPath string - if err != nil { - _, ok := err.(*NoCompatibleVCSFound) - if ok { - logrus.Debugf("source path provided is not compatible with any of the supported VCS. Info : %q", err) - } else { - logrus.Fatalf("failed to get vcs repo for the provided source path %s. Error : %v", input, err) - } - vcsSrcPath = "" - } else { - tempPath, err := filepath.Abs(common.RemoteTempPath) - if err != nil { - logrus.Fatalf("failed to get absolute path for the temp path - %s", tempPath) - } - cloneOpts := VCSCloneOptions{CommitDepth: 1, Overwrite: overwrite, CloneDestinationPath: filepath.Join(tempPath, folderName)} - logrus.Debugf("%+v", vcsRepo) - vcsSrcPath, err = vcsRepo.Clone(cloneOpts) - if err != nil { - logrus.Fatalf("failed to clone a repository with the provided vcs url %s and clone options %+v. Error : %+v", input, cloneOpts, err) - } - } - return vcsSrcPath -} +var ( + // maxRepoCloneSize is the maximum size (in bytes) allowed when cloning VCS repos + // default -1 means infinite + maxRepoCloneSize int64 = -1 +) -// IsRemotePath returns if the provided input is a remote path or not -func IsRemotePath(input string) bool { - return isGitVCS(input) +// SetMaxRepoCloneSize sets the maximum size (in bytes) for cloning a repo +func SetMaxRepoCloneSize(size int64) { + maxRepoCloneSize = size } // Error returns the error message for no valid vcs is found @@ -86,7 +66,17 @@ func (e *NoCompatibleVCSFound) Error() string { // Error returns the error message for failed push func (e *FailedVCSPush) Error() string { - return fmt.Sprintf("failed to commit and push for the given VCS path %s. Error : %+v", e.VCSPath, e.Err) + return fmt.Sprintf("failed to commit and push for the given VCS path %s. Error: %+v", e.VCSPath, e.Err) +} + +// IsRemotePath returns if the provided input is a remote path or not +func IsRemotePath(input string) bool { + return isGitVCS(input) +} + +// PushVCSRepo commits and pushes the changes in the provide vcs remote path +func PushVCSRepo(remotePath, folderName string) error { + return pushGitVCS(remotePath, folderName, maxRepoCloneSize) } // GetVCSRepo extracts information from the given vcsurl and returns a relevant vcs repo struct @@ -94,14 +84,40 @@ func GetVCSRepo(vcsurl string) (VCS, error) { if isGitVCS(vcsurl) { vcsRepo, err := getGitRepoStruct(vcsurl) if err != nil { - return nil, fmt.Errorf("failed to get git vcs repo for the input %s. Error : %v", vcsurl, err) + return nil, fmt.Errorf("failed to get git vcs repo for the input '%s' . Error: %w", vcsurl, err) } return vcsRepo, nil } return nil, &NoCompatibleVCSFound{URLInput: vcsurl} } -// PushVCSRepo commits and pushes the changes in the provide vcs remote path -func PushVCSRepo(remotePath, folderName string) error { - return pushGitVCS(remotePath, folderName) +// GetClonedPath takes a vcsurl and a folder name, +// performs a clone with the appropriate VCS, +// and then returns the file system and remote paths. +// If the VCS is not supported, the returned path will be an empty string. +func GetClonedPath(vcsurl, destDirName string, overwrite bool) (string, error) { + vcsRepo, err := GetVCSRepo(vcsurl) + if err != nil { + if _, ok := err.(*NoCompatibleVCSFound); ok { + logrus.Debugf("the vcsurl '%s' is not compatible with any supported VCS. Error: %+v", vcsurl, err) + return "", nil + } + return "", fmt.Errorf("failed to get a VCS for the provided vcsurl '%s'. Error: %w", vcsurl, err) + } + logrus.Debugf("vcsRepo: %+v", vcsRepo) + tempPath, err := filepath.Abs(common.RemoteTempPath) + if err != nil { + return "", fmt.Errorf("failed to get absolute path for the temp path '%s'", common.RemoteTempPath) + } + cloneOpts := VCSCloneOptions{ + CommitDepth: 1, + Overwrite: overwrite, + MaxSize: maxRepoCloneSize, + CloneDestinationPath: filepath.Join(tempPath, destDirName), + } + vcsSrcPath, err := vcsRepo.Clone(cloneOpts) + if err != nil { + return "", fmt.Errorf("failed to clone using vcs url '%s' and clone options %+v. Error: %w", vcsurl, cloneOpts, err) + } + return vcsSrcPath, nil } diff --git a/environment/environment.go b/environment/environment.go index f6611f238..310db9e66 100644 --- a/environment/environment.go +++ b/environment/environment.go @@ -223,8 +223,8 @@ func (e *Environment) Encode(obj interface{}) interface{} { } return e.Env.Upload(path) } - if reflect.ValueOf(obj).Kind() == reflect.String { - val, err := processPath(obj.(string)) + if objS, ok := obj.(string); ok { + val, err := processPath(objS) if err != nil { logrus.Errorf("Unable to process paths for obj %+v : %s", obj, err) } diff --git a/go.mod b/go.mod index 8c0c006c7..3c7dd5aae 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/konveyor/move2kube -go 1.18 +go 1.19 require ( code.cloudfoundry.org/cli v7.1.0+incompatible @@ -18,12 +18,14 @@ require ( github.com/docker/cli v23.0.3+incompatible github.com/docker/docker v23.0.3+incompatible github.com/docker/libcompose v0.4.1-0.20171025083809-57bd716502dc + github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/gobwas/glob v0.2.3 github.com/google/go-cmp v0.5.9 github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-version v1.6.0 github.com/joho/godotenv v1.4.0 + github.com/kopoli/go-terminal-size v0.0.0-20170219200355-5c97524c8b54 github.com/magiconair/properties v1.8.5 github.com/mikefarah/yq/v4 v4.16.2 github.com/mitchellh/mapstructure v1.4.3 @@ -109,7 +111,6 @@ require ( github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.0.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect - github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-kit/log v0.2.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -149,7 +150,6 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kevinburke/ssh_config v1.1.0 // indirect github.com/klauspost/compress v1.16.0 // indirect - github.com/kopoli/go-terminal-size v0.0.0-20170219200355-5c97524c8b54 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect diff --git a/lib/planner.go b/lib/planner.go index 9083b2a25..ad2f03a41 100644 --- a/lib/planner.go +++ b/lib/planner.go @@ -32,10 +32,16 @@ import ( func CreatePlan(ctx context.Context, inputPath, outputPath string, customizationsPath, transformerSelector, prjName string) (plantypes.Plan, error) { logrus.Trace("CreatePlan start") defer logrus.Trace("CreatePlan end") - remoteInputFSPath := vcs.GetClonedPath(inputPath, common.RemoteSourcesFolder, true) - remoteOutputFSPath := vcs.GetClonedPath(outputPath, common.RemoteOutputsFolder, true) - logrus.Debugf("common.TempPath: '%s' inputPath: '%s' remoteInputFSPath '%s'", common.TempPath, inputPath, remoteInputFSPath) plan := plantypes.NewPlan() + remoteInputFSPath, err := vcs.GetClonedPath(inputPath, common.RemoteSourcesFolder, true) + if err != nil { + return plan, fmt.Errorf("failed to clone the repo '%s'. Error: %w", inputPath, err) + } + remoteOutputFSPath, err := vcs.GetClonedPath(outputPath, common.RemoteOutputsFolder, true) + if err != nil { + return plan, fmt.Errorf("failed to clone the repo '%s'. Error: %w", outputPath, err) + } + logrus.Debugf("common.TempPath: '%s' inputPath: '%s' remoteInputFSPath '%s'", common.TempPath, inputPath, remoteInputFSPath) plan.Name = prjName common.ProjectName = prjName plan.Spec.SourceDir = inputPath diff --git a/lib/transformer.go b/lib/transformer.go index 03d92a2c2..7f0c86de7 100644 --- a/lib/transformer.go +++ b/lib/transformer.go @@ -33,9 +33,16 @@ import ( ) // Transform transforms the artifacts and writes output -func Transform(ctx context.Context, plan plantypes.Plan, preExistingPlan bool, outputPath string, transformerSelector string, maxIterations int) error { +func Transform( + ctx context.Context, + plan plantypes.Plan, + preExistingPlan bool, + outputPath string, + transformerSelector string, + maxIterations int, +) error { logrus.Infof("Starting transformation") - + defer logrus.Infof("Transformation done") common.ProjectName = plan.Name logrus.Debugf("common.TempPath: '%s'", common.TempPath) @@ -50,7 +57,10 @@ func Transform(ctx context.Context, plan plantypes.Plan, preExistingPlan bool, o requirements, _ := selectorsInPlan.Requirements() transformerSelectorObj = transformerSelectorObj.Add(requirements...) - remoteOutputFSPath := vcs.GetClonedPath(outputPath, common.RemoteOutputsFolder, true) + remoteOutputFSPath, err := vcs.GetClonedPath(outputPath, common.RemoteOutputsFolder, true) + if err != nil { + return fmt.Errorf("failed to clone the repo '%s'. Error: %w", outputPath, err) + } outputFSPath := outputPath if remoteOutputFSPath != "" { outputFSPath = remoteOutputFSPath @@ -112,11 +122,8 @@ func Transform(ctx context.Context, plan plantypes.Plan, preExistingPlan bool, o return fmt.Errorf("failed to transform using the plan. Error: %w", err) } - logrus.Infof("Transformation done") - if vcs.IsRemotePath(outputPath) { - err := vcs.PushVCSRepo(outputPath, common.RemoteOutputsFolder) - if err != nil { + if err := vcs.PushVCSRepo(outputPath, common.RemoteOutputsFolder); err != nil { logrus.Fatalf("failed to commit and push the output artifacts for the given remote path %s. Errro : %+v", outputPath, err) } logrus.Infof("move2kube generated artifcats are commited and pushed") @@ -128,19 +135,13 @@ func Transform(ctx context.Context, plan plantypes.Plan, preExistingPlan bool, o func Destroy() { logrus.Debugf("Cleaning up!") transformer.Destroy() - err := os.RemoveAll(common.TempPath) - if err != nil { - logrus.Debug("failed to delete temp directory. Error : ", err) + if err := os.RemoveAll(common.TempPath); err != nil { + logrus.Debugf("failed to delete temp directory. Error: %+v", err) } - - err = os.RemoveAll(external.DetectContainerOutputDir) - if err != nil { - logrus.Debug("failed to delete temp directory. Error : ", err) + if err := os.RemoveAll(external.DetectContainerOutputDir); err != nil { + logrus.Debugf("failed to delete temp directory. Error: %+v", err) } - - err = os.RemoveAll(external.TransformContainerOutputDir) - if err != nil { - logrus.Debug("failed to delete temp directory. Error : ", err) + if err := os.RemoveAll(external.TransformContainerOutputDir); err != nil { + logrus.Debugf("failed to delete temp directory. Error: %+v", err) } - } diff --git a/lib/utils.go b/lib/utils.go index 258337e44..b0461ca5e 100644 --- a/lib/utils.go +++ b/lib/utils.go @@ -32,7 +32,10 @@ const TransformerTypeMeta = "Transformer" // CheckAndCopyCustomizations checks if the customizations path is an existing directory and copies to assets func CheckAndCopyCustomizations(customizationsPath string) error { - remoteCustomizationsPath := vcs.GetClonedPath(customizationsPath, common.RemoteCustomizationsFolder, true) + remoteCustomizationsPath, err := vcs.GetClonedPath(customizationsPath, common.RemoteCustomizationsFolder, true) + if err != nil { + return fmt.Errorf("failed to clone the repo. Error: %w", err) + } customizationsFSPath := customizationsPath if remoteCustomizationsPath != "" { customizationsFSPath = remoteCustomizationsPath @@ -40,7 +43,7 @@ func CheckAndCopyCustomizations(customizationsPath string) error { if customizationsFSPath == "" { return nil } - customizationsFSPath, err := filepath.Abs(customizationsFSPath) + customizationsFSPath, err = filepath.Abs(customizationsFSPath) if err != nil { return fmt.Errorf("failed to make the customizations directory path '%s' absolute. Error: %w", customizationsFSPath, err) } diff --git a/types/plan/planutils.go b/types/plan/planutils.go index 749878d49..c055b17bf 100644 --- a/types/plan/planutils.go +++ b/types/plan/planutils.go @@ -17,6 +17,7 @@ package plan import ( + "fmt" "os" "path/filepath" @@ -33,41 +34,44 @@ func ReadPlan(path string, sourceDir string) (Plan, error) { var err error absSourceDir := "" if err = common.ReadMove2KubeYaml(path, &plan); err != nil { - logrus.Errorf("Failed to load the plan file at path %q Error %q", path, err) - return plan, err + return plan, fmt.Errorf("failed to load the plan file at path '%s' . Error: %w", path, err) } if sourceDir != "" { plan.Spec.SourceDir = sourceDir } if plan.Spec.SourceDir != "" { - remoteSrcPath := vcs.GetClonedPath(plan.Spec.SourceDir, common.RemoteSourcesFolder, false) + remoteSrcPath, err := vcs.GetClonedPath(plan.Spec.SourceDir, common.RemoteSourcesFolder, false) + if err != nil { + return plan, fmt.Errorf("failed to clone the repo. Error: %w", err) + } if remoteSrcPath != "" { plan.Spec.SourceDir = remoteSrcPath } absSourceDir, err = filepath.Abs(plan.Spec.SourceDir) if err != nil { - logrus.Errorf("Unable to convert sourceDir to full path : %s", err) - return plan, err + return plan, fmt.Errorf("failed to convert sourceDir to full path. Error: %w", err) } } if err = pathconverters.MakePlanPathsAbsolute(&plan, absSourceDir, common.TempPath); err != nil { return plan, err } plan.Spec.SourceDir = absSourceDir - return plan, err + return plan, nil } // WritePlan encodes the plan to yaml converting absolute paths to relative. func WritePlan(path string, plan Plan) error { inputFSPath := plan.Spec.SourceDir - remoteSrcPath := vcs.GetClonedPath(plan.Spec.SourceDir, common.RemoteSourcesFolder, false) + remoteSrcPath, err := vcs.GetClonedPath(plan.Spec.SourceDir, common.RemoteSourcesFolder, false) + if err != nil { + return fmt.Errorf("failed to clone the repo. error: %w", err) + } if remoteSrcPath != "" { inputFSPath = remoteSrcPath } newPlan := deepcopy.DeepCopy(plan).(Plan) if err := pathconverters.ChangePaths(&newPlan, map[string]string{inputFSPath: "", common.TempPath: ""}); err != nil { - logrus.Errorf("Unable to convert plan to use relative paths : %s", err) - return err + return fmt.Errorf("failed to convert plan to use relative paths. Error: %w", err) } wd, err := os.Getwd() if err != nil {