Fix scms prompting for credentials (#240)

CHANGELOG.md

@@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
+## [1.8.7] - 7/19/22
+### Added
+### Changed
+### Deprecated
+### Removed
+### Fixed
+- SCM providers prompting for credentials
+### Security
+
 ## [1.8.6] - 7/18/22
 ### Added
 ### Changed

README.md

@@ -308,35 +308,10 @@ ghorg reclone kubernetes-sig-staging kubernetes-sig
 ## Troubleshooting
 
 - If you are having trouble cloning repos. Try to clone one of the repos locally e.g. manually running `git clone https://github.com/your_private_org/your_private_repo.git` if this does not work, ghorg will also not work. Your git client must first be setup to clone the target repos. If you normally clone using an ssh key use the `--protocol=ssh` flag with ghorg. This will fetch the ssh clone urls instead of the https clone urls.
-
-- When cloning if you see something like `Username for 'https://gitlab.com': ` and the run won't finish. Make sure you have correctly set your token on the commandline, in your ghorg conf. If this does not work, try cloning via ssh (--protocol=ssh). If this still does not resolve your issue you can try following the process below.
-
-    1. Make sure that you can clone using SSH with no username/password using "git clone [email protected]:xxx/yyy/zzz.git" (replace the link to the correct git file). If you can't clone or it requires a password, fix this problem first (unrelated to ghorg)
-    2. In "git config", make sure that the email is correct
-    3. Delete all files and folders (git repos) in the ghorg directory
-    4. Run ghorg once again using -t (the gitlab personal access token, new tokens start with "glpat-"), --scm=gitlab --protocol=ssh
-
-    If this still does not resolve your issue you will need to update your git configs to match below, be sure to update the gitlab.mydomain.com portion
-
-    ```
-    git config --global url."[email protected]:".insteadOf http://gitlab.mydomain.com/
-    git config --global url."git://".insteadOf https://
-    ```
-
-- If you are cloning a large org you may see `Error: open /dev/null: too many open files` which means you need to increase your ulimits, there are lots of docs online for this. For mac the quick and dirty is below
-
-    ```
-    # reset the soft and hard file limit boundaries
-    $ sudo launchctl limit maxfiles 65536 200000
-
-    # actually now set the ulimit boundary
-    $ ulimit -n 20000
-    ```
-
-    Another solution is to decrease the number of concurrent clones. Use the `--concurrency` flag to set to lower than 25 (the default)
-
+- If you are cloning a large org you may see `Error: open /dev/null: too many open files` which means you need to increase your ulimits, there are lots of docs online for this. Another solution is to decrease the number of concurrent clones. Use the `--concurrency` flag to set to lower than 25 (the default)
 - If your GitHub org is behind SSO, you will need to authorize your token, see [here](https://docs.github.com/en/github/authenticating-to-github/authorizing-a-personal-access-token-for-use-with-saml-single-sign-on)
 - If your GitHub Personal Access Token is only finding public repos, give your token all the repos permissions
 - Make sure your `$ git --version` is >= 2.19.0
-- Check for other software, such as anti-malware, that could interfere with ghorgs ability to create large number of connections, see [issue 132](https://github.com/gabrie30/ghorg/issues/132#issuecomment-889357960)
+- Check for other software, such as anti-malware, that could interfere with ghorgs ability to create large number of connections, see [issue 132](https://github.com/gabrie30/ghorg/issues/132#issuecomment-889357960). You can also lower the concurrency with `--concurrency=n` default is 25.
+- To debug yourself you can call ghorg with the GHORG_DEBUG=true env e.g `GHORG_DEBUG=true ghorg clone kubernetes --concurrency=1`
 - If you've gotten this far and still have an issue feel free to raise an issue

cmd/clone.go

@@ -571,6 +571,14 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 
 			action := "cloning"
 			if repoExistsLocally(repo) {
+				// prevents git from asking for user for credentials, needs to be unset so creds aren't stored
+				err := git.SetOriginWithCredentials(repo)
+				if err != nil {
+					e := fmt.Sprintf("Problem setting remote with credentials Repo: %s Error: %v", repo.Name, err)
+					cloneErrors = append(cloneErrors, e)
+					return
+				}
+
 				if os.Getenv("GHORG_BACKUP") == "true" {
 					err := git.UpdateRemote(repo)
 					action = "updating remote"
@@ -649,6 +657,13 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 						}
 					}
 				}
+
+				err = git.SetOrigin(repo)
+				if err != nil {
+					e := fmt.Sprintf("Problem resetting remote Repo: %s Error: %v", repo.Name, err)
+					cloneErrors = append(cloneErrors, e)
+					return
+				}
 			} else {
 				// if https clone and github/gitlab add personal access token to url
 

cmd/clone_test.go

@@ -69,6 +69,10 @@ func (g MockGitClient) SetOrigin(repo scm.Repo) error {
 	return nil
 }
 
+func (g MockGitClient) SetOriginWithCredentials(repo scm.Repo) error {
+	return nil
+}
+
 func (g MockGitClient) Checkout(repo scm.Repo) error {
 	return nil
 }

cmd/version.go

@@ -6,7 +6,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const ghorgVersion = "v1.8.6"
+const ghorgVersion = "v1.8.7"
 
 var versionCmd = &cobra.Command{
 	Use:   "version",

git/git.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/exec"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/gabrie30/ghorg/scm"
 )
 
@@ -13,6 +14,7 @@ type Gitter interface {
 	Reset(scm.Repo) error
 	Pull(scm.Repo) error
 	SetOrigin(scm.Repo) error
+	SetOriginWithCredentials(scm.Repo) error
 	Clean(scm.Repo) error
 	Checkout(scm.Repo) error
 	UpdateRemote(scm.Repo) error
@@ -25,6 +27,12 @@ func NewGit() GitClient {
 	return GitClient{}
 }
 
+func printDebugCmd(cmd *exec.Cmd) {
+	fmt.Println("------------- GIT DEBUG -------------")
+	spew.Dump(*cmd)
+	fmt.Println("")
+}
+
 func (g GitClient) Clone(repo scm.Repo) error {
 	args := []string{"clone", repo.CloneURL, repo.HostPath}
 
@@ -45,10 +53,22 @@ func (g GitClient) Clone(repo scm.Repo) error {
 	}
 
 	cmd := exec.Command("git", args...)
+
+	if os.Getenv("GHORG_DEBUG") != "" {
+		printDebugCmd(cmd)
+	}
+
 	err := cmd.Run()
 	return err
 }
 
+func (g GitClient) SetOriginWithCredentials(repo scm.Repo) error {
+	args := []string{"remote", "set-url", "origin", repo.CloneURL}
+	cmd := exec.Command("git", args...)
+	cmd.Dir = repo.HostPath
+	return cmd.Run()
+}
+
 func (g GitClient) SetOrigin(repo scm.Repo) error {
 	args := []string{"remote", "set-url", "origin", repo.URL}
 	cmd := exec.Command("git", args...)
@@ -85,6 +105,11 @@ func (g GitClient) Pull(repo scm.Repo) error {
 
 	cmd := exec.Command("git", args...)
 	cmd.Dir = repo.HostPath
+
+	if os.Getenv("GHORG_DEBUG") != "" {
+		printDebugCmd(cmd)
+	}
+
 	return cmd.Run()
 }
 

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	code.gitea.io/sdk/gitea v0.15.1
+	github.com/davecgh/go-spew v1.1.1
 	github.com/fatih/color v1.13.0
 	github.com/google/go-github/v41 v41.0.0
 	github.com/korovkin/limiter v0.0.0-20220422174850-01f593e64cf7

scripts/gitlab_cloud_integration_tests.sh

@@ -7,10 +7,12 @@ echo "Running GitLab Cloud Integration Tests"
 cp ./ghorg /usr/local/bin
 
 # https://gitlab.com/gitlab-examples
-GITLAB_ORG=gitlab-examples
+GITLAB_GROUP=gitlab-examples
 GITLAB_SUB_GROUP=wayne-enterprises
 
-ghorg clone $GITLAB_ORG --token="${GITLAB_TOKEN}" --scm=gitlab --output-dir=examples-flat
+GITLAB_GROUP_2=ghorg-test-group
+
+ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --output-dir=examples-flat
 
 if [ -e "${HOME}"/ghorg/examples-flat/microservice ]
 then
@@ -20,7 +22,7 @@ else
     exit 1
 fi
 
-ghorg clone $GITLAB_ORG --token="${GITLAB_TOKEN}" --scm=gitlab --output-dir=examples --preserve-dir
+ghorg clone $GITLAB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --output-dir=examples --preserve-dir
 
 if [ -e "${HOME}"/ghorg/examples/"${GITLAB_SUB_GROUP}"/wayne-industries/microservice ]
 then
@@ -30,22 +32,40 @@ else
     exit 1
 fi
 
-ghorg clone $GITLAB_ORG/$GITLAB_SUB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab
+ghorg clone $GITLAB_GROUP/$GITLAB_SUB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab
 
-if [ -e "${HOME}"/ghorg/"${GITLAB_ORG}"/"${GITLAB_SUB_GROUP}"/microservice ]
+if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP}"/"${GITLAB_SUB_GROUP}"/microservice ]
 then
     echo "Pass: gitlab subgroup clone flat file"
 else
     echo "Fail: gitlab subgroup clone flat file"
     exit 1
 fi
 
-ghorg clone $GITLAB_ORG/$GITLAB_SUB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --preserve-dir
+ghorg clone $GITLAB_GROUP/$GITLAB_SUB_GROUP --token="${GITLAB_TOKEN}" --scm=gitlab --preserve-dir
 
-if [ -e "${HOME}"/ghorg/"${GITLAB_ORG}"/"${GITLAB_SUB_GROUP}"/wayne-industries/microservice ]
+if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP}"/"${GITLAB_SUB_GROUP}"/wayne-industries/microservice ]
 then
     echo "Pass: gitlab subgroup clone preserve directories"
 else
     echo "Fail: gitlab subgroup clone preserve directories"
     exit 1
 fi
+
+ghorg clone $GITLAB_GROUP_2 --token="${GITLAB_TOKEN}" --scm=gitlab
+
+if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/_subgroup-1_foobar ]
+then
+    echo "Pass: gitlab group clone with colliding repo names"
+else
+    echo "Fail: gitlab group clone with colliding repo names"
+    exit 1
+fi
+
+if [ -e "${HOME}"/ghorg/"${GITLAB_GROUP_2}"/_subgroup-2_foobar ]
+then
+    echo "Pass: gitlab group clone with colliding repo names"
+else
+    echo "Fail: gitlab group clone with colliding repo names"
+    exit 1
+fi