Add preserve scm hostname flag (#458)

CHANGELOG.md

@@ -3,9 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 
-## [1.10.1] - unreleased
+## [1.11.0] - unreleased
 ### Added
 - Reclone name and description to reclone output
+- GHORG_PRESERVE_SCM_HOSTNAME, note that this feature changes the directory struture that gitlab all-users and all-groups clone into; thanks @rrrix
 ### Changed
 ### Deprecated
 ### Removed

cmd/clone.go

@@ -163,6 +163,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
 		os.Setenv("GHORG_GIT_FILTER", filter)
 	}
 
+	if cmd.Flags().Changed("preserve-scm-hostname") {
+		os.Setenv("GHORG_PRESERVE_SCM_HOSTNAME", "true")
+	}
+
 	if cmd.Flags().Changed("skip-archived") {
 		os.Setenv("GHORG_SKIP_ARCHIVED", "true")
 	}
@@ -282,6 +286,10 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
 		os.Exit(1)
 	}
 
+	if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" {
+		updateAbsolutePathToCloneToWithHostname()
+	}
+
 	setOutputDirName(argz)
 	setOuputDirAbsolutePath()
 	args = argz
@@ -974,7 +982,14 @@ func CloneAllRepos(git git.Gitter, cloneTargets []scm.Repo) {
 }
 
 func writeGhorgStats(date string, allReposToCloneCount, cloneCount, pulledCount, cloneInfosCount, cloneErrorsCount, updateRemoteCount, newCommits, pruneCount int, hasCollisions bool) error {
-	statsFilePath := filepath.Join(os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO"), "_ghorg_stats.csv")
+	var statsFilePath string
+	absolutePath := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO")
+	if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" {
+		originalAbsolutePath := os.Getenv("GHORG_ORIGINAL_ABSOLUTE_PATH_TO_CLONE_TO")
+		statsFilePath = filepath.Join(originalAbsolutePath, "_ghorg_stats.csv")
+	} else {
+		statsFilePath = filepath.Join(absolutePath, "_ghorg_stats.csv")
+	}
 
 	fileExists := true
 
@@ -1398,22 +1413,23 @@ func setOutputDirName(argz []string) {
 
 	outputDirName = strings.ToLower(argz[0])
 
-	// If all-group is used set the parent folder to the name of the baseurl
-	if argz[0] == "all-groups" && os.Getenv("GHORG_SCM_BASE_URL") != "" {
-		u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL"))
-		if err != nil {
-			return
+	if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") != "true" {
+		// If all-group is used set the parent folder to the name of the baseurl
+		if argz[0] == "all-groups" && os.Getenv("GHORG_SCM_BASE_URL") != "" {
+			u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL"))
+			if err != nil {
+				colorlog.PrintError(fmt.Sprintf("Error parsing GHORG_SCM_BASE_URL, clone may be affected, error: %v", err))
+			}
+			outputDirName = u.Hostname()
 		}
-		outputDirName = strings.TrimSuffix(strings.TrimPrefix(u.Host, "www."), ".com")
-	}
 
-	if argz[0] == "all-users" && os.Getenv("GHORG_SCM_BASE_URL") != "" {
-		u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL"))
-		if err != nil {
-			return
+		if argz[0] == "all-users" && os.Getenv("GHORG_SCM_BASE_URL") != "" {
+			u, err := url.Parse(os.Getenv("GHORG_SCM_BASE_URL"))
+			if err != nil {
+				colorlog.PrintError(fmt.Sprintf("Error parsing GHORG_SCM_BASE_URL, clone may be affected, error: %v", err))
+			}
+			outputDirName = u.Hostname()
 		}
-		outputDirName = strings.TrimSuffix(strings.TrimPrefix(u.Host, "www."), ".com")
-		outputDirName = outputDirName + "_users"
 	}
 
 	if os.Getenv("GHORG_BACKUP") == "true" {

cmd/examples-copy/bitbucket.md

@@ -4,6 +4,8 @@
 
 To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help`
 
+1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For bitbucket cloud it will be `bitbucket.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL`.
+
 ## Bitbucket Cloud
 
 1. Clone the microsoft workspace using an app-password

cmd/examples-copy/gitea.md

@@ -4,6 +4,12 @@
 
 To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help`
 
+## Things to know
+
+1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the `GHORG_SCM_BASE_URL` you are cloning from.
+
+## Examples
+
 1. Clone an **org**
 
     ```

cmd/examples-copy/github.md

@@ -4,13 +4,25 @@
 
 To view all additional flags see the [sample-conf.yaml](https://github.com/gabrie30/ghorg/blob/master/sample-conf.yaml) or use `ghorg clone --help`
 
+## Things to know
+
+1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For github cloud it will be `github.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL` if set for enterpirse github orgs.
+
 ## GitHub Cloud
 
 1. Clone an **org**, using a token on the commandline
 
     ```
     ghorg clone <github_org> --token=XXXXXX
     ```
+    Will produce the following
+
+    ```sh
+    /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
+    └── github_org
+        ├── repo1
+        └── repo2
+    ```
 
 1. Clone all repos from a **github org** that are **prefixed** with "frontend" **into a folder** called "design_only"
 
@@ -24,6 +36,20 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
     ghorg clone <github_username> --clone-type=user --token=bGVhdmUgYSBjb21tZW50IG9uIGlzc3VlIDY2
     ```
 
+1. Clone an **org**, preserving the cloud scm hostname
+    ```
+    ghorg clone <github_org> --preserve-cloud-scm-hostname
+    ```
+    Will produce the following
+
+    ```sh
+    /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
+    └── github.com
+        └── github_org
+            ├── repo1
+            └── repo2
+    ```
+
 ## GitHub Enterprise
 
 1. Clone an **org**
@@ -43,3 +69,31 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
     ```
     ghorg clone <github_username> --clone-type=user --base-url=https://<your-hosted-github>.com --token=XXXXXX
     ```
+
+1. Clone an **org**, preserving the cloud scm hostname. The github_org may seem redudant in this example but when its needed for the case when you start to set an output directory.
+    ```
+    ghorg clone <github_org> --base-url=https://<your-hosted-github>.com --preserve-cloud-scm-hostname
+    ```
+    Will produce the following
+
+    ```sh
+    /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
+    └── your-hosted-github.com
+        └── github_org
+            ├── repo1
+            └── repo2
+    ```
+
+1. Clone an **org**, preserving the cloud scm hostname **with an output directory**
+    ```
+    ghorg clone <github_org> --base-url=https://<your-hosted-github>.com --output-dir=my-repos --preserve-cloud-scm-hostname
+    ```
+    Will produce the following
+
+    ```sh
+    /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
+    └── your-hosted-github.com
+        └── my-repos
+            ├── myrepo1
+            └── myrepo2
+    ```

cmd/examples-copy/gitlab.md

@@ -14,6 +14,8 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
 
 1. The `--output-dir` flag overrides the default name given to the folder ghorg creates to clone repos into. The default will be the instance name when cloning `all-groups` or `all-users` or the `group` name when cloning a specific group. The exception is when you are cloning a subgroup and preserving the directory structure, then it will preserve the parent groups of the subgroup.
 
+1. The `--preserve-scm-hostname` flag will always create a top level folder in your GHORG_ABSOLUTE_PATH_TO_CLONE_TO with the hostname of the instance you are cloning from. For gitlab cloud it will be `gitlab.com/` otherwise it will be what is set to the hostname of the `GHORG_SCM_BASE_URL`.
+
 1. If the group name you are cloning has spaces, substitute the spaces with "-" e.g.
 
     ```sh
@@ -45,7 +47,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
 
     ```sh
     /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
-    └── your.instance.gitlab
+    └── your.instance.gitlab.com
         ├── group1
         │   └── project1
         ├── group2
@@ -66,13 +68,34 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
 
     ```sh
     /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
-    └── your.instance.gitlab
+    └── your.instance.gitlab.com
         ├── project1
         ├── project2
         ├── project3
         └── project4
     ```
 
+1. Clone **all groups**, **preserving the directory structure** of users, preserving scm hostname
+
+    ```sh
+    ghorg clone all-groups --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --preserve-dir --preserve-scm-hostname
+    ```
+
+    This would produce a directory structure like
+
+    ```sh
+    /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
+    └── your.instance.gitlab.com
+        └── all-groups
+            ├── group1
+            │   └── project1
+            ├── group2
+            │   └── project2
+            └── group3
+                ├── project3
+                └── project4
+    ```
+
 #### Cloning Specific Groups
 
 1. Clone **a specific group**, **preserving the directory structure** of subgroups
@@ -175,7 +198,7 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
 
     ```sh
     /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
-    └── your.instance.gitlab_users
+    └── your.instance.gitlab.com
         ├── user1
         │   └── project1
         ├── user2
@@ -194,11 +217,30 @@ To view all additional flags see the [sample-conf.yaml](https://github.com/gabri
 
     ```sh
     /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
-    └── your.instance.gitlab_users
-        ├── project1
-        ├── project2
-        ├── project3
-        └── project4
+    └── your.instance.gitlab.com
+        ├── user1-repo1
+        └── user2-repo1
+    ```
+
+1. Clone **all users**, **preserving the directory structure** of users, preserving scm hostname
+
+    ```sh
+    ghorg clone all-users --base-url=https://<your.instance.gitlab.com> --scm=gitlab --token=XXXXXX --preserve-dir --preserve-scm-hostname
+    ```
+
+    This would produce a directory structure like
+
+    ```sh
+    /GHORG_ABSOLUTE_PATH_TO_CLONE_TO
+    └── your.instance.gitlab.com
+        └── all-users
+            ├── user1
+            │   └── project1
+            ├── user2
+            │   └── project2
+            └── user3
+                ├── project3
+                └── project4
     ```
 
 ## Cloud GitLab Orgs

cmd/root.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 
@@ -68,6 +69,7 @@ var (
 	quietMode                    bool
 	noDirSize                    bool
 	ghorgStatsEnabled            bool
+	ghorgPreserveScmHostname     bool
 	args                         []string
 	cloneErrors                  []string
 	cloneInfos                   []string
@@ -83,6 +85,40 @@ var rootCmd = &cobra.Command{
 	},
 }
 
+func getHostname() string {
+	var hostname string
+	baseURL := os.Getenv("GHORG_SCM_BASE_URL")
+	if baseURL != "" {
+		// Parse the URL to extract the hostname
+		parsedURL, err := url.Parse(baseURL)
+		if err != nil {
+			colorlog.PrintError(fmt.Sprintf("Error parsing GHORG_SCM_BASE_URL clone may be affected, error: %v", err))
+		}
+		// Append the hostname to the absolute path
+		hostname = parsedURL.Hostname()
+	} else {
+		// Use the predefined hostname based on the SCM type
+		hostname = configs.GetCloudScmTypeHostnames()
+	}
+
+	return hostname
+}
+
+// updateAbsolutePathToCloneToWithHostname modifies the absolute path by appending the hostname if the user has enabled it,
+// supporting the GHORG_PRESERVE_SCM_HOSTNAME feature. It checks the GHORG_PRESERVE_SCM_HOSTNAME environment variable, and if set to "true",
+// it uses the hostname from GHORG_SCM_BASE_URL if available, otherwise, it defaults to a predefined hostname based on the SCM type.
+func updateAbsolutePathToCloneToWithHostname() {
+	// Verify if GHORG_PRESERVE_SCM_HOSTNAME is set to "true"
+	if os.Getenv("GHORG_PRESERVE_SCM_HOSTNAME") == "true" {
+		// Retrieve the hostname from the environment variable
+		hostname := getHostname()
+		absolutePath := os.Getenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO")
+		os.Setenv("GHORG_ORIGINAL_ABSOLUTE_PATH_TO_CLONE_TO", absolutePath)
+		absolutePath = filepath.Join(absolutePath, hostname)
+		os.Setenv("GHORG_ABSOLUTE_PATH_TO_CLONE_TO", configs.EnsureTrailingSlashOnFilePath(absolutePath))
+	}
+}
+
 // reads in configuration file and updates anything not set to default
 func getOrSetDefaults(envVar string) {
 	if envVar == "GHORG_COLOR" {
@@ -145,6 +181,8 @@ func getOrSetDefaults(envVar string) {
 			os.Setenv(envVar, "owner")
 		case "GHORG_BACKUP":
 			os.Setenv(envVar, "false")
+		case "GHORG_PRESERVE_SCM_HOSTNAME":
+			os.Setenv(envVar, "false")
 		case "GHORG_NO_TOKEN":
 			os.Setenv(envVar, "false")
 		case "GHORG_NO_DIR_SIZE":
@@ -228,6 +266,7 @@ func InitConfig() {
 	getOrSetDefaults("GHORG_CLONE_PROTOCOL")
 	getOrSetDefaults("GHORG_CLONE_TYPE")
 	getOrSetDefaults("GHORG_SCM_TYPE")
+	getOrSetDefaults("GHORG_PRESERVE_SCM_HOSTNAME")
 	getOrSetDefaults("GHORG_SKIP_ARCHIVED")
 	getOrSetDefaults("GHORG_SKIP_FORKS")
 	getOrSetDefaults("GHORG_NO_CLEAN")
@@ -324,6 +363,7 @@ func init() {
 	cloneCmd.Flags().BoolVar(&quietMode, "quiet", false, "GHORG_QUIET - Emit critical output only")
 	cloneCmd.Flags().BoolVar(&includeSubmodules, "include-submodules", false, "GHORG_INCLUDE_SUBMODULES - Include submodules in all clone and pull operations.")
 	cloneCmd.Flags().BoolVar(&ghorgStatsEnabled, "stats-enabled", false, "GHORG_STATS_ENABLED - Creates a CSV in the GHORG_ABSOLUTE_PATH_TO_CLONE_TO called _ghorg_stats.csv with info about each clone. This allows you to track clone data over time such as number of commits and size in megabytes of the clone directory.")
+	cloneCmd.Flags().BoolVar(&ghorgPreserveScmHostname, "preserve-scm-hostname", false, "GHORG_PRESERVE_SCM_HOSTNAME - Appends the scm hostname to the GHORG_ABSOLUTE_PATH_TO_CLONE_TO which will organize your clones into specific folders by the scm provider. e.g. /github.com/kuberentes")
 	cloneCmd.Flags().StringVarP(&baseURL, "base-url", "", "", "GHORG_SCM_BASE_URL - Change SCM base url, for on self hosted instances (currently gitlab, gitea and github (use format of https://git.mydomain.com/api/v3))")
 	cloneCmd.Flags().StringVarP(&concurrency, "concurrency", "", "", "GHORG_CONCURRENCY - Max goroutines to spin up while cloning (default 25)")
 	cloneCmd.Flags().StringVarP(&cloneDepth, "clone-depth", "", "", "GHORG_CLONE_DEPTH - Create a shallow clone with a history truncated to the specified number of commits")

cmd/version.go

@@ -6,7 +6,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const ghorgVersion = "v1.10.1"
+const ghorgVersion = "v1.11.0"
 
 var versionCmd = &cobra.Command{
 	Use:   "version",