Merge pull request #32 from jfrog/GH-329-repo-assign-to-project-limit…

…ation-sm

Limitation of project resource

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.1.0 (Apr 13, 2022)
+
+IMPROVEMENTS:
+
+* Documentation improved for `project` resource to include limitations in Note section. [GH-32]
+
 ## 1.0.4 (Mar 22, 2022)
 
 BUG FIXES:

docs/resources/project.md

@@ -84,7 +84,7 @@ resource "project" "myproject" {
 - **id** (String) The ID of this resource.
 - **max_storage_in_gibibytes** (Number) Storage quota in GiB. Must be 1 or larger. Set to -1 for unlimited storage. This is translated to binary bytes for Artifactory API. So for 1TB quota, this should be set to 1024 (vs 1000) which will translate to 1099511627776 bytes for the API.
 - **member** (Block Set) Member of the project. Element has one to one mapping with the [JFrog Project Users API](https://www.jfrog.com/confluence/display/JFROG/Artifactory+REST+API#ArtifactoryRESTAPI-UpdateUserinProject). (see [below for nested schema](#nestedblock--member))
-- **repos** (Set of String) List of existing repo keys to be assigned to the project.
+- **repos** (Set of String) List of existing repo keys to be assigned to the project. By default, `repos` is capped at 100 keys. (see [below for limitations on the maximum number of repos](#repoNumLimitations)). You can remove the system's cap by setting an environment value `REPO_LIMIT_OVERRIDE` to `true`. This setting will remove the restriction of maximum allowable elements in the `repos` attribute. The default value of `REPO_LIMIT_OVERRIDE` is `false`, e.g. To override use export REPO_LIMIT_OVERRIDE=true` in the shell.
 - **role** (Block Set) Project role. Element has one to one mapping with the [JFrog Project Roles API](https://www.jfrog.com/confluence/display/JFROG/Artifactory+REST+API#ArtifactoryRESTAPI-AddaNewRole) (see [below for nested schema](#nestedblock--role))
 
 <a id="nestedblock--admin_privileges"></a>
@@ -128,3 +128,8 @@ Required:
 Optional:
 
 - **description** (String)
+
+## Note
+<a name="repoNumLimitations"></a>
+### Limitations of Repository assign/unassign to/from Project
+Artifactory's [Move Repository in a Project](https://www.jfrog.com/confluence/display/JFROG/Artifactory+REST+API#ArtifactoryRESTAPI-MoveRepositoryinaProject) & [Unassign a Project from a Repository](https://www.jfrog.com/confluence/display/JFROG/Artifactory+REST+API#ArtifactoryRESTAPI-UnassignaProjectfromaRepository) APIs have limitations to assign or unassign a large number of repositories to or from project. With more than a certain number of repos you might observe system disruptions and all the repositories might not be assigned/unassigned to/from the desired project. As per our analysis, it is **recommended to limit the number of repo keys to 100** to avoid internal platform limitations. The JFrog Engineering team is working on further improvement to ensure a better user experience.

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/hashicorp/terraform-plugin-docs v0.5.1
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.7.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 )
 
 require (
@@ -32,7 +33,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect
 	github.com/hashicorp/go-getter v1.5.3 // indirect
-	github.com/hashicorp/go-hclog v0.15.0 // indirect
+	github.com/hashicorp/go-hclog v1.2.0 // indirect
 	github.com/hashicorp/go-multierror v1.0.0 // indirect
 	github.com/hashicorp/go-plugin v1.4.1 // indirect
 	github.com/hashicorp/go-safetemp v1.0.0 // indirect
@@ -43,6 +44,7 @@ require (
 	github.com/hashicorp/terraform-exec v0.15.0 // indirect
 	github.com/hashicorp/terraform-json v0.13.0 // indirect
 	github.com/hashicorp/terraform-plugin-go v0.3.0 // indirect
+	github.com/hashicorp/terraform-plugin-log v0.3.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
@@ -54,7 +56,7 @@ require (
 	github.com/mitchellh/cli v1.1.2 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/mitchellh/go-testing-interface v1.0.4 // indirect
+	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect

go.sum

@@ -145,6 +145,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0 h1:pMen7vLs8nvgEYhywH3KDWJIJTeEr2ULsVWHWYHQyBs=
@@ -179,6 +181,8 @@ github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9
 github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-hclog v0.15.0 h1:qMuK0wxsoW4D0ddCCYwPSTm4KQv1X1ke3WmPWZ0Mvsk=
 github.com/hashicorp/go-hclog v0.15.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
+github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM=
+github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-plugin v1.3.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0=
@@ -208,6 +212,8 @@ github.com/hashicorp/terraform-plugin-docs v0.5.1 h1:WwrUcamix9x0TqfTw/WGHMRqoTe
 github.com/hashicorp/terraform-plugin-docs v0.5.1/go.mod h1:SQwEgy0/B0UPQ07rNEG1Wpt6E3jvRcCwkVHPNybGgc0=
 github.com/hashicorp/terraform-plugin-go v0.3.0 h1:AJqYzP52JFYl9NABRI7smXI1pNjgR5Q/y2WyVJ/BOZA=
 github.com/hashicorp/terraform-plugin-go v0.3.0/go.mod h1:dFHsQMaTLpON2gWhVWT96fvtlc/MF1vSy3OdMhWBzdM=
+github.com/hashicorp/terraform-plugin-log v0.3.0 h1:NPENNOjaJSVX0f7JJTl4f/2JKRPQ7S2ZN9B4NSqq5kA=
+github.com/hashicorp/terraform-plugin-log v0.3.0/go.mod h1:EjueSP/HjlyFAsDqt+okpCPjkT4NDynAe32AeDC4vps=
 github.com/hashicorp/terraform-plugin-sdk/v2 v2.7.0 h1:SuI59MqNjYDrL7EfqHX9V6P/24isgqYx/FdglwVs9bg=
 github.com/hashicorp/terraform-plugin-sdk/v2 v2.7.0/go.mod h1:grseeRo9g3yNkYW09iFlV8LG78jTa1ssBgouogQg/RU=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
@@ -271,6 +277,8 @@ github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.0.4 h1:ZU1VNC02qyufSZsjjs7+khruk2fKvbQ3TwRV/IBCeFA=
 github.com/mitchellh/go-testing-interface v1.0.4/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
+github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
@@ -481,6 +489,7 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

pkg/project/provider.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/url"
 	"regexp"
+	"time"
 
 	"github.com/go-resty/resty/v2"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
@@ -86,7 +87,10 @@ func buildResty(URL string) (*resty.Client, error) {
 		SetHeader("content-type", "application/json").
 		SetHeader("accept", "*/*").
 		SetHeader("user-agent", "jfrog/terraform-provider-project:"+Version).
-		SetRetryCount(5)
+		SetRetryCount(20).
+		SetRetryWaitTime(5 * time.Second).
+		SetRetryMaxWaitTime(20 * time.Second).
+		AddRetryCondition(retryOnServiceUnavailable)
 
 	restyBase.DisableWarn = true
 

pkg/project/resource_project.go

@@ -217,10 +217,19 @@ func projectResource() *schema.Resource {
 		},
 
 		"repos": {
-			Type:        schema.TypeSet,
-			Optional:    true,
-			Elem:        &schema.Schema{Type: schema.TypeString},
-			Description: "List of existing repo keys to be assigned to the project.",
+			Type:     schema.TypeSet,
+			Optional: true,
+			Elem: &schema.Schema{
+				Type: schema.TypeString,
+			},
+			MinItems: 0,
+			MaxItems: func() int {
+				if isOverride := getBoolEnvVar("REPO_LIMIT_OVERRIDE", false); isOverride {
+					return 2147483647
+				}
+				return 100
+			}(),
+			Description: "(Optional) List of existing repo keys to be assigned to the project.",
 		},
 	}
 

pkg/project/resource_project_repo.go

@@ -130,7 +130,11 @@ var updateRepos = func(projectKey string, terraformRepoKeys []RepoKey, m interfa
 var addRepo = func(projectKey string, repoKey RepoKey, m interface{}) error {
 	log.Println("[DEBUG] addRepo")
 
-	_, err := m.(*resty.Client).R().
+	_, err := m.(*resty.Client).
+		R().
+		AddRetryCondition(retryOnSpecificMsgBody("A timeout occurred")).
+		AddRetryCondition(retryOnSpecificMsgBody("Web server is down")).
+		AddRetryCondition(retryOnSpecificMsgBody("Web server is returning an unknown error")).
 		SetPathParams(map[string]string{
 			"projectKey": projectKey,
 			"repoKey":    string(repoKey),
@@ -156,13 +160,13 @@ var deleteRepos = func(projectKey string, repoKeys []RepoKey, m interface{}, g *
 var deleteRepo = func(projectKey string, repoKey RepoKey, m interface{}) error {
 	log.Println("[DEBUG] deleteRepo")
 
-	_, err := m.(*resty.Client).R().
+	_, err := m.(*resty.Client).
+		R().
+		AddRetryCondition(retryOnSpecificMsgBody("A timeout occurred")).
+		AddRetryCondition(retryOnSpecificMsgBody("Web server is down")).
+		AddRetryCondition(retryOnSpecificMsgBody("Web server is returning an unknown error")).
 		SetPathParam("repoKey", string(repoKey)).
 		Delete(projectsUrl + "/_/attach/repositories/{repoKey}")
 
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return err
 }

pkg/project/resource_project_repo_test.go

@@ -1,7 +1,9 @@
 package project
 
 import (
+	"encoding/json"
 	"fmt"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -114,3 +116,128 @@ func TestAccProjectRepo(t *testing.T) {
 		},
 	})
 }
+
+/*
+Test to assign large number of repositories to a project
+*/
+func TestAccAssignMultipleReposInProject(t *testing.T) {
+
+	const numRepos = 100
+	const repoNameInitial = "repo-"
+
+	name := "tftestprojects" + randSeq(10)
+	resourceName := "project." + name
+	projectKey := strings.ToLower(randSeq(6))
+
+	randomRepoNames := func(repoCount int) []string {
+		var repoNames []string
+		for i := 0; i < repoCount; i++ {
+			repoNames = append(repoNames, fmt.Sprintf("%s%s", repoNameInitial, randSeq(10)))
+		}
+		return repoNames
+	}(numRepos)
+
+	repoNamesStr := func(repoNames []string) string {
+		jsonByteArr, err := json.Marshal(repoNames)
+		if err != nil {
+			return "[]"
+		}
+		return string(jsonByteArr)
+	}
+
+	preCheck := func(t *testing.T, repoNames []string) func() {
+		return func() {
+			testAccPreCheck(t)
+			for _, repoName := range repoNames {
+				createTestRepo(t, repoName)
+			}
+		}
+	}
+
+	params := map[string]interface{}{
+		"name":        name,
+		"project_key": projectKey,
+		"repos":       repoNamesStr(randomRepoNames),
+	}
+
+	initialConfig := executeTemplate("TestAccProjectRepo", `
+		resource "project" "{{ .name }}" {
+			key = "{{ .project_key }}"
+			display_name = "{{ .name }}"
+			description = "test description"
+			admin_privileges {
+				manage_members = true
+				manage_resources = true
+				index_resources = true
+			}
+		}
+	`, params)
+
+	addRepoConfig := executeTemplate("TestAccProjectRepo", `
+		resource "project" "{{ .name }}" {
+			key = "{{ .project_key }}"
+			display_name = "{{ .name }}"
+			description = "test description"
+			admin_privileges {
+				manage_members = true
+				manage_resources = true
+				index_resources = true
+			}
+			repos = {{ .repos }}
+		}
+	`, params)
+
+	noReposConfig := executeTemplate("TestAccProjectRepo", `
+		resource "project" "{{ .name }}" {
+			key = "{{ .project_key }}"
+			display_name = "{{ .name }}"
+			description = "test description"
+			admin_privileges {
+				manage_members = true
+				manage_resources = true
+				index_resources = true
+			}
+		}
+	`, params)
+
+	resource.Test(t, resource.TestCase{
+		PreCheck: preCheck(t, randomRepoNames),
+		CheckDestroy: verifyDeleted(resourceName, func(id string, request *resty.Request) (*resty.Response, error) {
+			for _, repoName := range randomRepoNames {
+				deleteTestRepo(t, repoName)
+			}
+			resp, err := verifyProject(id, request)
+			return resp, err
+		}),
+		ProviderFactories: testAccProviders(),
+		Steps: []resource.TestStep{
+			{
+				Config: initialConfig,
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr(resourceName, "key", fmt.Sprintf("%s", params["project_key"])),
+					resource.TestCheckResourceAttr(resourceName, "description", "test description"),
+					resource.TestCheckNoResourceAttr(resourceName, "repos"),
+					resource.TestCheckResourceAttr(resourceName, "repos.#", "0"),
+				),
+			},
+			{
+				Config: addRepoConfig,
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr(resourceName, "key", fmt.Sprintf("%s", params["project_key"])),
+					resource.TestCheckResourceAttr(resourceName, "display_name", name),
+					resource.TestCheckResourceAttr(resourceName, "description", "test description"),
+					resource.TestCheckResourceAttr(resourceName, "repos.#", strconv.Itoa(numRepos)),
+				),
+			},
+			{
+				Config: noReposConfig,
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr(resourceName, "key", fmt.Sprintf("%s", params["project_key"])),
+					resource.TestCheckResourceAttr(resourceName, "display_name", name),
+					resource.TestCheckResourceAttr(resourceName, "description", "test description"),
+					resource.TestCheckNoResourceAttr(resourceName, "repos"),
+				),
+			},
+		},
+	})
+}

pkg/project/util.go

@@ -4,8 +4,13 @@ import (
 	"bytes"
 	"log"
 	"math"
+	"net/http"
+	"os"
+	"regexp"
+	"strconv"
 	"text/template"
 
+	"github.com/go-resty/resty/v2"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 )
 
@@ -175,3 +180,25 @@ var intersection = apply(func(bs []Equatable, a Equatable) bool {
 var difference = apply(func(bs []Equatable, a Equatable) bool {
 	return !contains(bs, a)
 })
+
+var getBoolEnvVar = func(key string, fallback bool) bool {
+	value, exists := os.LookupEnv(key)
+	if exists {
+		boolValue, err := strconv.ParseBool(value)
+		if err == nil {
+			return boolValue
+		}
+	}
+	return fallback
+}
+
+func retryOnSpecificMsgBody(matchString string) func(response *resty.Response, err error) bool {
+	return func(response *resty.Response, err error) bool {
+		var responseBodyRegex = regexp.MustCompile(matchString)
+		return responseBodyRegex.MatchString(string(response.Body()[:]))
+	}
+}
+
+var retryOnServiceUnavailable = func(response *resty.Response, err error) bool {
+	return response.StatusCode() == http.StatusServiceUnavailable
+}

pkg/project/util_test.go

@@ -226,16 +226,28 @@ func createTestRepo(t *testing.T, name string) {
 		RClass: "local",
 	}
 
-	_, err := restyClient.R().SetBody(repo).Put("/artifactory/api/repositories/" + name)
+	_, err := restyClient.
+		R().
+		AddRetryCondition(retryOnSpecificMsgBody("A timeout occurred")).
+		AddRetryCondition(retryOnSpecificMsgBody("Web server is down")).
+		AddRetryCondition(retryOnSpecificMsgBody("Web server is returning an unknown error")).
+		SetBody(repo).
+		Put("/artifactory/api/repositories/" + name)
+
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func deleteTestRepo(t *testing.T, name string) {
 	restyClient := getTestResty(t)
+	_, err := restyClient.
+		R().
+		AddRetryCondition(retryOnSpecificMsgBody("A timeout occurred")).
+		AddRetryCondition(retryOnSpecificMsgBody("Web server is down")).
+		AddRetryCondition(retryOnSpecificMsgBody("Web server is returning an unknown error")).
+		Delete("/artifactory/api/repositories/" + name)
 
-	_, err := restyClient.R().Delete("/artifactory/api/repositories/" + name)
 	if err != nil {
 		t.Fatal(err)
 	}