Support for big access policies (1000+ rules) (#207)

gen/definitions/access_control_policy.yaml

@@ -102,7 +102,6 @@ attributes:
     type: List
     description: The ordered list of categories.
     ordered_list: true
-    max_list: 1000
     attributes:
       - model_name: id
         type: String
@@ -138,7 +137,6 @@ attributes:
       `category_name`. Uncategorized non-mandatory rules must be below all other rules. The first matching rule
       is selected. Except for MONITOR rules, the system does not continue to evaluate traffic against additional rules
       after that traffic matches a rule.
-    max_list: 1000
     attributes:
       - model_name: id
         type: String

internal/provider/model_fmc_access_control_policy.go

@@ -2171,19 +2171,6 @@ func NewValidAccessControlPolicy(ctx context.Context, tfplan tfsdk.Plan) (Access
 		return plan, diags
 	}
 
-	// Validate lenghts. TODO(#66): make lenghts unlimited
-	if len(plan.Categories) > 1000 {
-		diags.AddAttributeError(path.Root("categories"), "Too many categories",
-			fmt.Sprintf("%d categories specified, expected at most %d.", len(plan.Categories), 1000))
-		return plan, diags
-	}
-
-	if len(plan.Rules) > 1000 {
-		diags.AddAttributeError(path.Root("rules"), "Too many rules",
-			fmt.Sprintf("%d rules specified, expected at most %d.", len(plan.Rules), 1000))
-		return plan, diags
-	}
-
 	// Validate categories.*.section
 	def := types.StringNull()
 	insertion := len(plan.Categories)

internal/provider/resource_fmc_access_control_policy.go

@@ -26,7 +26,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/hashicorp/terraform-plugin-framework-validators/listvalidator"
 	"github.com/hashicorp/terraform-plugin-framework-validators/stringvalidator"
 	"github.com/hashicorp/terraform-plugin-framework/diag"
 	"github.com/hashicorp/terraform-plugin-framework/path"
@@ -174,9 +173,6 @@ func (r *AccessControlPolicyResource) Schema(ctx context.Context, req resource.S
 						},
 					},
 				},
-				Validators: []validator.List{
-					listvalidator.SizeAtMost(1000),
-				},
 			},
 			"rules": schema.ListNestedAttribute{
 				MarkdownDescription: helpers.NewAttributeDescription("The ordered list of rules. Rules must be sorted in the order of the corresponding categories, if they have `category_name`. Uncategorized non-mandatory rules must be below all other rules. The first matching rule is selected. Except for MONITOR rules, the system does not continue to evaluate traffic against additional rules after that traffic matches a rule.").String,
@@ -555,9 +551,6 @@ func (r *AccessControlPolicyResource) Schema(ctx context.Context, req resource.S
 						},
 					},
 				},
-				Validators: []validator.List{
-					listvalidator.SizeAtMost(1000),
-				},
 			},
 		},
 	}
@@ -659,13 +652,13 @@ func (r *AccessControlPolicyResource) Read(ctx context.Context, req resource.Rea
 		return
 	}
 
-	resCats, err := r.client.Get(state.getPath()+"/"+url.QueryEscape(state.Id.ValueString())+"/categories?expanded=true&offset=0&limit=1000", reqMods...)
+	resCats, err := r.client.Get(state.getPath()+"/"+url.QueryEscape(state.Id.ValueString())+"/categories?expanded=true", reqMods...)
 	if err != nil {
 		resp.Diagnostics.AddError("Client Error", fmt.Sprintf("Failed to retrieve object (GET), got error: %s, %s", err, resGet.String()))
 		return
 	}
 
-	resRules, err := r.client.Get(state.getPath()+"/"+url.QueryEscape(state.Id.ValueString())+"/accessrules?expanded=true&offset=0&limit=1000", reqMods...)
+	resRules, err := r.client.Get(state.getPath()+"/"+url.QueryEscape(state.Id.ValueString())+"/accessrules?expanded=true", reqMods...)
 	if err != nil {
 		resp.Diagnostics.AddError("Client Error", fmt.Sprintf("Failed to retrieve object (GET), got error: %s, %s", err, resGet.String()))
 		return
@@ -835,7 +828,7 @@ func (r *AccessControlPolicyResource) truncateRulesAt(ctx context.Context, state
 		}
 		b.WriteString(state.Rules[i].Id.ValueString())
 		count++
-		if b.Len() >= 3700-2 {
+		if b.Len() >= maxUrlParamLength {
 			bulks = append(bulks, b.String())
 			b.Reset()
 			counts = append(counts, count)
@@ -915,6 +908,8 @@ func (r *AccessControlPolicyResource) createRulesAt(ctx context.Context, plan Ac
 	for i := startIndex; i < len(body); i++ {
 		bulk := `{"dummy_rules":[]}`
 		j := i
+		bulkCount := 0
+		bodyLength := 0
 		head := plan.Rules[i]
 		for ; i < len(body); i++ {
 			if !head.CategoryName.Equal(plan.Rules[i].CategoryName) || head.GetSection() != plan.Rules[i].GetSection() {
@@ -926,7 +921,20 @@ func (r *AccessControlPolicyResource) createRulesAt(ctx context.Context, plan Ac
 			rule, _ = sjson.Delete(rule, "metadata.category")
 			rule, _ = sjson.Delete(rule, "metadata.section")
 
+			// Check if the body is too big for a single POST
+			bodyLength += len(rule)
+			if bodyLength >= maxPayloadSize {
+				i--
+				break
+			}
+
 			bulk, _ = sjson.SetRaw(bulk, "dummy_rules.-1", rule)
+
+			// Count the number of rules in the bulk
+			bulkCount++
+			if bulkCount >= bulkSizeCreate {
+				break
+			}
 		}
 
 		param := "?bulk=true"