Adding State Checks for Known Type and Value, and Sensitive Checks (#275

)

* Adding StateCheck interface (#266)

  * Configuring when state checks are executed.
  * Testing that state checks are executed.

* Adding validation to ensure state checks are only defined for config (apply) tests (#266)

* Adding ExpectKnownValue state check (#266)

* Adding ExpectKnownOutputValue state check (#266)

* Adding ExpectKnownOutputValueAtPath state check (#266)

* Modifying ExpectKnown<Value|OutputValue|OutputValueAtPath> to allow for checking of null values (#266)

* Adding ExpectSensitiveValue state check (#266)

* Adding documentation for state checks and null known value check type (#266)

* Adding to the documentation for the custom known value check (#266)

* Adding changelog entries (#266)

* Refactoring to use updated known value check types (#266)

* Correcting documentation for revised naming of known value check types (#266)

* Renaming nul known value check (#266)

* Fixing tests (#266)

* Adding address and path to state check errors (#266)

* Fixing navigation (#266)

* Fixing changelog entries

* Modifying ExpectKnown<Value|OutputValue|OutputValueAtPath> to handle null checking (#266)

* Deprecating ExpectNullOutputValue and ExpectNullOutputValueAtPath plan checks (#266)

* Adding return statements (#266)

* Adding change log entry for deprecation of `ExpectNullOutputValue` and `ExpectNullOutputValueAtPath` plan checks (#266)

* Modifying return value of nullExact.String() (#266)

* Renaming variable (#266)

* Adding comment for Terraform v1.4.6 (#266)

* Adding further tests for null exact known value type check (#266)

* Linting (#266)

* Renaming BoolExact to Bool, and NullExact to Null (#266)

* Removing ConfigStateChecks type (#266)

* Move execution of ConfigStateChecks (#266)

.changes/unreleased/FEATURES-20240111-142126.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Introduced new `statecheck` package with interface and built-in
+  state check functionality'
+time: 2024-01-11T14:21:26.261094Z
+custom:
+  Issue: "275"

.changes/unreleased/FEATURES-20240111-142223.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Added `ExpectKnownValue` state check, which asserts that a given
+  resource attribute has a defined type, and value'
+time: 2024-01-11T14:22:23.072321Z
+custom:
+  Issue: "275"

.changes/unreleased/FEATURES-20240111-142314.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Added `ExpectKnownOutputValue` state check, which asserts that
+  a given output value has a defined type, and value'
+time: 2024-01-11T14:23:14.025585Z
+custom:
+  Issue: "275"

.changes/unreleased/FEATURES-20240111-142353.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Added `ExpectKnownOutputValueAtPath` plan check, which asserts
+  that a given output value at a specified path has a defined type, and value'
+time: 2024-01-11T14:23:53.633255Z
+custom:
+  Issue: "275"

.changes/unreleased/FEATURES-20240111-142544.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Added `ExpectSensitiveValue` built-in state check, which asserts
+  that a given attribute has a sensitive value'
+time: 2024-01-11T14:25:44.598583Z
+custom:
+  Issue: "275"

.changes/unreleased/NOTES-20240122-082628.yaml

@@ -0,0 +1,7 @@
+kind: NOTES
+body: 'plancheck: Deprecated `ExpectNullOutputValue` and `ExpectNullOutputValueAtPath`.
+  Use `ExpectKnownOutputValue` and `ExpectKnownOutputValueAtPath` with 
+  `knownvalue.NullExact` instead'
+time: 2024-01-22T08:26:28.053303Z
+custom:
+  Issue: "275"

helper/resource/state_checks.go

@@ -0,0 +1,29 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package resource
+
+import (
+	"context"
+	"errors"
+
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/mitchellh/go-testing-interface"
+
+	"github.com/hashicorp/terraform-plugin-testing/statecheck"
+)
+
+func runStateChecks(ctx context.Context, t testing.T, state *tfjson.State, stateChecks []statecheck.StateCheck) error {
+	t.Helper()
+
+	var result []error
+
+	for _, stateCheck := range stateChecks {
+		resp := statecheck.CheckStateResponse{}
+		stateCheck.CheckState(ctx, statecheck.CheckStateRequest{State: state}, &resp)
+
+		result = append(result, resp.Error)
+	}
+
+	return errors.Join(result...)
+}

helper/resource/state_checks_test.go

@@ -0,0 +1,22 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package resource
+
+import (
+	"context"
+
+	"github.com/hashicorp/terraform-plugin-testing/statecheck"
+)
+
+var _ statecheck.StateCheck = &stateCheckSpy{}
+
+type stateCheckSpy struct {
+	err    error
+	called bool
+}
+
+func (s *stateCheckSpy) CheckState(ctx context.Context, req statecheck.CheckStateRequest, resp *statecheck.CheckStateResponse) {
+	s.called = true
+	resp.Error = s.err
+}

helper/resource/testing.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/hashicorp/terraform-plugin-testing/config"
 	"github.com/hashicorp/terraform-plugin-testing/plancheck"
+	"github.com/hashicorp/terraform-plugin-testing/statecheck"
 	"github.com/hashicorp/terraform-plugin-testing/terraform"
 	"github.com/hashicorp/terraform-plugin-testing/tfversion"
 
@@ -590,6 +591,13 @@ type TestStep struct {
 	// [plancheck]: https://pkg.go.dev/github.com/hashicorp/terraform-plugin-testing/plancheck
 	RefreshPlanChecks RefreshPlanChecks
 
+	// ConfigStateChecks allow assertions to be made against the state file during a Config (apply) test using a state check.
+	// Custom state checks can be created by implementing the [StateCheck] interface, or by using a StateCheck implementation from the provided [statecheck] package
+	//
+	// [StateCheck]: https://pkg.go.dev/github.com/hashicorp/terraform-plugin-testing/statecheck#StateCheck
+	// [statecheck]: https://pkg.go.dev/github.com/hashicorp/terraform-plugin-testing/statecheck
+	ConfigStateChecks []statecheck.StateCheck
+
 	// PlanOnly can be set to only run `plan` with this configuration, and not
 	// actually apply it. This is useful for ensuring config changes result in
 	// no-op plans

helper/resource/testing_new_config.go

@@ -205,6 +205,26 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint
 				}
 			}
 		}
+
+		// Run state checks
+		if len(step.ConfigStateChecks) > 0 {
+			var state *tfjson.State
+
+			err = runProviderCommand(ctx, t, func() error {
+				var err error
+				state, err = wd.State(ctx)
+				return err
+			}, wd, providers)
+
+			if err != nil {
+				return fmt.Errorf("Error retrieving post-apply, post-refresh state: %w", err)
+			}
+
+			err = runStateChecks(ctx, t, state, step.ConfigStateChecks)
+			if err != nil {
+				return fmt.Errorf("Post-apply refresh state check(s) failed:\n%w", err)
+			}
+		}
 	}
 
 	// Test for perpetual diffs by performing a plan, a refresh, and another plan

helper/resource/testing_new_config_test.go

@@ -10,10 +10,12 @@ import (
 
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
 	"github.com/hashicorp/terraform-plugin-go/tftypes"
+
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/providerserver"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/resource"
 	"github.com/hashicorp/terraform-plugin-testing/plancheck"
+	"github.com/hashicorp/terraform-plugin-testing/statecheck"
 	"github.com/hashicorp/terraform-plugin-testing/tfversion"
 )
 
@@ -717,3 +719,126 @@ func Test_ConfigPlanChecks_PostApplyPostRefresh_Errors(t *testing.T) {
 		},
 	})
 }
+
+func Test_ConfigStateChecks_Called(t *testing.T) {
+	t.Parallel()
+
+	spy1 := &stateCheckSpy{}
+	spy2 := &stateCheckSpy{}
+	UnitTest(t, TestCase{
+		TerraformVersionChecks: []tfversion.TerraformVersionCheck{
+			tfversion.SkipBelow(tfversion.Version1_0_0), // ProtoV6ProviderFactories
+		},
+		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
+			"test": providerserver.NewProviderServer(testprovider.Provider{
+				Resources: map[string]testprovider.Resource{
+					"test_resource": {
+						CreateResponse: &resource.CreateResponse{
+							NewState: tftypes.NewValue(
+								tftypes.Object{
+									AttributeTypes: map[string]tftypes.Type{
+										"id": tftypes.String,
+									},
+								},
+								map[string]tftypes.Value{
+									"id": tftypes.NewValue(tftypes.String, "test"),
+								},
+							),
+						},
+						SchemaResponse: &resource.SchemaResponse{
+							Schema: &tfprotov6.Schema{
+								Block: &tfprotov6.SchemaBlock{
+									Attributes: []*tfprotov6.SchemaAttribute{
+										{
+											Name:     "id",
+											Type:     tftypes.String,
+											Computed: true,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}),
+		},
+		Steps: []TestStep{
+			{
+				Config: `resource "test_resource" "test" {}`,
+				ConfigStateChecks: []statecheck.StateCheck{
+					spy1,
+					spy2,
+				},
+			},
+		},
+	})
+
+	if !spy1.called {
+		t.Error("expected ConfigStateChecks spy1 to be called at least once")
+	}
+
+	if !spy2.called {
+		t.Error("expected ConfigStateChecks spy2 to be called at least once")
+	}
+}
+
+func Test_ConfigStateChecks_Errors(t *testing.T) {
+	t.Parallel()
+
+	spy1 := &stateCheckSpy{}
+	spy2 := &stateCheckSpy{
+		err: errors.New("spy2 check failed"),
+	}
+	spy3 := &stateCheckSpy{
+		err: errors.New("spy3 check failed"),
+	}
+	UnitTest(t, TestCase{
+		TerraformVersionChecks: []tfversion.TerraformVersionCheck{
+			tfversion.SkipBelow(tfversion.Version1_0_0), // ProtoV6ProviderFactories
+		},
+		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
+			"test": providerserver.NewProviderServer(testprovider.Provider{
+				Resources: map[string]testprovider.Resource{
+					"test_resource": {
+						CreateResponse: &resource.CreateResponse{
+							NewState: tftypes.NewValue(
+								tftypes.Object{
+									AttributeTypes: map[string]tftypes.Type{
+										"id": tftypes.String,
+									},
+								},
+								map[string]tftypes.Value{
+									"id": tftypes.NewValue(tftypes.String, "test"),
+								},
+							),
+						},
+						SchemaResponse: &resource.SchemaResponse{
+							Schema: &tfprotov6.Schema{
+								Block: &tfprotov6.SchemaBlock{
+									Attributes: []*tfprotov6.SchemaAttribute{
+										{
+											Name:     "id",
+											Type:     tftypes.String,
+											Computed: true,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}),
+		},
+		Steps: []TestStep{
+			{
+				Config: `resource "test_resource" "test" {}`,
+				ConfigStateChecks: []statecheck.StateCheck{
+					spy1,
+					spy2,
+					spy3,
+				},
+				ExpectError: regexp.MustCompile(`.*?(spy2 check failed)\n.*?(spy3 check failed)`),
+			},
+		},
+	})
+}

helper/resource/teststep_validate.go

@@ -15,7 +15,7 @@ import (
 // testStepValidateRequest contains data for the (TestStep).validate() method.
 type testStepValidateRequest struct {
 	// StepConfiguration contains the TestStep configuration derived from
-	// TestStep.Config or TestStep.ConfigDirectory.
+	// TestStep.Config, TestStep.ConfigDirectory, or TestStep.ConfigFile.
 	StepConfiguration teststep.Config
 
 	// StepNumber is the index of the TestStep in the TestCase.Steps.
@@ -235,5 +235,11 @@ func (s TestStep) validate(ctx context.Context, req testStepValidateRequest) err
 		return err
 	}
 
+	if len(s.ConfigStateChecks) > 0 && req.StepConfiguration == nil {
+		err := fmt.Errorf("TestStep ConfigStateChecks must only be specified with Config, ConfigDirectory or ConfigFile")
+		logging.HelperResourceError(ctx, "TestStep validation error", map[string]interface{}{logging.KeyError: err})
+		return err
+	}
+
 	return nil
 }

helper/resource/teststep_validate_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-testing/config"
 	"github.com/hashicorp/terraform-plugin-testing/internal/teststep"
 	"github.com/hashicorp/terraform-plugin-testing/plancheck"
+	"github.com/hashicorp/terraform-plugin-testing/statecheck"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 )
@@ -466,6 +467,16 @@ func TestTestStepValidate(t *testing.T) {
 			testStepValidateRequest: testStepValidateRequest{TestCaseHasProviders: true},
 			expectedError:           errors.New("TestStep ConfigPlanChecks.PostApplyPostRefresh must only be specified with Config"),
 		},
+		"configstatechecks-not-config-mode": {
+			testStep: TestStep{
+				ConfigStateChecks: []statecheck.StateCheck{
+					&stateCheckSpy{},
+				},
+				RefreshState: true,
+			},
+			testStepValidateRequest: testStepValidateRequest{TestCaseHasProviders: true},
+			expectedError:           errors.New("TestStep ConfigStateChecks must only be specified with Config"),
+		},
 		"refreshplanchecks-postrefresh-not-refresh-mode": {
 			testStep: TestStep{
 				RefreshPlanChecks: RefreshPlanChecks{

knownvalue/bool.go

@@ -8,37 +8,37 @@ import (
 	"strconv"
 )
 
-var _ Check = boolExact{}
+var _ Check = boolValue{}
 
-type boolExact struct {
+type boolValue struct {
 	value bool
 }
 
 // CheckValue determines whether the passed value is of type bool, and
 // contains a matching bool value.
-func (v boolExact) CheckValue(other any) error {
+func (v boolValue) CheckValue(other any) error {
 	otherVal, ok := other.(bool)
 
 	if !ok {
-		return fmt.Errorf("expected bool value for BoolExact check, got: %T", other)
+		return fmt.Errorf("expected bool value for Bool check, got: %T", other)
 	}
 
 	if otherVal != v.value {
-		return fmt.Errorf("expected value %t for BoolExact check, got: %t", v.value, otherVal)
+		return fmt.Errorf("expected value %t for Bool check, got: %t", v.value, otherVal)
 	}
 
 	return nil
 }
 
 // String returns the string representation of the bool value.
-func (v boolExact) String() string {
+func (v boolValue) String() string {
 	return strconv.FormatBool(v.value)
 }
 
-// BoolExact returns a Check for asserting equality between the
+// Bool returns a Check for asserting equality between the
 // supplied bool and the value passed to the CheckValue method.
-func BoolExact(value bool) boolExact {
-	return boolExact{
+func Bool(value bool) boolValue {
+	return boolValue{
 		value: value,
 	}
 }