Adding State Checks for Known Type and Value, and Sensitive Checks

.changes/unreleased/FEATURES-20240111-142126.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Introduced new `statecheck` package with interface and built-in
+  state check functionality'
+time: 2024-01-11T14:21:26.261094Z
+custom:
+  Issue: "275"

.changes/unreleased/FEATURES-20240111-142223.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Added `ExpectKnownValue` state check, which asserts that a given
+  resource attribute has a defined type, and value'
+time: 2024-01-11T14:22:23.072321Z
+custom:
+  Issue: "275"

.changes/unreleased/FEATURES-20240111-142314.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Added `ExpectKnownOutputValue` state check, which asserts that
+  a given output value has a defined type, and value'
+time: 2024-01-11T14:23:14.025585Z
+custom:
+  Issue: "275"

.changes/unreleased/FEATURES-20240111-142353.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Added `ExpectKnownOutputValueAtPath` plan check, which asserts
+  that a given output value at a specified path has a defined type, and value'
+time: 2024-01-11T14:23:53.633255Z
+custom:
+  Issue: "275"

.changes/unreleased/FEATURES-20240111-142544.yaml

@@ -0,0 +1,6 @@
+kind: FEATURES
+body: 'statecheck: Added `ExpectSensitiveValue` built-in state check, which asserts
+  that a given attribute has a sensitive value'
+time: 2024-01-11T14:25:44.598583Z
+custom:
+  Issue: "275"

.changes/unreleased/NOTES-20240122-082628.yaml

@@ -0,0 +1,7 @@
+kind: NOTES
+body: 'plancheck: Deprecated `ExpectNullOutputValue` and `ExpectNullOutputValueAtPath`.
+  Use `ExpectKnownOutputValue` and `ExpectKnownOutputValueAtPath` with 
+  `knownvalue.NullExact` instead'
+time: 2024-01-22T08:26:28.053303Z
+custom:
+  Issue: "275"

helper/resource/state_checks.go

@@ -0,0 +1,29 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package resource
+
+import (
+	"context"
+	"errors"
+
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/mitchellh/go-testing-interface"
+
+	"github.com/hashicorp/terraform-plugin-testing/statecheck"
+)
+
+func runStateChecks(ctx context.Context, t testing.T, state *tfjson.State, stateChecks []statecheck.StateCheck) error {
+	t.Helper()
+
+	var result []error
+
+	for _, stateCheck := range stateChecks {
+		resp := statecheck.CheckStateResponse{}
+		stateCheck.CheckState(ctx, statecheck.CheckStateRequest{State: state}, &resp)
+
+		result = append(result, resp.Error)
+	}
+
+	return errors.Join(result...)
+}

helper/resource/state_checks_test.go

@@ -0,0 +1,22 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package resource
+
+import (
+	"context"
+
+	"github.com/hashicorp/terraform-plugin-testing/statecheck"
+)
+
+var _ statecheck.StateCheck = &stateCheckSpy{}
+
+type stateCheckSpy struct {
+	err    error
+	called bool
+}
+
+func (s *stateCheckSpy) CheckState(ctx context.Context, req statecheck.CheckStateRequest, resp *statecheck.CheckStateResponse) {
+	s.called = true
+	resp.Error = s.err
+}

helper/resource/testing.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/hashicorp/terraform-plugin-testing/config"
 	"github.com/hashicorp/terraform-plugin-testing/plancheck"
+	"github.com/hashicorp/terraform-plugin-testing/statecheck"
 	"github.com/hashicorp/terraform-plugin-testing/terraform"
 	"github.com/hashicorp/terraform-plugin-testing/tfversion"
 
@@ -590,6 +591,13 @@ type TestStep struct {
 	// [plancheck]: https://pkg.go.dev/github.com/hashicorp/terraform-plugin-testing/plancheck
 	RefreshPlanChecks RefreshPlanChecks
 
+	// ConfigStateChecks allow assertions to be made against the state file at different points of a Config (apply) test using a state check.
+	// Custom state checks can be created by implementing the [StateCheck] interface, or by using a StateCheck implementation from the provided [statecheck] package
+	//
+	// [StateCheck]: https://pkg.go.dev/github.com/hashicorp/terraform-plugin-testing/statecheck#StateCheck
+	// [statecheck]: https://pkg.go.dev/github.com/hashicorp/terraform-plugin-testing/statecheck
+	ConfigStateChecks ConfigStateChecks
+
 	// PlanOnly can be set to only run `plan` with this configuration, and not
 	// actually apply it. This is useful for ensuring config changes result in
 	// no-op plans
@@ -795,6 +803,10 @@ type RefreshPlanChecks struct {
 	PostRefresh []plancheck.PlanCheck
 }
 
+// ConfigStateChecks runs all state checks in the slice. This occurs after the apply and refresh of a Config test are run.
+// All errors by state checks in this slice are aggregated, reported, and will result in a test failure.
+type ConfigStateChecks []statecheck.StateCheck
+
 // ParallelTest performs an acceptance test on a resource, allowing concurrency
 // with other ParallelTest. The number of concurrent tests is controlled by the
 // "go test" command -parallel flag.

helper/resource/testing_new_config.go

@@ -313,6 +313,26 @@ func testStepNewConfig(ctx context.Context, t testing.T, c TestCase, wd *plugint
 		}
 	}
 
+	// Run post-apply, post-refresh state checks
+	if len(step.ConfigStateChecks) > 0 {
+		var state *tfjson.State
+
+		err = runProviderCommand(ctx, t, func() error {
+			var err error
+			state, err = wd.State(ctx)
+			return err
+		}, wd, providers)
+
+		if err != nil {
+			return fmt.Errorf("Error retrieving post-apply, post-refresh state: %w", err)
+		}
+
+		err = runStateChecks(ctx, t, state, step.ConfigStateChecks)
+		if err != nil {
+			return fmt.Errorf("Post-apply refresh state check(s) failed:\n%w", err)
+		}
+	}
+
 	// check if plan is empty
 	if !planIsEmpty(plan, helper.TerraformVersion()) && !step.ExpectNonEmptyPlan {
 		var stdout string

helper/resource/testing_new_config_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/hashicorp/terraform-plugin-go/tfprotov6"
 	"github.com/hashicorp/terraform-plugin-go/tftypes"
+
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testprovider"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/providerserver"
 	"github.com/hashicorp/terraform-plugin-testing/internal/testing/testsdk/resource"
@@ -717,3 +718,126 @@ func Test_ConfigPlanChecks_PostApplyPostRefresh_Errors(t *testing.T) {
 		},
 	})
 }
+
+func Test_ConfigStateChecks_Called(t *testing.T) {
+	t.Parallel()
+
+	spy1 := &stateCheckSpy{}
+	spy2 := &stateCheckSpy{}
+	UnitTest(t, TestCase{
+		TerraformVersionChecks: []tfversion.TerraformVersionCheck{
+			tfversion.SkipBelow(tfversion.Version1_0_0), // ProtoV6ProviderFactories
+		},
+		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
+			"test": providerserver.NewProviderServer(testprovider.Provider{
+				Resources: map[string]testprovider.Resource{
+					"test_resource": {
+						CreateResponse: &resource.CreateResponse{
+							NewState: tftypes.NewValue(
+								tftypes.Object{
+									AttributeTypes: map[string]tftypes.Type{
+										"id": tftypes.String,
+									},
+								},
+								map[string]tftypes.Value{
+									"id": tftypes.NewValue(tftypes.String, "test"),
+								},
+							),
+						},
+						SchemaResponse: &resource.SchemaResponse{
+							Schema: &tfprotov6.Schema{
+								Block: &tfprotov6.SchemaBlock{
+									Attributes: []*tfprotov6.SchemaAttribute{
+										{
+											Name:     "id",
+											Type:     tftypes.String,
+											Computed: true,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}),
+		},
+		Steps: []TestStep{
+			{
+				Config: `resource "test_resource" "test" {}`,
+				ConfigStateChecks: ConfigStateChecks{
+					spy1,
+					spy2,
+				},
+			},
+		},
+	})
+
+	if !spy1.called {
+		t.Error("expected ConfigStateChecks spy1 to be called at least once")
+	}
+
+	if !spy2.called {
+		t.Error("expected ConfigStateChecks spy2 to be called at least once")
+	}
+}
+
+func Test_ConfigStateChecks_Errors(t *testing.T) {
+	t.Parallel()
+
+	spy1 := &stateCheckSpy{}
+	spy2 := &stateCheckSpy{
+		err: errors.New("spy2 check failed"),
+	}
+	spy3 := &stateCheckSpy{
+		err: errors.New("spy3 check failed"),
+	}
+	UnitTest(t, TestCase{
+		TerraformVersionChecks: []tfversion.TerraformVersionCheck{
+			tfversion.SkipBelow(tfversion.Version1_0_0), // ProtoV6ProviderFactories
+		},
+		ProtoV6ProviderFactories: map[string]func() (tfprotov6.ProviderServer, error){
+			"test": providerserver.NewProviderServer(testprovider.Provider{
+				Resources: map[string]testprovider.Resource{
+					"test_resource": {
+						CreateResponse: &resource.CreateResponse{
+							NewState: tftypes.NewValue(
+								tftypes.Object{
+									AttributeTypes: map[string]tftypes.Type{
+										"id": tftypes.String,
+									},
+								},
+								map[string]tftypes.Value{
+									"id": tftypes.NewValue(tftypes.String, "test"),
+								},
+							),
+						},
+						SchemaResponse: &resource.SchemaResponse{
+							Schema: &tfprotov6.Schema{
+								Block: &tfprotov6.SchemaBlock{
+									Attributes: []*tfprotov6.SchemaAttribute{
+										{
+											Name:     "id",
+											Type:     tftypes.String,
+											Computed: true,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}),
+		},
+		Steps: []TestStep{
+			{
+				Config: `resource "test_resource" "test" {}`,
+				ConfigStateChecks: ConfigStateChecks{
+					spy1,
+					spy2,
+					spy3,
+				},
+				ExpectError: regexp.MustCompile(`.*?(spy2 check failed)\n.*?(spy3 check failed)`),
+			},
+		},
+	})
+}

helper/resource/teststep_validate.go

@@ -15,7 +15,7 @@ import (
 // testStepValidateRequest contains data for the (TestStep).validate() method.
 type testStepValidateRequest struct {
 	// StepConfiguration contains the TestStep configuration derived from
-	// TestStep.Config or TestStep.ConfigDirectory.
+	// TestStep.Config, TestStep.ConfigDirectory, or TestStep.ConfigFile.
 	StepConfiguration teststep.Config
 
 	// StepNumber is the index of the TestStep in the TestCase.Steps.
@@ -235,5 +235,11 @@ func (s TestStep) validate(ctx context.Context, req testStepValidateRequest) err
 		return err
 	}
 
+	if len(s.ConfigStateChecks) > 0 && req.StepConfiguration == nil {
+		err := fmt.Errorf("TestStep ConfigStateChecks must only be specified with Config, ConfigDirectory or ConfigFile")
+		logging.HelperResourceError(ctx, "TestStep validation error", map[string]interface{}{logging.KeyError: err})
+		return err
+	}
+
 	return nil
 }

helper/resource/teststep_validate_test.go

@@ -466,6 +466,16 @@ func TestTestStepValidate(t *testing.T) {
 			testStepValidateRequest: testStepValidateRequest{TestCaseHasProviders: true},
 			expectedError:           errors.New("TestStep ConfigPlanChecks.PostApplyPostRefresh must only be specified with Config"),
 		},
+		"configstatechecks-not-config-mode": {
+			testStep: TestStep{
+				ConfigStateChecks: ConfigStateChecks{
+					&stateCheckSpy{},
+				},
+				RefreshState: true,
+			},
+			testStepValidateRequest: testStepValidateRequest{TestCaseHasProviders: true},
+			expectedError:           errors.New("TestStep ConfigStateChecks must only be specified with Config"),
+		},
 		"refreshplanchecks-postrefresh-not-refresh-mode": {
 			testStep: TestStep{
 				RefreshPlanChecks: RefreshPlanChecks{

knownvalue/null.go

@@ -0,0 +1,32 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package knownvalue
+
+import (
+	"fmt"
+)
+
+var _ Check = nullExact{}
+
+type nullExact struct{}
+
+// CheckValue determines whether the passed value is nil.
+func (v nullExact) CheckValue(other any) error {
+	if other != nil {
+		return fmt.Errorf("expected value nil for NullExact check, got: %T", other)
+	}
+
+	return nil
+}
+
+// String returns the string representation of null.
+func (v nullExact) String() string {
+	return "null"
+}
+
+// NullExact returns a Check for asserting equality nil
+// and the value passed to the CheckValue method.
+func NullExact() nullExact {
+	return nullExact{}
+}