repository: consolidate changes for common types to support importer-…

…msgraph-metadata

tools/data-api-repository/repository/helpers/filesystem.go

@@ -69,23 +69,30 @@ func (f *FileSystem) Stage(path FilePath, body any) error {
 	return fmt.Errorf("internal-error: unexpected file extension %q for %q", fileExtension, path)
 }
 
-func PersistFileSystem(workingDirectory string, dataType models.SourceDataType, serviceName string, input *FileSystem, logger hclog.Logger) error {
-	// TODO: note this is going to need to take SourceDataOrigin into account too
+func PersistFileSystem(workingDirectory string, dataType models.SourceDataType, relativeOutputDirectory string, serviceName *string, input *FileSystem, logger hclog.Logger) error {
+	// TODO: note this is going to need to take SourceDataOrigin into account too (manicminer: should it? SourceDataOrigin seems to be a separate concern?)
 
 	rootDir := filepath.Join(workingDirectory, string(dataType))
 	logger.Trace(fmt.Sprintf("Persisting files into %q", rootDir))
 
+	outputDir := filepath.Join(rootDir, relativeOutputDirectory)
+
 	// Delete any existing directory with this service name
-	serviceDir := filepath.Join(rootDir, serviceName)
-	logger.Debug(fmt.Sprintf("Removing any existing Directory for Service %q", serviceName))
-	_ = os.RemoveAll(serviceDir)
-	if err := os.MkdirAll(serviceDir, directoryPermissions); err != nil {
-		return fmt.Errorf("recreating directory %q: %+v", serviceDir, err)
+	// TODO: (manicminer: is this really needed since there is a RemoveService() method to achieve this? we don't always want to blow away the entire service)
+	//logging.Log.Debug(fmt.Sprintf("Removing any existing Directory for Service %q", serviceName))
+	//_ = os.RemoveAll(outputDir)
+
+	if err := os.MkdirAll(outputDir, directoryPermissions); err != nil {
+		return fmt.Errorf("recreating directory %q: %+v", outputDir, err)
 	}
 
 	// pull out a list of directories
 	directories := uniqueDirectories(input.f)
-	logger.Debug(fmt.Sprintf("Creating directories for Service %q", serviceName))
+	if serviceName != nil {
+		logger.Debug(fmt.Sprintf("Creating directories for Service %q", *serviceName))
+	} else {
+		logger.Debug("Creating directories")
+	}
 	for _, dir := range directories {
 		dirPath := filepath.Join(rootDir, dir)
 		if err := os.MkdirAll(dirPath, directoryPermissions); err != nil {

tools/importer-msgraph-metadata/components/dataapigeneratorjson/remove_common_types.go → tools/data-api-repository/repository/remove_common_types.go

@@ -1,7 +1,7 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: MPL-2.0
 
-package dataapigeneratorjson
+package repository
 
 import (
 	"fmt"

tools/data-api-repository/repository/remove_service.go

@@ -20,13 +20,21 @@ type RemoveServiceOptions struct {
 
 	// SourceDataType specifies the type of the source data (e.g. ResourceManagerSourceDataType).
 	SourceDataType models.SourceDataType
+
+	// Version is an optional API version to remove. When omitted, all versions are removed.
+	Version string
 }
 
 // RemoveService removes any existing API Definitions for the Service specified in opts.
 func (r repositoryImpl) RemoveService(opts RemoveServiceOptions) error {
 	// TODO: note this is going to need to take SourceDataOrigin into account too
 
 	serviceDirectory := path.Join(r.workingDirectory, string(opts.SourceDataType), opts.ServiceName)
+
+	if opts.Version != "" {
+		serviceDirectory = path.Join(serviceDirectory, opts.Version)
+	}
+
 	if err := os.RemoveAll(serviceDirectory); err != nil && os.IsNotExist(err) {
 		return fmt.Errorf("removing any existing directory at %q: %+v", serviceDirectory, err)
 	}

tools/data-api-repository/repository/repository.go

@@ -8,6 +8,12 @@ import "github.com/hashicorp/go-hclog"
 // Repository is an interface defining how to load and save API Definitions from disk.
 // This interface is designed to allow the implementation to be switched out for testing purposes if needed.
 type Repository interface {
+	// RemoveCommonTypes removes any existing Common Types Definitions>
+	RemoveCommonTypes(opts RemoveCommonTypesOptions) error
+
+	// SaveCommonTypes persists the Common Types Definitions.
+	SaveCommonTypes(opts SaveCommonTypesOptions) error
+
 	// RemoveService removes any existing API Definitions for the Service specified in opts.
 	RemoveService(opts RemoveServiceOptions) error
 

tools/importer-msgraph-metadata/components/dataapigeneratorjson/save_common_types.go → tools/data-api-repository/repository/save_common_types.go

@@ -1,31 +1,30 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: MPL-2.0
 
-package dataapigeneratorjson
+package repository
 
 import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/hashicorp/pandora/tools/data-api-repository/repository/helpers"
+	"github.com/hashicorp/pandora/tools/data-api-repository/repository/stages"
 	"github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
-	"github.com/hashicorp/pandora/tools/importer-msgraph-metadata/components/dataapigeneratorjson/helpers"
-	"github.com/hashicorp/pandora/tools/importer-msgraph-metadata/components/dataapigeneratorjson/stages"
-	"github.com/hashicorp/pandora/tools/importer-msgraph-metadata/internal/logging"
 )
 
 const commonTypesDirectoryName = "common-types"
 
 type SaveCommonTypesOptions struct {
-	// AzureRestAPISpecsGitSHA specifies the Git Commit SHA that these API Definitions were imported from.
-	AzureRestAPISpecsGitSHA *string
-
 	// ResourceProvider optionally specifies the Azure Resource Provider associated with this Service.
 	// This is only present when SourceDataType is ResourceManagerSourceDataType.
 	ResourceProvider *string
 
 	// CommonTypes specifies the CommonTypes to be saved.
 	CommonTypes map[string]models.CommonTypes
 
+	// SourceCommitSHA specifies the Git Commit SHA that these API Definitions were imported from.
+	SourceCommitSHA *string
+
 	// SourceDataOrigin specifies the origin of this set of source data (e.g. AzureRestAPISpecsSourceDataOrigin).
 	SourceDataOrigin models.SourceDataOrigin
 
@@ -35,12 +34,12 @@ type SaveCommonTypesOptions struct {
 
 // SaveCommonTypes persists the Common Types Definitions for the specified version.
 func (r repositoryImpl) SaveCommonTypes(opts SaveCommonTypesOptions) error {
-	logging.Log.Info("Processing Common Types")
+	r.logger.Info("Processing Common Types")
 
 	items := make([]stages.Stage, 0)
 
 	for apiVersion, commonTypes := range opts.CommonTypes {
-		logging.Log.Info(fmt.Sprintf("Processing Common Types for API Version %q..", apiVersion))
+		r.logger.Info(fmt.Sprintf("Processing Common Types for API Version %q..", apiVersion))
 		items = append(items, stages.ConstantStage{
 			Constants:       commonTypes.Constants,
 			OutputDirectory: filepath.Join(commonTypesDirectoryName, apiVersion),
@@ -57,16 +56,16 @@ func (r repositoryImpl) SaveCommonTypes(opts SaveCommonTypesOptions) error {
 
 	fs := helpers.NewFileSystem()
 
-	logging.Log.Debug("Running stages..")
+	r.logger.Debug("Running stages..")
 	for _, stage := range items {
-		logging.Log.Trace(fmt.Sprintf("Processing Stage %q", stage.Name()))
-		if err := stage.Generate(fs); err != nil {
+		r.logger.Trace(fmt.Sprintf("Processing Stage %q", stage.Name()))
+		if err := stage.Generate(fs, r.logger); err != nil {
 			return fmt.Errorf("running Stage %q: %+v", stage.Name(), err)
 		}
 	}
 
-	logging.Log.Debug("Persisting files to disk..")
-	if err := helpers.PersistFileSystem(r.workingDirectory, opts.SourceDataType, commonTypesDirectoryName, nil, fs); err != nil {
+	r.logger.Debug("Persisting files to disk..")
+	if err := helpers.PersistFileSystem(r.workingDirectory, opts.SourceDataType, commonTypesDirectoryName, nil, fs, r.logger); err != nil {
 		return fmt.Errorf("persisting files: %+v", err)
 	}
 

tools/data-api-repository/repository/save_service.go

@@ -5,13 +5,18 @@ package repository
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/hashicorp/pandora/tools/data-api-repository/repository/helpers"
 	"github.com/hashicorp/pandora/tools/data-api-repository/repository/stages"
 	"github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
 )
 
 type SaveServiceOptions struct {
+	// CommonTypes specifies a map of API Version (key) to CommonTypes (value)
+	// which defines the available Common Types for this Service.
+	CommonTypes map[string]models.CommonTypes
+
 	// ResourceProvider optionally specifies the Azure Resource Provider associated with this Service.
 	// This is only present when SourceDataType is ResourceManagerSourceDataType.
 	ResourceProvider *string
@@ -65,25 +70,24 @@ func (r repositoryImpl) SaveService(opts SaveServiceOptions) error {
 			// Output the API Definitions for this APIResource
 
 			items = append(items, stages.ConstantStage{
-				ServiceName: opts.ServiceName,
-				APIVersion:  apiVersion,
-				APIResource: apiResourceName,
-				Constants:   apiResourceDetails.Constants,
-				ResourceIDs: apiResourceDetails.ResourceIDs,
+				Constants:       apiResourceDetails.Constants,
+				OutputDirectory: filepath.Join(opts.ServiceName, apiVersion, apiResourceName),
+				ResourceIDs:     apiResourceDetails.ResourceIDs,
 			})
 
 			items = append(items, stages.ModelsStage{
-				ServiceName: opts.ServiceName,
-				APIVersion:  apiVersion,
-				APIResource: apiResourceName,
-				Constants:   apiResourceDetails.Constants,
-				Models:      apiResourceDetails.Models,
+				APIVersion:      apiVersion,
+				CommonTypes:     opts.CommonTypes,
+				Constants:       apiResourceDetails.Constants,
+				Models:          apiResourceDetails.Models,
+				OutputDirectory: filepath.Join(opts.ServiceName, apiVersion, apiResourceName),
 			})
 
 			items = append(items, stages.OperationsStage{
 				ServiceName: opts.ServiceName,
 				APIVersion:  apiVersion,
 				APIResource: apiResourceName,
+				CommonTypes: opts.CommonTypes,
 				Constants:   apiResourceDetails.Constants,
 				Models:      apiResourceDetails.Models,
 				Operations:  apiResourceDetails.Operations,
@@ -137,7 +141,7 @@ func (r repositoryImpl) SaveService(opts SaveServiceOptions) error {
 	// TODO: ensure that any existing directory for this service is removed
 
 	r.logger.Debug("Persisting files to disk..")
-	if err := helpers.PersistFileSystem(r.workingDirectory, opts.SourceDataType, opts.ServiceName, fs, r.logger); err != nil {
+	if err := helpers.PersistFileSystem(r.workingDirectory, opts.SourceDataType, opts.ServiceName, &opts.ServiceName, fs, r.logger); err != nil {
 		return fmt.Errorf("persisting files: %+v", err)
 	}
 

tools/data-api-repository/repository/stages/constants.go

@@ -16,22 +16,16 @@ import (
 var _ Stage = ConstantStage{}
 
 type ConstantStage struct {
-	// APIVersion specifies the APIVersion within the Service where the Constants exist.
-	APIVersion string
-
-	// APIResource specifies the APIResource within the APIVersion where the Constants exist.
-	APIResource string
-
 	// Constants specifies the map of Constant Name (key) to SDKConstant (value) which should be
 	// persisted.
 	Constants map[string]models.SDKConstant
 
+	// OutputDirectory specifies the path where constants should be persisted.
+	OutputDirectory string
+
 	// ResourceIDs specifies a map of Resource ID Name (key) to ResourceID (value) that should
 	// be persisted.
 	ResourceIDs map[string]models.ResourceID
-
-	// ServiceName specifies the name of the Service within which the Constants exist.
-	ServiceName string
 }
 
 func (g ConstantStage) Name() string {
@@ -41,6 +35,10 @@ func (g ConstantStage) Name() string {
 func (g ConstantStage) Generate(input *helpers.FileSystem, logger hclog.Logger) error {
 	logger.Debug("Generating Constants")
 
+	if g.OutputDirectory == "" {
+		return fmt.Errorf("internal: OutputDirectory cannot be empty")
+	}
+
 	for constantName, constantVal := range g.Constants {
 		logger.Trace(fmt.Sprintf("Processing Constant %q", constantName))
 
@@ -49,8 +47,7 @@ func (g ConstantStage) Generate(input *helpers.FileSystem, logger hclog.Logger)
 			return fmt.Errorf("mapping SDKConstant %q: %+v", constantName, err)
 		}
 
-		// {workingDirectory}/Service/APIVersion/APIResource/Constant-{Name}.json
-		path := filepath.Join(g.ServiceName, g.APIVersion, g.APIResource, fmt.Sprintf("Constant-%s.json", constantName))
+		path := filepath.Join(g.OutputDirectory, fmt.Sprintf("Constant-%s.json", constantName))
 		logger.Trace(fmt.Sprintf("Staging to %s", path))
 		if err := input.Stage(path, *mapped); err != nil {
 			return fmt.Errorf("staging Constant %q: %+v", constantName, err)
@@ -69,8 +66,7 @@ func (g ConstantStage) Generate(input *helpers.FileSystem, logger hclog.Logger)
 				return fmt.Errorf("mapping SDKConstant %q: %+v", constantName, err)
 			}
 
-			// {workingDirectory}/Service/APIVersion/APIResource/Constant-{Name}.json
-			path := filepath.Join(g.ServiceName, g.APIVersion, g.APIResource, fmt.Sprintf("Constant-%s.json", constantName))
+			path := filepath.Join(g.OutputDirectory, fmt.Sprintf("Constant-%s.json", constantName))
 			logger.Trace(fmt.Sprintf("Staging to %s", path))
 			if err := input.Stage(path, *mapped); err != nil {
 				return fmt.Errorf("staging Constant %q: %+v", constantName, err)

tools/data-api-repository/repository/stages/models.go

@@ -19,8 +19,9 @@ type ModelsStage struct {
 	// APIVersion specifies the APIVersion within the Service where the Models exist.
 	APIVersion string
 
-	// APIResource specifies the APIResource within the APIVersion where the Models exist.
-	APIResource string
+	// CommonTypes specifies a map of API Version (key) to CommonTypes (value)
+	// which defines the available Common Types for this Service.
+	CommonTypes map[string]models.CommonTypes
 
 	// Constants specifies the map of Constant Name (key) to SDKConstant (value) which should be
 	// persisted.
@@ -30,12 +31,17 @@ type ModelsStage struct {
 	// persisted.
 	Models map[string]models.SDKModel
 
-	// ServiceName specifies the name of the Service within which the Models exist.
-	ServiceName string
+	// OutputDirectory specifies the path where constants should be persisted.
+	OutputDirectory string
 }
 
 func (g ModelsStage) Generate(input *helpers.FileSystem, logger hclog.Logger) error {
 	logger.Debug("Generating Models")
+
+	if g.OutputDirectory == "" {
+		return fmt.Errorf("internal: OutputDirectory cannot be empty")
+	}
+
 	for modelName := range g.Models {
 		logger.Trace(fmt.Sprintf("Generating Model %q..", modelName))
 		modelValue := g.Models[modelName]
@@ -49,15 +55,19 @@ func (g ModelsStage) Generate(input *helpers.FileSystem, logger hclog.Logger) er
 			}
 		}
 
-		mapped, err := transforms.MapSDKModelToRepository(modelName, modelValue, parent, g.Constants, g.Models)
+		var commonTypes models.CommonTypes
+		if commonTypesForVersion, ok := g.CommonTypes[g.APIVersion]; ok {
+			commonTypes = commonTypesForVersion
+		}
+
+		mapped, err := transforms.MapSDKModelToRepository(modelName, modelValue, parent, g.Constants, g.Models, commonTypes)
 		if err != nil {
 			return fmt.Errorf("mapping model %q: %+v", modelName, err)
 		}
 
-		// {workingDirectory}/Service/APIVersion/APIResource/Model-{Name}.json
-		path := filepath.Join(g.ServiceName, g.APIVersion, g.APIResource, fmt.Sprintf("Model-%s.json", modelName))
+		path := filepath.Join(g.OutputDirectory, fmt.Sprintf("Model-%s.json", modelName))
 		logger.Trace(fmt.Sprintf("Staging to %s", path))
-		if err := input.Stage(path, *mapped); err != nil {
+		if err = input.Stage(path, *mapped); err != nil {
 			return fmt.Errorf("staging Model %q: %+v", modelName, err)
 		}
 	}

tools/data-api-repository/repository/stages/operations.go

@@ -30,6 +30,10 @@ type OperationsStage struct {
 	// persisted.
 	Models map[string]models.SDKModel
 
+	// CommonTypes specifies a map of API Version (key) to CommonTypes (value)
+	// which defines the available Common Types for this Service.
+	CommonTypes map[string]models.CommonTypes
+
 	// Operations specifies the map of Operation Name (key) to SDKOperation (value) which should be
 	// persisted.
 	Operations map[string]models.SDKOperation
@@ -44,15 +48,21 @@ func (g OperationsStage) Generate(input *helpers.FileSystem, logger hclog.Logger
 		logger.Trace(fmt.Sprintf("Generating Operation %q..", operationName))
 
 		operationDetails := g.Operations[operationName]
-		mapped, err := transforms.MapSDKOperationToRepository(operationName, operationDetails, g.Constants, g.Models)
+
+		var commonTypes models.CommonTypes
+		if commonTypesForVersion, ok := g.CommonTypes[g.APIVersion]; ok {
+			commonTypes = commonTypesForVersion
+		}
+
+		mapped, err := transforms.MapSDKOperationToRepository(operationName, operationDetails, g.Constants, g.Models, commonTypes)
 		if err != nil {
 			return fmt.Errorf("mapping Operation %q: %+v", operationName, err)
 		}
 
 		// {workingDirectory}/Service/APIVersion/APIResource/Operation-{Name}.json
 		path := filepath.Join(g.ServiceName, g.APIVersion, g.APIResource, fmt.Sprintf("Operation-%s.json", operationName))
 		logger.Trace(fmt.Sprintf("Staging to %s", path))
-		if err := input.Stage(path, *mapped); err != nil {
+		if err = input.Stage(path, *mapped); err != nil {
 			return fmt.Errorf("staging Operation %q: %+v", operationName, err)
 		}
 	}

tools/data-api-repository/repository/transforms/sdk_field.go

@@ -10,9 +10,9 @@ import (
 	sdkModels "github.com/hashicorp/pandora/tools/data-api-sdk/v1/models"
 )
 
-func mapSDKFieldToRepository(fieldName string, fieldDetails sdkModels.SDKField, isTypeHint bool, constants map[string]sdkModels.SDKConstant, knownModels map[string]sdkModels.SDKModel) (*repositoryModels.ModelField, error) {
+func mapSDKFieldToRepository(fieldName string, fieldDetails sdkModels.SDKField, isTypeHint bool, constants map[string]sdkModels.SDKConstant, knownModels map[string]sdkModels.SDKModel, commonTypes sdkModels.CommonTypes) (*repositoryModels.ModelField, error) {
 	// TODO: thread through logging
-	objectDefinition, err := mapSDKObjectDefinitionToRepository(fieldDetails.ObjectDefinition, constants, knownModels)
+	objectDefinition, err := mapSDKObjectDefinitionToRepository(fieldDetails.ObjectDefinition, constants, knownModels, commonTypes)
 	if err != nil {
 		return nil, fmt.Errorf("mapping the ObjectDefinition for field %q: %+v", fieldName, err)
 	}