From 38712ff1cfe5ed231d29918b5740cbfa0dbad3b8 Mon Sep 17 00:00:00 2001 From: Tom Bamford Date: Thu, 13 Jul 2023 12:04:32 +0100 Subject: [PATCH] WIP common models --- .../generator/common_types.go | 99 +++++++++++++++++++ tools/generator-go-sdk/generator/data.go | 8 +- tools/generator-go-sdk/generator/helpers.go | 4 +- tools/generator-go-sdk/generator/service.go | 3 +- .../generator/templater_meta_client.go | 4 +- .../generator/templater_methods.go | 3 +- tools/generator-go-sdk/main.go | 68 ++++++++++--- tools/sdk/resourcemanager/client.go | 6 ++ tools/sdk/resourcemanager/common_types.go | 37 +++++++ tools/sdk/resourcemanager/service_version.go | 5 + tools/sdk/services/data.go | 19 +++- tools/sdk/services/resourcemanager.go | 5 + 12 files changed, 235 insertions(+), 26 deletions(-) create mode 100644 tools/generator-go-sdk/generator/common_types.go create mode 100644 tools/sdk/resourcemanager/common_types.go diff --git a/tools/generator-go-sdk/generator/common_types.go b/tools/generator-go-sdk/generator/common_types.go new file mode 100644 index 00000000000..6419bd9c883 --- /dev/null +++ b/tools/generator-go-sdk/generator/common_types.go @@ -0,0 +1,99 @@ +package generator + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/pandora/tools/sdk/resourcemanager" + "github.com/hashicorp/pandora/tools/sdk/services" +) + +type CommonTypesGenerator struct{} + +func NewCommonTypesGenerator() CommonTypesGenerator { + return CommonTypesGenerator{} +} + +type SDKInput struct { + CommonPackageName string + CommonPackagePath string + CommonTypes *services.ResourceManagerCommonTypes + OutputDirectory string + VersionName string +} + +func (s *CommonTypesGenerator) GenerateForSDK(input SDKInput) error { + input.VersionName = strings.ToLower(input.VersionName) + commonTypesDirectory := filepath.Join(input.OutputDirectory, input.CommonPackagePath) + + if err := cleanAndRecreateWorkingDirectory(commonTypesDirectory); err != nil { + return fmt.Errorf("cleaning/recreating working directory %q: %+v", commonTypesDirectory, err) + } + + stages := map[string]func(SDKInput, string) error{ + "commonTypes": s.commonTypes, + } + for name, stage := range stages { + log.Printf("[DEBUG] Running Stage %q..", name) + if err := stage(input, commonTypesDirectory); err != nil { + return fmt.Errorf("generating %s: %+v", name, err) + } + } + + runGoFmt(commonTypesDirectory) + runGoImports(commonTypesDirectory) + return nil +} + +func (s *CommonTypesGenerator) commonTypes(input SDKInput, outputDirectory string) error { + data := ServiceGeneratorData{ + apiVersion: input.VersionName, + constants: input.CommonTypes.Constants, + models: input.CommonTypes.Models, + packageName: "models", + source: resourcemanager.ApiDefinitionsSourceMicrosoftGraphMetadata, + } + + gen := constantsTemplater{ + constantTemplateFunc: templateForConstant, + } + if err := s.writeToPath(outputDirectory, "constants.go", gen, data); err != nil { + return fmt.Errorf("templating constants: %+v", err) + } + + for modelName, model := range input.CommonTypes.Models { + fileName := fmt.Sprintf("model_%s.go", strings.ToLower(modelName)) + gen := modelsTemplater{ + name: modelName, + model: model, + } + if err := s.writeToPath(outputDirectory, fileName, gen, data); err != nil { + return fmt.Errorf("templating model for %q: %+v", modelName, err) + } + } + + return nil +} + +func (s *CommonTypesGenerator) writeToPath(directory, filePath string, templater templaterForResource, data ServiceGeneratorData) error { + fileContents, err := templater.template(data) + if err != nil { + return fmt.Errorf("templating: %+v", err) + } + + fullFilePath := filepath.Join(directory, filePath) + + // remove any existing file if it exists + _ = os.Remove(fullFilePath) + file, err := os.Create(fullFilePath) + defer file.Close() + if err != nil { + return fmt.Errorf("opening %q: %+v", fullFilePath, err) + } + + _, _ = file.WriteString(*fileContents) + return nil +} diff --git a/tools/generator-go-sdk/generator/data.go b/tools/generator-go-sdk/generator/data.go index 075d8cc83ba..e7264d91ec7 100644 --- a/tools/generator-go-sdk/generator/data.go +++ b/tools/generator-go-sdk/generator/data.go @@ -58,13 +58,14 @@ type ServiceGeneratorData struct { baseClientMethod string baseClientPackage string + commonPackagePath string } func (i ServiceGeneratorInput) generatorData(settings Settings) ServiceGeneratorData { - servicePackageName := golangPackageName(i.ServiceName) - versionPackageName := golangPackageName(i.VersionName) + servicePackageName := GolangPackageName(i.ServiceName) + versionPackageName := GolangPackageName(i.VersionName) // TODO: it'd be nice to make these snake_case but that's a problem for another day - resourcePackageName := golangPackageName(i.ResourceName) + resourcePackageName := GolangPackageName(i.ResourceName) versionOutputPath := filepath.Join(i.OutputDirectory, servicePackageName, versionPackageName) resourceOutputPath := filepath.Join(versionOutputPath, resourcePackageName) idsPath := filepath.Join(versionOutputPath, "ids") @@ -75,6 +76,7 @@ func (i ServiceGeneratorInput) generatorData(settings Settings) ServiceGenerator apiVersion: i.VersionName, baseClientMethod: i.BaseClientMethod, baseClientPackage: i.BaseClientPackage, + commonPackagePath: i.CommonPackagePath, constants: i.ResourceDetails.Schema.Constants, idsOutputPath: idsPath, models: i.ResourceDetails.Schema.Models, diff --git a/tools/generator-go-sdk/generator/helpers.go b/tools/generator-go-sdk/generator/helpers.go index 8146c15785e..e25a9c4b4cc 100644 --- a/tools/generator-go-sdk/generator/helpers.go +++ b/tools/generator-go-sdk/generator/helpers.go @@ -109,14 +109,14 @@ func golangTypeNameForConstantType(input resourcemanager.ConstantType) (*string, return &segmentType, nil } -func golangPackageName(input string) (output string) { +func GolangPackageName(input string) (output string) { output = input output = regexp.MustCompile("[^0-9A-z-]").ReplaceAllString(output, "_") output = strings.ToLower(output) return } -func golangPackageNameForVersion(input string) (output string) { +func GolangPackageNameForVersion(input string) (output string) { output = input output = regexp.MustCompile("[^0-9A-z]").ReplaceAllString(output, "_") output = strings.ToLower(output) diff --git a/tools/generator-go-sdk/generator/service.go b/tools/generator-go-sdk/generator/service.go index 1136d4c518a..5342b8cf361 100644 --- a/tools/generator-go-sdk/generator/service.go +++ b/tools/generator-go-sdk/generator/service.go @@ -31,6 +31,7 @@ type ServiceGeneratorInput struct { ResourceDetails services.Resource BaseClientMethod string BaseClientPackage string + CommonPackagePath string OutputDirectory string Source resourcemanager.ApiDefinitionsSource } @@ -84,7 +85,7 @@ type VersionInput struct { func (s *ServiceGenerator) GenerateForVersion(input VersionInput) error { input.ServiceName = strings.ToLower(input.ServiceName) input.VersionName = strings.ToLower(input.VersionName) - versionDirectory := filepath.Join(input.OutputDirectory, golangPackageName(input.ServiceName), golangPackageName(input.VersionName)) + versionDirectory := filepath.Join(input.OutputDirectory, GolangPackageName(input.ServiceName), GolangPackageName(input.VersionName)) stages := map[string]func(data VersionInput, versionDirectory string) error{ "metaClient": s.metaClient, diff --git a/tools/generator-go-sdk/generator/templater_meta_client.go b/tools/generator-go-sdk/generator/templater_meta_client.go index 6156e8361b4..2559b433fe4 100644 --- a/tools/generator-go-sdk/generator/templater_meta_client.go +++ b/tools/generator-go-sdk/generator/templater_meta_client.go @@ -37,7 +37,7 @@ func (m metaClientTemplater) template() (*string, error) { for _, resourceName := range resourceNames { variableName := fmt.Sprintf("%s%sClient", strings.ToLower(string(resourceName[0])), resourceName[1:]) - imports = append(imports, fmt.Sprintf(`"github.com/hashicorp/go-azure-sdk/%s/%s/%s/%s"`, m.sdkPackage, golangPackageName(m.serviceName), golangPackageName(m.apiVersion), golangPackageName(resourceName))) + imports = append(imports, fmt.Sprintf(`"github.com/hashicorp/go-azure-sdk/%s/%s/%s/%s"`, m.sdkPackage, GolangPackageName(m.serviceName), GolangPackageName(m.apiVersion), GolangPackageName(resourceName))) fields = append(fields, fmt.Sprintf("%[1]s *%[2]s.%[1]sClient", resourceName, strings.ToLower(resourceName))) clientInitializationTemplate := fmt.Sprintf(`%[1]s, err := %[2]s.New%[3]sClientWithBaseURI(api) if err != nil { @@ -54,7 +54,7 @@ configureFunc(%[1]s.Client) sort.Strings(fields) sort.Strings(imports) - packageName := golangPackageNameForVersion(m.apiVersion) + packageName := GolangPackageNameForVersion(m.apiVersion) out := fmt.Sprintf(`package %[1]s diff --git a/tools/generator-go-sdk/generator/templater_methods.go b/tools/generator-go-sdk/generator/templater_methods.go index d444c305694..49ef728b15c 100644 --- a/tools/generator-go-sdk/generator/templater_methods.go +++ b/tools/generator-go-sdk/generator/templater_methods.go @@ -40,12 +40,13 @@ import ( "github.com/hashicorp/go-azure-sdk/sdk/client/pollers" "github.com/hashicorp/go-azure-sdk/sdk/client/resourcemanager" "github.com/hashicorp/go-azure-sdk/sdk/odata" + "github.com/hashicorp/go-azure-sdk/%[4]s" ) %[2]s %[3]s -`, data.packageName, *copyrightLines, *methods) +`, data.packageName, *copyrightLines, *methods, data.commonPackagePath) return &template, nil } diff --git a/tools/generator-go-sdk/main.go b/tools/generator-go-sdk/main.go index 995932b6aa4..7b02eb59a63 100644 --- a/tools/generator-go-sdk/main.go +++ b/tools/generator-go-sdk/main.go @@ -24,19 +24,24 @@ type GeneratorInput struct { } type SDK struct { - baseClientMethod string - baseClientPackage string - clients []func(string) resourcemanager.Client - outputSubDirectory string - versionMapper func(string) (*string, error) + baseClientMethod string + baseClientPackage string + clients map[string]func(string) resourcemanager.Client + commonModelsPackageFromVersion func(string) (*string, error) + outputSubDirectory string + versionMapper func(string) (*string, error) } var availableSDKs = map[string]SDK{ "resource-manager": { baseClientMethod: "NewResourceManagerClient", baseClientPackage: "resourcemanager", - clients: []func(string) resourcemanager.Client{ - resourcemanager.NewResourceManagerClient, + clients: map[string]func(string) resourcemanager.Client{ + "resource-manager": resourcemanager.NewResourceManagerClient, + }, + commonModelsPackageFromVersion: func(in string) (*string, error) { + path := "common/models" + return &path, nil }, outputSubDirectory: "resource-manager", versionMapper: func(in string) (*string, error) { return &in, nil }, @@ -45,9 +50,13 @@ var availableSDKs = map[string]SDK{ "microsoft-graph": { baseClientMethod: "NewMsGraphClient", baseClientPackage: "msgraph", - clients: []func(string) resourcemanager.Client{ - resourcemanager.NewMicrosoftGraphStableV1Client, - resourcemanager.NewMicrosoftGraphBetaClient, + clients: map[string]func(string) resourcemanager.Client{ + "v1.0": resourcemanager.NewMicrosoftGraphStableV1Client, + "beta": resourcemanager.NewMicrosoftGraphBetaClient, + }, + commonModelsPackageFromVersion: func(in string) (*string, error) { + path := fmt.Sprintf("common/%s/models", fmt.Sprintf(generator.GolangPackageNameForVersion(in))) + return &path, nil }, outputSubDirectory: "microsoft-graph", versionMapper: func(in string) (*string, error) { @@ -200,9 +209,13 @@ func run(input GeneratorInput) error { input.outputDirectory = path.Join(input.outputDirectory, sdk.outputSubDirectory) - for _, apiClient := range sdk.clients { + for clientName, apiClient := range sdk.clients { client := apiClient(input.apiServerEndpoint) + commonTypes, err := services.GetResourceManagerCommonTypes(client) + if err != nil { + return fmt.Errorf("retrieving resource manager common types: %+v", err) + } var loadedServices services.ResourceManagerServices if len(input.services) > 0 { log.Printf("[DEBUG] Loading the Services from the Data API %q..", strings.Join(input.services, " / ")) @@ -231,6 +244,30 @@ func run(input GeneratorInput) error { } } + wg.Add(1) + go func(input GeneratorInput) { + defer wg.Done() + log.Printf("[DEBUG] Common Types for %q..", clientName) + commonPackagePath, err := sdk.commonModelsPackageFromVersion(clientName) + if err != nil { + addErr(fmt.Errorf("generating Common Types for %q: %+v", clientName, err)) + return + } + generatorTypes := generator.NewCommonTypesGenerator() + generatorData := generator.SDKInput{ + CommonTypes: commonTypes, + OutputDirectory: input.outputDirectory, + CommonPackagePath: *commonPackagePath, + VersionName: clientName, + } + log.Printf("[DEBUG] Generating Common Types..") + if err := generatorTypes.GenerateForSDK(generatorData); err != nil { + addErr(fmt.Errorf("generating common types: %+v", err)) + return + } + log.Printf("[DEBUG] Generated Common Types..") + }(input) + generatorService := generator.NewServiceGenerator(input.settings) for serviceName, service := range loadedServices.Services { log.Printf("[DEBUG] Service %q..", serviceName) @@ -245,22 +282,25 @@ func run(input GeneratorInput) error { log.Printf("[DEBUG] Service %q", serviceName) for versionNumber, versionDetails := range service.Versions { log.Printf("[DEBUG] Version %q", versionNumber) - versionName, err := sdk.versionMapper(versionNumber) + + commonPackagePath, err := sdk.commonModelsPackageFromVersion(clientName) if err != nil { addErr(fmt.Errorf("generating Service %q / Version %q: %+v", serviceName, versionNumber, err)) return } + for resourceName, resourceDetails := range versionDetails.Resources { log.Printf("[DEBUG] Resource %q..", resourceName) generatorData := generator.ServiceGeneratorInput{ ServiceName: serviceName, ServiceDetails: service, - VersionName: *versionName, + VersionName: clientName, VersionDetails: versionDetails, ResourceName: resourceName, ResourceDetails: resourceDetails, BaseClientMethod: sdk.baseClientMethod, BaseClientPackage: sdk.baseClientPackage, + CommonPackagePath: *commonPackagePath, OutputDirectory: input.outputDirectory, Source: versionDetails.Details.Source, } @@ -276,7 +316,7 @@ func run(input GeneratorInput) error { generatorData := generator.VersionInput{ OutputDirectory: input.outputDirectory, ServiceName: serviceName, - VersionName: *versionName, + VersionName: clientName, BaseClientPackage: sdk.baseClientPackage, SdkPackage: sdk.outputSubDirectory, Resources: versionDetails.Resources, diff --git a/tools/sdk/resourcemanager/client.go b/tools/sdk/resourcemanager/client.go index 23080be1608..bd1f75261c4 100644 --- a/tools/sdk/resourcemanager/client.go +++ b/tools/sdk/resourcemanager/client.go @@ -52,6 +52,12 @@ func (c Client) ApiSchema() ApiSchemaClient { } } +func (c Client) CommonTypes() CommonTypesClient { + return CommonTypesClient{ + Client: c, + } +} + func (c Client) ServiceDetails() ServiceDetailsClient { return ServiceDetailsClient{ Client: c, diff --git a/tools/sdk/resourcemanager/common_types.go b/tools/sdk/resourcemanager/common_types.go new file mode 100644 index 00000000000..76d162e4de7 --- /dev/null +++ b/tools/sdk/resourcemanager/common_types.go @@ -0,0 +1,37 @@ +package resourcemanager + +import ( + "encoding/json" + "fmt" +) + +type CommonTypesClient struct { + Client +} + +func (c CommonTypesClient) Get() (*CommonTypesDetails, error) { + endpoint := fmt.Sprintf("%s%s/commonTypes", c.endpoint, c.apiEndpoint) + resp, err := c.client.Get(endpoint) + if err != nil { + return nil, err + } + + // TODO: handle this being a 404 etc + + var response CommonTypesDetails + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, err + } + + return &response, nil +} + +type CommonTypesDetails struct { + // Constants is a map of key (Constant Name) to value (ConstantDetails) describing + // each common Constant supported by this API + Constants map[string]ConstantDetails `json:"constants"` + + // Models is a map of key (Model Name) to value (ModelDetails) describing + // each common Model supported by this API + Models map[string]ModelDetails `json:"models"` +} diff --git a/tools/sdk/resourcemanager/service_version.go b/tools/sdk/resourcemanager/service_version.go index 47d0af769c8..a884883cb66 100644 --- a/tools/sdk/resourcemanager/service_version.go +++ b/tools/sdk/resourcemanager/service_version.go @@ -57,4 +57,9 @@ const ( // that this set of API Definitions is based on data within the Azure // Rest API Specs repository. ApiDefinitionsSourceResourceManagerRestApiSpecs ApiDefinitionsSource = "ResourceManagerRestApiSpecs" + + // ApiDefinitionsSourceMicrosoftGraphMetadata is used to signify that + // this set of API Definitions is based on data within the + // Microsoft-Graph/MSGraph-Metadata repository. + ApiDefinitionsSourceMicrosoftGraphMetadata ApiDefinitionsSource = "MicrosoftGraphMetadata" ) diff --git a/tools/sdk/services/data.go b/tools/sdk/services/data.go index 2efe7864195..4ef187a3f76 100644 --- a/tools/sdk/services/data.go +++ b/tools/sdk/services/data.go @@ -6,7 +6,7 @@ import ( "github.com/hashicorp/pandora/tools/sdk/resourcemanager" ) -// GetResourceManagerServices returns all of the Services supported by the Resource Manager endpoint +// GetResourceManagerServices returns all the Services supported by the Resource Manager endpoint func GetResourceManagerServices(client resourcemanager.Client) (*ResourceManagerServices, error) { return GetResourceManagerServicesByName(client, nil) } @@ -18,9 +18,9 @@ func GetResourceManagerServicesByName(client resourcemanager.Client, servicesToL return nil, err } - serviceNames := make(map[string]struct{}) + serviceNames := make(map[string]bool) for _, serviceName := range servicesToLoad { - serviceNames[serviceName] = struct{}{} + serviceNames[serviceName] = true } resourceManagerServices := make(map[string]ResourceManagerService, 0) @@ -121,3 +121,16 @@ func GetResourceManagerServicesByName(client resourcemanager.Client, servicesToL Services: resourceManagerServices, }, nil } + +// GetResourceManagerCommonTypes returns the specified Services from the Data API +func GetResourceManagerCommonTypes(client resourcemanager.Client) (*ResourceManagerCommonTypes, error) { + commonTypes, err := client.CommonTypes().Get() + if err != nil { + return nil, err + } + + return &ResourceManagerCommonTypes{ + Constants: commonTypes.Constants, + Models: commonTypes.Models, + }, nil +} diff --git a/tools/sdk/services/resourcemanager.go b/tools/sdk/services/resourcemanager.go index dd4697f8fab..7269b2124bb 100644 --- a/tools/sdk/services/resourcemanager.go +++ b/tools/sdk/services/resourcemanager.go @@ -22,3 +22,8 @@ type Resource struct { Operations resourcemanager.ApiOperationDetails Schema resourcemanager.ApiSchemaDetails } + +type ResourceManagerCommonTypes struct { + Constants map[string]resourcemanager.ConstantDetails + Models map[string]resourcemanager.ModelDetails +}