feat: Add import support for ethernet and add additional specs (#130)

* Update ethernet layer3 imports specification

* Implement the new location import API

* Terraform part of importing resources

* Mark datasource attributes other than location and name as computed

* Update to specs and codegen to support more resources (#135)

* Update administrative tag and add all constant values

* Add panorama template variable specification

Also, update terraform generator to use always use defined suffix
to generate resource names, and not the name provided in the spec.

* Add device group parent specification and custom codegen

* Fix sensitive not being properly set for terraform attributes (#136)

pkg/commands/codegen/codegen.go

@@ -119,6 +119,8 @@ func (c *Command) Execute() error {
 					resourceTyp = properties.ResourceEntry
 				case properties.TerraformResourceUuid:
 					resourceTyp = properties.ResourceUuid
+				case properties.TerraformResourceCustom:
+					resourceTyp = properties.ResourceCustom
 				case properties.TerraformResourceConfig:
 					panic("missing implementation for config type resources")
 				}
@@ -140,6 +142,8 @@ func (c *Command) Execute() error {
 					resourceTyp = properties.ResourceEntryPlural
 				case properties.TerraformResourceUuid:
 					resourceTyp = properties.ResourceUuidPlural
+				case properties.TerraformResourceCustom:
+					resourceTyp = properties.ResourceCustom
 				case properties.TerraformResourceConfig:
 					panic("missing implementation for config type resources")
 				}
@@ -153,7 +157,7 @@ func (c *Command) Execute() error {
 				resourceList = append(resourceList, resources...)
 				dataSourceList = append(dataSourceList, dataSources...)
 			}
-		} else if c.commandType == properties.CommandTypeSDK {
+		} else if c.commandType == properties.CommandTypeSDK && !spec.GoSdkSkip {
 			generator := generate.NewCreator(config.Output.GoSdk, c.templatePath, spec)
 			if err = generator.RenderTemplate(); err != nil {
 				return fmt.Errorf("error rendering %s - %s", specPath, err)

pkg/generate/generator.go

@@ -64,7 +64,7 @@ func (c *Creator) RenderTerraformProviderFile(spec *properties.Normalization, ty
 		name = fmt.Sprintf("%s_%s", spec.TerraformProviderConfig.Suffix, spec.TerraformProviderConfig.PluralName)
 	case properties.ResourceEntryPlural:
 		name = spec.TerraformProviderConfig.PluralSuffix
-	case properties.ResourceEntry, properties.ResourceUuid:
+	case properties.ResourceEntry, properties.ResourceUuid, properties.ResourceCustom:
 		name = spec.Name
 	}
 
@@ -95,7 +95,7 @@ func (c *Creator) RenderTerraformProviderFile(spec *properties.Normalization, ty
 	case properties.ResourceEntryPlural:
 		name = spec.TerraformProviderConfig.PluralSuffix
 		filePath = c.createTerraformProviderFilePath(name)
-	case properties.ResourceEntry, properties.ResourceUuid:
+	case properties.ResourceEntry, properties.ResourceUuid, properties.ResourceCustom:
 		filePath = c.createTerraformProviderFilePath(spec.TerraformProviderConfig.Suffix)
 	}
 
@@ -243,6 +243,7 @@ func (c *Creator) parseTemplate(templateName string) (*template.Template, error)
 	templatePath := filepath.Join(c.TemplatesDir, templateName)
 	funcMap := template.FuncMap{
 		"renderImports":             translate.RenderImports,
+		"RenderEntryImportStructs":  func() (string, error) { return translate.RenderEntryImportStructs(c.Spec) },
 		"packageName":               translate.PackageName,
 		"locationType":              translate.LocationType,
 		"specParamType":             translate.SpecParamType,

pkg/properties/normalized.go

@@ -18,22 +18,43 @@ import (
 type Normalization struct {
 	Name                    string                  `json:"name" yaml:"name"`
 	TerraformProviderConfig TerraformProviderConfig `json:"terraform_provider_config" yaml:"terraform_provider_config"`
+	GoSdkSkip               bool                    `json:"go_sdk_skip" yaml:"go_sdk_skip"`
 	GoSdkPath               []string                `json:"go_sdk_path" yaml:"go_sdk_path"`
 	XpathSuffix             []string                `json:"xpath_suffix" yaml:"xpath_suffix"`
 	Locations               map[string]*Location    `json:"locations" yaml:"locations"`
 	Entry                   *Entry                  `json:"entry" yaml:"entry"`
-	Imports                 map[string]*Import      `json:"imports" yaml:"imports"`
+	Imports                 []Import                `json:"imports" yaml:"imports"`
 	Version                 string                  `json:"version" yaml:"version"`
 	Spec                    *Spec                   `json:"spec" yaml:"spec"`
 	Const                   map[string]*Const       `json:"const" yaml:"const"`
 }
 
+type Import struct {
+	Variant   *NameVariant
+	Type      *NameVariant
+	Locations map[string]ImportLocation
+}
+
+type ImportLocation struct {
+	Name           *NameVariant
+	Required       bool
+	XpathElements  []string
+	XpathVariables map[string]ImportXpathVariable
+}
+
+type ImportXpathVariable struct {
+	Name        *NameVariant
+	Description string
+	Default     string
+}
+
 type TerraformResourceType string
 
 const (
 	TerraformResourceEntry  TerraformResourceType = "entry"
 	TerraformResourceUuid   TerraformResourceType = "uuid"
 	TerraformResourceConfig TerraformResourceType = "config"
+	TerraformResourceCustom TerraformResourceType = "custom"
 )
 
 type TerraformResourceVariant string
@@ -48,6 +69,7 @@ type TerraformProviderConfig struct {
 	SkipDatasource        bool                       `json:"skip_datasource" yaml:"skip_datasource"`
 	SkipDatasourceListing bool                       `json:"skip_datasource_listing" yaml:"skip_datasource_listing"`
 	ResourceType          TerraformResourceType      `json:"resource_type" yaml:"resource_type"`
+	CustomFuncs           map[string]string          `json:"custom_functions" yaml:"custom_functions"`
 	ResourceVariants      []TerraformResourceVariant `json:"resource_variants" yaml:"resource_variants"`
 	Suffix                string                     `json:"suffix" yaml:"suffix"`
 	PluralSuffix          string                     `json:"plural_suffix" yaml:"plural_suffix"`
@@ -60,6 +82,14 @@ type NameVariant struct {
 	LowerCamelCase string
 }
 
+func NewNameVariant(name string) *NameVariant {
+	return &NameVariant{
+		Underscore:     naming.Underscore("", name, ""),
+		CamelCase:      naming.CamelCase("", name, "", true),
+		LowerCamelCase: naming.CamelCase("", name, "", false),
+	}
+}
+
 type Location struct {
 	Name        *NameVariant
 	Description string                  `json:"description" yaml:"description"`
@@ -90,19 +120,6 @@ type Entry struct {
 	Name *EntryName `json:"name" yaml:"name"`
 }
 
-type Import struct {
-	Name          *NameVariant
-	Xpath         []string              `json:"xpath" yaml:"xpath"`
-	Vars          map[string]*ImportVar `json:"vars" yaml:"vars"`
-	OnlyForParams []string              `json:"only_for_params" yaml:"only_for_params"`
-}
-
-type ImportVar struct {
-	Name        *NameVariant
-	Description string `json:"description" yaml:"description"`
-	Default     string `json:"default" yaml:"default"`
-}
-
 type EntryName struct {
 	Description string           `json:"description" yaml:"description"`
 	Length      *EntryNameLength `json:"length" yaml:"length"`
@@ -371,8 +388,11 @@ func schemaParameterToSpecParameter(schemaSpec *parameter.Parameter) (*SpecParam
 			Type: schemaSpec.Hashing.Type,
 		}
 	}
+
+	var sensitive bool
 	var terraformProviderConfig *SpecParamTerraformProviderConfig
 	if schemaSpec.CodegenOverrides != nil {
+		sensitive = schemaSpec.CodegenOverrides.Terraform.Sensitive
 		terraformProviderConfig = &SpecParamTerraformProviderConfig{
 			Computed: schemaSpec.CodegenOverrides.Terraform.Computed,
 		}
@@ -382,6 +402,7 @@ func schemaParameterToSpecParameter(schemaSpec *parameter.Parameter) (*SpecParam
 		Type:                    specType,
 		Default:                 defaultVal,
 		Required:                schemaSpec.Required,
+		Sensitive:               sensitive,
 		TerraformProviderConfig: terraformProviderConfig,
 		Hashing:                 specHashing,
 		Profiles:                profiles,
@@ -447,19 +468,6 @@ func generateXpathVariables(variables []xpathschema.Variable) map[string]*Locati
 	return xpathVars
 }
 
-func generateImportVariables(variables []xpathschema.Variable) map[string]*ImportVar {
-	importVars := make(map[string]*ImportVar)
-	for _, variable := range variables {
-		entry := &ImportVar{
-			Description: variable.Description,
-			Default:     variable.Default,
-		}
-		importVars[variable.Name] = entry
-	}
-
-	return importVars
-}
-
 func schemaToSpec(object object.Object) (*Normalization, error) {
 	var resourceVariants []TerraformResourceVariant
 	for _, elt := range object.TerraformConfig.ResourceVariants {
@@ -472,13 +480,14 @@ func schemaToSpec(object object.Object) (*Normalization, error) {
 			SkipDatasource:        object.TerraformConfig.SkipDatasource,
 			SkipDatasourceListing: object.TerraformConfig.SkipdatasourceListing,
 			ResourceType:          TerraformResourceType(object.TerraformConfig.ResourceType),
+			CustomFuncs:           object.TerraformConfig.CustomFunctions,
 			ResourceVariants:      resourceVariants,
 			Suffix:                object.TerraformConfig.Suffix,
 			PluralSuffix:          object.TerraformConfig.PluralSuffix,
 			PluralName:            object.TerraformConfig.PluralName,
 		},
 		Locations:   make(map[string]*Location),
-		Imports:     make(map[string]*Import),
+		GoSdkSkip:   object.GoSdkConfig.Skip,
 		GoSdkPath:   object.GoSdkConfig.Package,
 		XpathSuffix: object.XpathSuffix,
 		Version:     object.Version,
@@ -560,39 +569,53 @@ func schemaToSpec(object object.Object) (*Normalization, error) {
 
 	}
 
-	imports := make(map[string]*Import, len(object.Imports))
+	var imports []Import
 	for _, elt := range object.Imports {
-		var xpath []string
-
-		schemaXpathVars := make(map[string]xpathschema.Variable)
-		for _, xpathVariable := range elt.Xpath.Variables {
-			schemaXpathVars[xpathVariable.Name] = xpathVariable
-		}
-
-		for _, element := range elt.Xpath.Elements {
-			var eltEntry string
-			if xpathVar, ok := schemaXpathVars[element[1:]]; ok {
-				if xpathVar.Type == "entry" {
-					eltEntry = fmt.Sprintf("{{ Entry %s }}", elt.Name)
-				} else if xpathVar.Type == "object" {
-					eltEntry = fmt.Sprintf("{{ Object %s }}", elt.Name)
+		locations := make(map[string]ImportLocation, len(elt.Locations))
+		for _, location := range elt.Locations {
+			schemaXpathVars := make(map[string]xpathschema.Variable, len(location.Xpath.Variables))
+			xpathVars := make(map[string]ImportXpathVariable, len(location.Xpath.Variables))
+			for _, xpathVariable := range location.Xpath.Variables {
+				schemaXpathVars[xpathVariable.Name] = xpathVariable
+				xpathVars[xpathVariable.Name] = ImportXpathVariable{
+					Name: &NameVariant{
+						Underscore:     naming.Underscore("", xpathVariable.Name, ""),
+						CamelCase:      naming.CamelCase("", xpathVariable.Name, "", true),
+						LowerCamelCase: naming.CamelCase("", xpathVariable.Name, "", false),
+					},
+					Description: xpathVariable.Description,
+					Default:     xpathVariable.Default,
 				}
-			} else {
-				eltEntry = element
 			}
-			xpath = append(xpath, eltEntry)
-		}
 
-		importVariables := generateImportVariables(elt.Xpath.Variables)
-		if len(importVariables) == 0 {
-			importVariables = nil
-		}
+			var xpath []string
+			xpath = append(xpath, location.Xpath.Elements...)
 
-		imports[elt.Name] = &Import{
-			Xpath:         xpath,
-			Vars:          importVariables,
-			OnlyForParams: elt.OnlyForParams,
+			locations[location.Name] = ImportLocation{
+				Name: &NameVariant{
+					Underscore:     naming.Underscore("", location.Name, ""),
+					CamelCase:      naming.CamelCase("", location.Name, "", true),
+					LowerCamelCase: naming.CamelCase("", location.Name, "", false),
+				},
+				Required:       location.Required,
+				XpathVariables: xpathVars,
+				XpathElements:  xpath,
+			}
 		}
+
+		imports = append(imports, Import{
+			Type: &NameVariant{
+				Underscore:     naming.Underscore("", elt.Type, ""),
+				CamelCase:      naming.CamelCase("", elt.Type, "", true),
+				LowerCamelCase: naming.CamelCase("", elt.Type, "", false),
+			},
+			Variant: &NameVariant{
+				Underscore:     naming.Underscore("", elt.Variant, ""),
+				CamelCase:      naming.CamelCase("", elt.Variant, "", true),
+				LowerCamelCase: naming.CamelCase("", elt.Variant, "", false),
+			},
+			Locations: locations,
+		})
 	}
 
 	if len(imports) > 0 {
@@ -660,11 +683,6 @@ func ParseSpec(input []byte) (*Normalization, error) {
 		return nil, err
 	}
 
-	err = spec.AddNameVariantsForImports()
-	if err != nil {
-		return nil, err
-	}
-
 	err = spec.AddNameVariantsForParams()
 	if err != nil {
 		return nil, err
@@ -704,27 +722,6 @@ func (spec *Normalization) AddNameVariantsForLocation() error {
 	return nil
 }
 
-// AddNameVariantsForImports add name variants for imports (under_score and CamelCase).
-func (spec *Normalization) AddNameVariantsForImports() error {
-	for key, imp := range spec.Imports {
-		imp.Name = &NameVariant{
-			Underscore:     naming.Underscore("", key, ""),
-			CamelCase:      naming.CamelCase("", key, "", true),
-			LowerCamelCase: naming.CamelCase("", key, "", false),
-		}
-
-		for subkey, variable := range imp.Vars {
-			variable.Name = &NameVariant{
-				Underscore:     naming.Underscore("", subkey, ""),
-				CamelCase:      naming.CamelCase("", subkey, "", true),
-				LowerCamelCase: naming.CamelCase("", subkey, "", false),
-			}
-		}
-	}
-
-	return nil
-}
-
 // AddNameVariantsForParams recursively add name variants for params for nested specs.
 func AddNameVariantsForParams(name string, param *SpecParam) error {
 	param.Name = &NameVariant{

pkg/properties/resourcetype.go

@@ -4,6 +4,7 @@ type ResourceType int
 
 const (
 	ResourceEntry       ResourceType = iota
+	ResourceCustom      ResourceType = iota
 	ResourceEntryPlural ResourceType = iota
 	ResourceUuid        ResourceType = iota
 	ResourceUuidPlural  ResourceType = iota

pkg/schema/imports/import.go

@@ -1,9 +1,11 @@
 package imports
 
-import "github.com/paloaltonetworks/pan-os-codegen/pkg/schema/xpath"
+import (
+	"github.com/paloaltonetworks/pan-os-codegen/pkg/schema/location"
+)
 
 type Import struct {
-	Name          string            `yaml:"name"`
-	Xpath         xpathschema.Xpath `yaml:"xpath"`
-	OnlyForParams []string          `yaml:"only_for_params"`
+	Variant   string              `yaml:"variant"`
+	Type      string              `yaml:"type"`
+	Locations []location.Location `yaml:"locations"`
 }

pkg/schema/location/location.go

@@ -14,6 +14,8 @@ const (
 
 type Location struct {
 	Name        string                       `yaml:"name"`
+	Required    bool                         `yaml:"required"`
+	ReadOnly    bool                         `yaml:"read_only"`
 	Description string                       `yaml:"description"`
 	Devices     []Device                     `yaml:"devices"`
 	Xpath       xpathschema.Xpath            `yaml:"xpath"`

pkg/schema/object/object.go

@@ -15,6 +15,7 @@ const (
 	TerraformResourceEntry  TerraformResourceType = "entry"
 	TerraformResourceUuid   TerraformResourceType = "uuid"
 	TerraformResourceConfig TerraformResourceType = "config"
+	TerraformResourceCustom TerraformResourceType = "custom"
 )
 
 type TerraformResourceVariant string
@@ -29,14 +30,16 @@ type TerraformConfig struct {
 	SkipDatasource        bool                       `yaml:"skip_datasource"`
 	SkipdatasourceListing bool                       `yaml:"skip_datasource_listing"`
 	ResourceType          TerraformResourceType      `yaml:"resource_type"`
+	CustomFunctions       map[string]string          `yaml:"custom_functions"`
 	ResourceVariants      []TerraformResourceVariant `yaml:"resource_variants"`
 	Suffix                string                     `yaml:"suffix"`
 	PluralSuffix          string                     `yaml:"plural_suffix"`
 	PluralName            string                     `yaml:"plural_name"`
 }
 
 type GoSdkConfig struct {
-	Package []string
+	Skip    bool     `yaml:"skip"`
+	Package []string `yaml:"package"`
 }
 
 type Entry struct {

pkg/schema/parameter/parameter.go

@@ -43,7 +43,8 @@ type EnumSpecValue struct {
 }
 
 type CodegenOverridesTerraform struct {
-	Computed bool `yaml:"computed"`
+	Sensitive bool `yaml:"sensitive"`
+	Computed  bool `yaml:"computed"`
 }
 
 type CodegenOverrides struct {

pkg/translate/imports.go

@@ -47,6 +47,8 @@ func RenderImports(templateTypes ...string) (string, error) {
 			manager.AddSdkImport("github.com/PaloAltoNetworks/pango/version", "")
 		case "template":
 			manager.AddSdkImport("github.com/PaloAltoNetworks/pango/panorama/template", "")
+		case "imports":
+			manager.AddStandardImport("encoding/xml", "")
 		}
 	}