moving jsonschema generator

cmd/jsonschema-generator/main.go

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build ignore
+
+// This program will utilize the json/jsonschema tags
+// on structs in compiler/types/yaml to generate the
+// majority of the final jsonschema for a
+// Vela pipeline.
+//
+// Some manual intervention is needed for custom types
+// and/or custom marshalling that is in place. For reference
+// we use the provided mechanisms, see:
+// https://github.com/invopop/jsonschema?tab=readme-ov-file#custom-type-definitions
+// for hooking into the schema generation process. Some
+// types will have a JSONSchema or JSONSchemaExtend method
+// attached to handle the overrides.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/invopop/jsonschema"
+	"github.com/sirupsen/logrus"
+
+	"github.com/go-vela/server/compiler/types/yaml"
+)
+
+func main() {
+	ref := jsonschema.Reflector{
+		ExpandedStruct: true,
+	}
+	d := ref.Reflect(yaml.Build{})
+
+	if d == nil {
+		logrus.Fatal("reflection failed")
+	}
+
+	d.Title = "Vela Pipeline Configuration"
+
+	// allows folks to have other top level arbitrary
+	// keys without validation errors
+	d.AdditionalProperties = nil
+
+	// rules can currently live at ruleset level or
+	// nested within 'if' (default) or 'unless'.
+	// without changes the struct would only allow
+	// the nested version.
+	//
+	// note: we have to do the modification here,
+	// because the custom type hooks can't provide
+	// access to the top level definitions, even if
+	// they were already processed, so we have to
+	// do it at this top level.
+	ruleSetWithRules := *d.Definitions["Rules"]
+	ruleSet := *d.Definitions["Ruleset"]
+
+	// copy every property from Ruleset, other than `if` and `unless`
+	for item := ruleSet.Properties.Newest(); item != nil; item = item.Prev() {
+		if item.Key != "if" && item.Key != "unless" {
+			ruleSetWithRules.Properties.Set(item.Key, item.Value)
+		}
+	}
+
+	// create a new definition for Ruleset
+	d.Definitions["Ruleset"].AnyOf = []*jsonschema.Schema{
+		&ruleSet,
+		&ruleSetWithRules,
+	}
+	d.Definitions["Ruleset"].Properties = nil
+	d.Definitions["Ruleset"].Type = ""
+	d.Definitions["Ruleset"].AdditionalProperties = nil
+
+	// output json
+	j, err := json.MarshalIndent(d, "", "  ")
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	fmt.Printf("%s\n", j)
+}

compiler/types/raw/map.go

@@ -7,6 +7,8 @@ import (
 	"encoding/json"
 	"errors"
 	"strings"
+
+	"github.com/invopop/jsonschema"
 )
 
 // StringSliceMap represents an array of strings or a map of strings.
@@ -138,3 +140,27 @@ func (s *StringSliceMap) UnmarshalYAML(unmarshal func(interface{}) error) error
 
 	return errors.New("unable to unmarshal into StringSliceMap")
 }
+
+// JSONSchema handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Without these changes it would only allow a map of string,
+// but we do some special handling to support array of strings.
+func (StringSliceMap) JSONSchema() *jsonschema.Schema {
+	return &jsonschema.Schema{
+		OneOf: []*jsonschema.Schema{
+			{
+				Type: "array",
+				Items: &jsonschema.Schema{
+					Type: "string",
+				},
+			},
+			{
+				Type: "object",
+				AdditionalProperties: &jsonschema.Schema{
+					Type: "string",
+				},
+			},
+		},
+	}
+}

compiler/types/raw/slice.go

@@ -5,6 +5,8 @@ package raw
 import (
 	"encoding/json"
 	"errors"
+
+	"github.com/invopop/jsonschema"
 )
 
 // StringSlice represents a string or an array of strings.
@@ -71,3 +73,24 @@ func (s *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 	return errors.New("unable to unmarshal into StringSlice")
 }
+
+// JSONSchema handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Without these changes it would only allow an array of strings,
+// but we do some special handling to support plain string also.
+func (StringSlice) JSONSchema() *jsonschema.Schema {
+	return &jsonschema.Schema{
+		OneOf: []*jsonschema.Schema{
+			{
+				Type: "string",
+			},
+			{
+				Type: "array",
+				Items: &jsonschema.Schema{
+					Type: "string",
+				},
+			},
+		},
+	}
+}

compiler/types/yaml/ruleset.go

@@ -3,6 +3,8 @@
 package yaml
 
 import (
+	"github.com/invopop/jsonschema"
+
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
 	"github.com/go-vela/server/constants"
@@ -22,13 +24,15 @@ type (
 	// Rules is the yaml representation of the ruletypes
 	// from a ruleset block for a step in a pipeline.
 	Rules struct {
-		Branch   []string `yaml:"branch,omitempty,flow"   json:"branch,omitempty"   jsonschema:"description=Limits the execution of a step to matching build branches.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Comment  []string `yaml:"comment,omitempty,flow"  json:"comment,omitempty"  jsonschema:"description=Limits the execution of a step to matching a pull request comment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Event    []string `yaml:"event,omitempty,flow"    json:"event,omitempty"    jsonschema:"description=Limits the execution of a step to matching build events.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Path     []string `yaml:"path,omitempty,flow"     json:"path,omitempty"     jsonschema:"description=Limits the execution of a step to matching files changed in a repository.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Repo     []string `yaml:"repo,omitempty,flow"     json:"repo,omitempty"     jsonschema:"description=Limits the execution of a step to matching repos.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Sender   []string `yaml:"sender,omitempty,flow"   json:"sender,omitempty"   jsonschema:"description=Limits the execution of a step to matching build senders.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
-		Status   []string `yaml:"status,omitempty,flow"   json:"status,omitempty"   jsonschema:"enum=[failure],enum=[success],description=Limits the execution of a step to matching build statuses.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Branch  []string `yaml:"branch,omitempty,flow"  json:"branch,omitempty"  jsonschema:"description=Limits the execution of a step to matching build branches.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Comment []string `yaml:"comment,omitempty,flow" json:"comment,omitempty" jsonschema:"description=Limits the execution of a step to matching a pull request comment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		// enums for 'event' jsonschema are set in JSONSchemaExtend() method below
+		Event  []string `yaml:"event,omitempty,flow"  json:"event,omitempty"  jsonschema:"description=Limits the execution of a step to matching build events.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Path   []string `yaml:"path,omitempty,flow"   json:"path,omitempty"   jsonschema:"description=Limits the execution of a step to matching files changed in a repository.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Repo   []string `yaml:"repo,omitempty,flow"   json:"repo,omitempty"   jsonschema:"description=Limits the execution of a step to matching repos.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		Sender []string `yaml:"sender,omitempty,flow" json:"sender,omitempty" jsonschema:"description=Limits the execution of a step to matching build senders.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
+		// enums for 'status' jsonschema are set in JSONSchemaExtend() method below
+		Status   []string `yaml:"status,omitempty,flow"   json:"status,omitempty"   jsonschema:"description=Limits the execution of a step to matching build statuses.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
 		Tag      []string `yaml:"tag,omitempty,flow"      json:"tag,omitempty"      jsonschema:"description=Limits the execution of a step to matching build tag references.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
 		Target   []string `yaml:"target,omitempty,flow"   json:"target,omitempty"   jsonschema:"description=Limits the execution of a step to matching build deployment targets.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
 		Label    []string `yaml:"label,omitempty,flow"    json:"label,omitempty"    jsonschema:"description=Limits step execution to match on pull requests labels.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"`
@@ -186,3 +190,89 @@ func (r *Rules) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 	return err
 }
+
+// JSONSchemaExtend handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Mainly it handles the fact that all Rules fields are raw.StringSlice
+// but also handles adding enums to select fields as they would be too
+// cumbersome to maintain in the jsonschema struct tag.
+func (Rules) JSONSchemaExtend(schema *jsonschema.Schema) {
+	for item := schema.Properties.Newest(); item != nil; item = item.Prev() {
+		currSchema := *item.Value
+
+		// store the current description so we can lift it to top level
+		currDescription := currSchema.Description
+		currSchema.Description = ""
+
+		// handle each field as needed
+		switch item.Key {
+		case "status":
+			// possible values for 'status'
+			enums := []string{
+				"success",
+				"failure",
+			}
+
+			for _, str := range enums {
+				currSchema.Items.Enum = append(currSchema.Items.Enum, str)
+			}
+
+			schema.Properties.Set(item.Key, &jsonschema.Schema{
+				OneOf: []*jsonschema.Schema{
+					&currSchema,
+					{
+						Type: "string",
+						Enum: currSchema.Items.Enum,
+					},
+				},
+				Description: currDescription,
+			})
+		case "event":
+			// possible values for 'event'
+			enums := []string{
+				"comment",
+				"comment:created",
+				"comment:edited",
+				"delete:branch",
+				"delete:tag",
+				"deployment",
+				"pull_request",
+				"pull_request*",
+				"pull_request:edited",
+				"pull_request:labeled",
+				"pull_request:opened",
+				"pull_request:reopened",
+				"pull_request:synchronize",
+				"pull_request:unlabeled",
+				"push",
+				"schedule",
+				"tag",
+			}
+
+			for _, str := range enums {
+				currSchema.Items.Enum = append(currSchema.Items.Enum, str)
+			}
+
+			schema.Properties.Set(item.Key, &jsonschema.Schema{
+				OneOf: []*jsonschema.Schema{
+					&currSchema,
+					{
+						Type: "string",
+						Enum: currSchema.Items.Enum,
+					},
+				},
+				Description: currDescription,
+			})
+		default:
+			// all other fields are raw.StringSlice
+			schema.Properties.Set(item.Key, &jsonschema.Schema{
+				OneOf: []*jsonschema.Schema{
+					&currSchema,
+					{Type: "string"},
+				},
+				Description: currDescription,
+			})
+		}
+	}
+}

compiler/types/yaml/secret.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/invopop/jsonschema"
+
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
 	"github.com/go-vela/server/constants"
@@ -203,8 +205,8 @@ type (
 	// StepSecret is the yaml representation of a secret
 	// from a secrets block for a step in a pipeline.
 	StepSecret struct {
-		Source string `yaml:"source,omitempty"`
-		Target string `yaml:"target,omitempty"`
+		Source string `yaml:"source,omitempty" json:"source"`
+		Target string `yaml:"target,omitempty" json:"target"`
 	}
 )
 
@@ -269,3 +271,22 @@ func (s *StepSecretSlice) UnmarshalYAML(unmarshal func(interface{}) error) error
 
 	return errors.New("failed to unmarshal StepSecretSlice")
 }
+
+// JSONSchemaExtend handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Allows using simple strings or objects.
+func (StepSecret) JSONSchemaExtend(schema *jsonschema.Schema) {
+	old := *schema
+	schema.OneOf = []*jsonschema.Schema{
+		{
+			Type:                 "string",
+			AdditionalProperties: jsonschema.FalseSchema,
+		},
+		&old,
+	}
+	schema.Type = ""
+	schema.Required = nil
+	schema.AdditionalProperties = nil
+	schema.Properties = nil
+}

compiler/types/yaml/stage.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/buildkite/yaml"
+	"github.com/invopop/jsonschema"
 
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
@@ -123,6 +124,22 @@ func (s StageSlice) MarshalYAML() (interface{}, error) {
 	return output, nil
 }
 
+// JSONSchemaExtend handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Stages are not really a slice of stages to the user. This change
+// supports the map they really are.
+func (StageSlice) JSONSchemaExtend(schema *jsonschema.Schema) {
+	schema.Type = "object"
+	schema.Items = nil
+	schema.PatternProperties = map[string]*jsonschema.Schema{
+		".*": {
+			Ref: "#/$defs/Stage",
+		},
+	}
+	schema.AdditionalProperties = jsonschema.FalseSchema
+}
+
 // MergeEnv takes a list of environment variables and attempts
 // to set them in the stage environment. If the environment
 // variable already exists in the stage, than this will

compiler/types/yaml/ulimit.go

@@ -7,6 +7,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/invopop/jsonschema"
+
 	"github.com/go-vela/server/compiler/types/pipeline"
 	"github.com/go-vela/server/compiler/types/raw"
 )
@@ -130,3 +132,32 @@ func (u *UlimitSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
 
 	return nil
 }
+
+// JSONSchemaExtend handles some overrides that need to be in place
+// for this type for the jsonschema generation.
+//
+// Without these changes it would only allow an object per the struct,
+// but we do some special handling to allow specially formatted strings.
+func (Ulimit) JSONSchemaExtend(schema *jsonschema.Schema) {
+	oldProps := schema.Properties
+	oldReq := schema.Required
+	oldAddProps := schema.AdditionalProperties
+
+	schema.Type = ""
+	schema.Properties = nil
+	schema.AdditionalProperties = nil
+	schema.Required = nil
+	schema.OneOf = []*jsonschema.Schema{
+		{
+			Type:                 "string",
+			Pattern:              "[a-z]+=[0-9]+:[0-9]+",
+			AdditionalProperties: oldAddProps,
+		},
+		{
+			Type:                 "object",
+			Properties:           oldProps,
+			Required:             oldReq,
+			AdditionalProperties: oldAddProps,
+		},
+	}
+}