Skip to content

Commit

Permalink
Resolve package collisions (#153)
Browse files Browse the repository at this point in the history
## Proposed Changes

This PR ensures the correct CRD versions are used when parsing.
Previously, the Swagger version generated by `builder.BuildOpenAPIV2`
defaulted to `0.1.0`, but we should be pulling the version directly from
the CRD spec itself. This change ensures that SDKs are generated for all
CRD versions. Additionally, this PR incorporates the latest updates from
p-k to prevent package collisions.

## Related Issues (optional)
Closes: #152
  • Loading branch information
rquitales authored Oct 1, 2024
1 parent d05a577 commit 7a5693d
Show file tree
Hide file tree
Showing 6 changed files with 268 additions and 31 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

## Unreleased

## 1.5.3 (2024-09-30)

- Fix crd2pulumi not generating all CRD versions. [#152](https://github.com/pulumi/crd2pulumi/issues/152)
- Fix crd2pulumi generating packages and types with incorrect group names. [#152](https://github.com/pulumi/crd2pulumi/issues/152)

## 1.5.2 (2024-09-16)

- Set the pulumi-kubernetes dependency for Python packages to v4.18.0. [#148](https://github.com/pulumi/crd2pulumi/issues/148)
Expand Down
12 changes: 5 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ toolchain go1.22.7
require (
github.com/go-openapi/jsonreference v0.21.0
github.com/iancoleman/strcase v0.3.0
github.com/pulumi/pulumi-java/pkg v0.14.0
github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240916231657-74673fdf488b
github.com/pulumi/pulumi/pkg/v3 v3.130.0
github.com/pulumi/pulumi/sdk/v3 v3.130.0
github.com/pulumi/pulumi-java/pkg v0.16.1
github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240930223531-b7dae6b4ea56
github.com/pulumi/pulumi/pkg/v3 v3.134.1
github.com/pulumi/pulumi/sdk/v3 v3.134.1
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
golang.org/x/text v0.17.0
Expand Down Expand Up @@ -41,7 +41,6 @@ require (
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/ahmetb/go-linq v3.0.0+incompatible // indirect
github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
Expand Down Expand Up @@ -181,7 +180,7 @@ require (
github.com/prometheus/procfs v0.15.1 // indirect
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
github.com/pulumi/cloud-ready-checks v1.1.1-0.20240731201114-3a703c6bee71 // indirect
github.com/pulumi/esc v0.9.1 // indirect
github.com/pulumi/esc v0.10.0 // indirect
github.com/pulumi/inflector v0.1.1 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
Expand All @@ -197,7 +196,6 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
github.com/x448/float16 v0.8.4 // indirect
Expand Down
24 changes: 10 additions & 14 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmH
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/ahmetb/go-linq v3.0.0+incompatible h1:qQkjjOXKrKOTy83X8OpRmnKflXKQIL/mC/gMVVDMhOA=
github.com/ahmetb/go-linq v3.0.0+incompatible/go.mod h1:PFffvbdbtw+QTB0WKRP0cNht7vnCfnGlEpak/DVg5cY=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
Expand Down Expand Up @@ -469,18 +467,18 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE=
github.com/pulumi/cloud-ready-checks v1.1.1-0.20240731201114-3a703c6bee71 h1:r0nNcIW3Y+wvXBfstXoD+ALyqJnIbcCxbDz0pHME+pQ=
github.com/pulumi/cloud-ready-checks v1.1.1-0.20240731201114-3a703c6bee71/go.mod h1:Rkj3dUQiwjbCpuk+6d2fAtSzSbDiaJVx2pI4kBB/HkY=
github.com/pulumi/esc v0.9.1 h1:HH5eEv8sgyxSpY5a8yePyqFXzA8cvBvapfH8457+mIs=
github.com/pulumi/esc v0.9.1/go.mod h1:oEJ6bOsjYlQUpjf70GiX+CXn3VBmpwFDxUTlmtUN84c=
github.com/pulumi/esc v0.10.0 h1:jzBKzkLVW0mePeanDRfqSQoCJ5yrkux0jIwAkUxpRKE=
github.com/pulumi/esc v0.10.0/go.mod h1:2Bfa+FWj/xl8CKqRTWbWgDX0SOD4opdQgvYSURTGK2c=
github.com/pulumi/inflector v0.1.1 h1:dvlxlWtXwOJTUUtcYDvwnl6Mpg33prhK+7mzeF+SobA=
github.com/pulumi/inflector v0.1.1/go.mod h1:HUFCjcPTz96YtTuUlwG3i3EZG4WlniBvR9bd+iJxCUY=
github.com/pulumi/pulumi-java/pkg v0.14.0 h1:CKL7lLF81Fq6VRhA5TNFsSMnHraTNCUzIhqCzYX8Wzk=
github.com/pulumi/pulumi-java/pkg v0.14.0/go.mod h1:VybuJMWJtJc9ZNbt1kcYH4TbpocMx9mEi7YWL2Co99c=
github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240916231657-74673fdf488b h1:Eb5/zL+RY2TnHn0zciQ+3lylAuvsWoi8XvLKRpc9xGE=
github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240916231657-74673fdf488b/go.mod h1:glS84cIQOYkQBbAtHQjlNnx28XAZrPMwlPah4+Izj2U=
github.com/pulumi/pulumi/pkg/v3 v3.130.0 h1:lS51XeCnhg72LXkMiw2FP1cGP+Y85wYD3quWhCPD5+M=
github.com/pulumi/pulumi/pkg/v3 v3.130.0/go.mod h1:jhZ1Ug5Rl1qivexgEWvmwSWYIT/jRnKSFhLwwv6PrZ0=
github.com/pulumi/pulumi/sdk/v3 v3.130.0 h1:gGJNd+akPqhZ+vrsZmAjSNJn6kGJkitjjkwrmIQMmn8=
github.com/pulumi/pulumi/sdk/v3 v3.130.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY=
github.com/pulumi/pulumi-java/pkg v0.16.1 h1:orHnDWFbpOERwaBLry9f+6nqPX7x0MsrIkaa5QDGAns=
github.com/pulumi/pulumi-java/pkg v0.16.1/go.mod h1:QH0DihZkWYle9XFc+LJ76m4hUo+fA3RdyaM90pqOaSM=
github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240930223531-b7dae6b4ea56 h1:r0lPAtVXpold10wxlSz8YkMM2w336xKMzxkeUUNP+h4=
github.com/pulumi/pulumi-kubernetes/provider/v4 v4.0.0-20240930223531-b7dae6b4ea56/go.mod h1:caVn9OjrsHrldGBZ4/VvZreVCV46asZ0DRnkONEja38=
github.com/pulumi/pulumi/pkg/v3 v3.134.1 h1:iGKvaSHEoPCGBqDoIGQUXUm3qkrawfd513lL0I9vnNQ=
github.com/pulumi/pulumi/pkg/v3 v3.134.1/go.mod h1:1iCee1QIwXYvkIQJ/HnBjsPsmYJ/arBPWX6hAao/Pro=
github.com/pulumi/pulumi/sdk/v3 v3.134.1 h1:v1zd0d+B9gpUhsdJ483YUMHwHXqDvXvZ+mh/A4HhPWg=
github.com/pulumi/pulumi/sdk/v3 v3.134.1/go.mod h1:J5kQEX8v87aeUhk6NdQXnjCo1DbiOnOiL3Sf2DuDda8=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
Expand Down Expand Up @@ -536,8 +534,6 @@ github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqa
github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 h1:X9dsIWPuuEJlPX//UmRKophhOKCGXc46RVIGuttks68=
github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7/go.mod h1:UxoP3EypF8JfGEjAII8jx1q8rQyDnX8qdTCs/UQBVIE=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg=
Expand Down
13 changes: 5 additions & 8 deletions pkg/codegen/customresourcegenerator.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,15 +214,12 @@ func sanitizeReferenceName(fieldName string) string {
}

// crdToOpenAPI generates the OpenAPI specs for a given CRD manifest.
func crdToOpenAPI(crd *extensionv1.CustomResourceDefinition) ([]*spec.Swagger, error) {
var openAPIManifests []*spec.Swagger
func crdToOpenAPI(crd *extensionv1.CustomResourceDefinition) (map[string]*spec.Swagger, error) {
openAPIManifests := make(map[string]*spec.Swagger)

setCRDDefaults(crd)

for _, v := range crd.Spec.Versions {
if !v.Served {
continue
}
// Defaults are not pruned here, but before being served.
sw, err := builder.BuildOpenAPIV2(crd, v.Name, builder.Options{V2: true, StripValueValidation: true, StripNullable: true, AllowNonStructural: true, IncludeSelectableFields: true})
if err != nil {
Expand All @@ -234,7 +231,7 @@ func crdToOpenAPI(crd *extensionv1.CustomResourceDefinition) ([]*spec.Swagger, e
return nil, fmt.Errorf("error flattening OpenAPI spec: %w", err)
}

openAPIManifests = append(openAPIManifests, sw)
openAPIManifests[v.Name] = sw
}

return openAPIManifests, nil
Expand All @@ -261,8 +258,8 @@ func NewCustomResourceGenerator(crd extensionv1.CustomResourceDefinition) (Custo
return CustomResourceGenerator{}, fmt.Errorf("could not generate OpenAPI spec for CRD: %w", err)
}

for _, sw := range swagger {
schemas[sw.Info.Version] = *sw
for version, sw := range swagger {
schemas[version] = *sw
}

kind := crd.Spec.Names.Kind
Expand Down
233 changes: 233 additions & 0 deletions pkg/codegen/customresourcegenerator_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
// Copyright 2016-2024, Pulumi Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package codegen

import (
"testing"

extensionv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestSetCRDDefaults(t *testing.T) {
tests := []struct {
name string
crd extensionv1.CustomResourceDefinition
expected extensionv1.CustomResourceDefinition
}{
{
name: "Singular and ListKind are empty",
crd: extensionv1.CustomResourceDefinition{
Spec: extensionv1.CustomResourceDefinitionSpec{
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
},
},
},
expected: extensionv1.CustomResourceDefinition{
Spec: extensionv1.CustomResourceDefinitionSpec{
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
Singular: "testkind",
ListKind: "TestKindList",
},
},
},
},
{
name: "Singular is set, ListKind is empty",
crd: extensionv1.CustomResourceDefinition{
Spec: extensionv1.CustomResourceDefinitionSpec{
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
Singular: "customsingular",
},
},
},
expected: extensionv1.CustomResourceDefinition{
Spec: extensionv1.CustomResourceDefinitionSpec{
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
Singular: "customsingular",
ListKind: "TestKindList",
},
},
},
},
{
name: "Singular is empty, ListKind is set",
crd: extensionv1.CustomResourceDefinition{
Spec: extensionv1.CustomResourceDefinitionSpec{
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
ListKind: "CustomListKind",
},
},
},
expected: extensionv1.CustomResourceDefinition{
Spec: extensionv1.CustomResourceDefinitionSpec{
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
Singular: "testkind",
ListKind: "CustomListKind",
},
},
},
},
{
name: "Singular and ListKind are set",
crd: extensionv1.CustomResourceDefinition{
Spec: extensionv1.CustomResourceDefinitionSpec{
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
Singular: "customsingular",
ListKind: "CustomListKind",
},
},
},
expected: extensionv1.CustomResourceDefinition{
Spec: extensionv1.CustomResourceDefinitionSpec{
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
Singular: "customsingular",
ListKind: "CustomListKind",
},
},
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
setCRDDefaults(&tt.crd)
if tt.crd.Spec.Names.Singular != tt.expected.Spec.Names.Singular {
t.Errorf("expected Singular %s, got %s", tt.expected.Spec.Names.Singular, tt.crd.Spec.Names.Singular)
}
if tt.crd.Spec.Names.ListKind != tt.expected.Spec.Names.ListKind {
t.Errorf("expected ListKind %s, got %s", tt.expected.Spec.Names.ListKind, tt.crd.Spec.Names.ListKind)
}
})
}
}
func TestNewCustomResourceGenerator(t *testing.T) {
tests := []struct {
name string
crd extensionv1.CustomResourceDefinition
expected CustomResourceGenerator
wantErr bool
}{
{
name: "Valid CRD",
crd: extensionv1.CustomResourceDefinition{
TypeMeta: metav1.TypeMeta{
APIVersion: "apiextensions.k8s.io/v1",
},
Spec: extensionv1.CustomResourceDefinitionSpec{
Group: "example.com",
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
Plural: "testkinds",
},
Versions: []extensionv1.CustomResourceDefinitionVersion{
{Name: "v1"},
},
},
},
expected: CustomResourceGenerator{
APIVersion: "apiextensions.k8s.io/v1",
Kind: "TestKind",
Plural: "testkinds",
Group: "example.com",
Versions: []string{"v1"},
GroupVersions: []string{
"example.com/v1",
},
ResourceTokens: []string{
"example.com:TestKind:v1",
},
},
wantErr: false,
},
{
name: "Valid CRD with multiple versions",
crd: extensionv1.CustomResourceDefinition{
TypeMeta: metav1.TypeMeta{
APIVersion: "apiextensions.k8s.io/v1",
},
Spec: extensionv1.CustomResourceDefinitionSpec{
Group: "example.com",
Names: extensionv1.CustomResourceDefinitionNames{
Kind: "TestKind",
Plural: "testkinds",
},
Versions: []extensionv1.CustomResourceDefinitionVersion{
{Name: "v1"},
{Name: "v1alpha1"},
},
},
},
expected: CustomResourceGenerator{
APIVersion: "apiextensions.k8s.io/v1",
Kind: "TestKind",
Plural: "testkinds",
Group: "example.com",
Versions: []string{"v1alpha1", "v1"},
GroupVersions: []string{
"example.com/v1",
"example.com/v1alpha1",
},
ResourceTokens: []string{
"example.com:TestKind:v1",
"example.com:TestKind:v1alpha1",
},
},
wantErr: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NewCustomResourceGenerator(tt.crd)
if (err != nil) != tt.wantErr {
t.Errorf("NewCustomResourceGenerator() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.wantErr != false {
return
}
if got.APIVersion != tt.expected.APIVersion {
t.Errorf("expected APIVersion %s, got %s", tt.expected.APIVersion, got.APIVersion)
}
if got.Kind != tt.expected.Kind {
t.Errorf("expected Kind %s, got %s", tt.expected.Kind, got.Kind)
}
if got.Plural != tt.expected.Plural {
t.Errorf("expected Plural %s, got %s", tt.expected.Plural, got.Plural)
}
if got.Group != tt.expected.Group {
t.Errorf("expected Group %s, got %s", tt.expected.Group, got.Group)
}
if len(got.Versions) != len(tt.expected.Versions) {
t.Errorf("expected Versions %v, got %v", tt.expected.Versions, got.Versions)
}
if len(got.GroupVersions) != len(tt.expected.GroupVersions) {
t.Errorf("expected GroupVersions %v, got %v", tt.expected.GroupVersions, got.GroupVersions)
}
if len(got.ResourceTokens) != len(tt.expected.ResourceTokens) {
t.Errorf("expected ResourceTokens %v, got %v", tt.expected.ResourceTokens, got.ResourceTokens)
}
})
}
}
12 changes: 10 additions & 2 deletions tests/crds_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,17 @@ func TestCRDsFromUrl(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
for _, lang := range languages {
t.Run(lang, func(t *testing.T) {
if lang == "dotnet" && (tt.name == "CertManager" || tt.name == "GKEManagedCerts") {
t.Skip("Skipping compilation for dotnet. See https://github.com/pulumi/crd2pulumi/issues/17")
if lang == "dotnet" {
if tt.name == "CertManager" || tt.name == "GKEManagedCerts" {
t.Skip("Skipping compilation for dotnet. See https://github.com/pulumi/crd2pulumi/issues/17")
}

if tt.name == "Percona" {
t.Skip("Skipping dotnet compilation for Percona as we generate invalid code with hyphens that are not allowed in C# identifiers.")
}

}

execCrd2Pulumi(t, lang, tt.url, compileValidationFn[lang])
})
}
Expand Down

0 comments on commit 7a5693d

Please sign in to comment.