Skip to content

Commit

Permalink
bib: use the new uploader interface from images
Browse files Browse the repository at this point in the history
This commit switches bib to the new uploader interface in images.

This requires osbuild/images#1185
  • Loading branch information
mvo5 committed Feb 3, 2025
1 parent 089b467 commit 3d5b28d
Show file tree
Hide file tree
Showing 8 changed files with 208 additions and 369 deletions.
49 changes: 28 additions & 21 deletions bib/cmd/bootc-image-builder/cloud.go
Original file line number Diff line number Diff line change
@@ -1,35 +1,22 @@
package main

import (
"fmt"
"io"
"os"

"github.com/cheggaaa/pb/v3"
"github.com/osbuild/bootc-image-builder/bib/internal/uploader"
"github.com/osbuild/images/pkg/cloud/awscloud"
"github.com/spf13/pflag"

"github.com/osbuild/images/pkg/cloud"
)

func uploadAMI(path, targetArch string, flags *pflag.FlagSet) error {
region, err := flags.GetString("aws-region")
if err != nil {
return err
}
bucketName, err := flags.GetString("aws-bucket")
if err != nil {
return err
}
imageName, err := flags.GetString("aws-ami-name")
if err != nil {
return err
}
func upload(uploader cloud.Uploader, path string, flags *pflag.FlagSet) error {
progress, err := flags.GetString("progress")
if err != nil {
return err
}

client, err := awscloud.NewDefault(region)
if err != nil {
return err
}

// TODO: extract this as a helper once we add "uploadAzure" or
// similar. Eventually we may provide json progress here too.
var pbar *pb.ProgressBar

Check failure on line 22 in bib/cmd/bootc-image-builder/cloud.go

View workflow job for this annotation

GitHub Actions / ⌨ Lint & unittests

undefined: pb (typecheck)
Expand All @@ -38,5 +25,25 @@ func uploadAMI(path, targetArch string, flags *pflag.FlagSet) error {
pbar = pb.New(0)

Check failure on line 25 in bib/cmd/bootc-image-builder/cloud.go

View workflow job for this annotation

GitHub Actions / ⌨ Lint & unittests

undefined: pb (typecheck)
}

return uploader.UploadAndRegister(client, path, bucketName, imageName, targetArch, pbar)
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("cannot upload: %v", err)
}
defer file.Close()

var r io.Reader = file
if pbar != nil {
st, err := file.Stat()
if err != nil {
return err
}
pbar.SetTotal(st.Size())
pbar.Set(pb.Bytes, true)

Check failure on line 41 in bib/cmd/bootc-image-builder/cloud.go

View workflow job for this annotation

GitHub Actions / ⌨ Lint & unittests

undefined: pb (typecheck)
pbar.SetWriter(osStdout)
r = pbar.NewProxyReader(file)
pbar.Start()
defer pbar.Finish()
}

return uploader.UploadAndRegister(r, osStderr)
}
58 changes: 23 additions & 35 deletions bib/cmd/bootc-image-builder/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"encoding/json"
"fmt"
"io"
"log"
"os"
"os/exec"
Expand All @@ -17,6 +18,7 @@ import (
"golang.org/x/exp/slices"

"github.com/osbuild/images/pkg/arch"
"github.com/osbuild/images/pkg/cloud"
"github.com/osbuild/images/pkg/cloud/awscloud"
"github.com/osbuild/images/pkg/container"
"github.com/osbuild/images/pkg/dnfjson"
Expand Down Expand Up @@ -53,6 +55,9 @@ var distroDefPaths = []string{
var (
osGetuid = os.Getuid
osGetgid = os.Getgid

osStdout = os.Stdout
osStderr = os.Stderr
)

// canChownInPath checks if the ownership of files can be set in a given path.
Expand Down Expand Up @@ -355,53 +360,36 @@ func cmdManifest(cmd *cobra.Command, args []string) error {
return nil
}

func handleAWSFlags(cmd *cobra.Command) (upload bool, err error) {
func handleAWSFlags(cmd *cobra.Command) (cloud.Uploader, error) {
imgTypes, _ := cmd.Flags().GetStringArray("type")
region, _ := cmd.Flags().GetString("aws-region")
if region == "" {
return false, nil
return nil, nil
}
bucketName, _ := cmd.Flags().GetString("aws-bucket")
imageName, _ := cmd.Flags().GetString("aws-ami-name")
targetArch, _ := cmd.Flags().GetString("target-arch")

if !slices.Contains(imgTypes, "ami") {

Check failure on line 373 in bib/cmd/bootc-image-builder/main.go

View workflow job for this annotation

GitHub Actions / ⌨ Lint & unittests

cannot infer S (/home/runner/go/pkg/mod/golang.org/x/[email protected]/slices/slices.go:116:1) (typecheck)
return false, fmt.Errorf("aws flags set for non-ami image type (type is set to %s)", strings.Join(imgTypes, ","))
return nil, fmt.Errorf("aws flags set for non-ami image type (type is set to %s)", strings.Join(imgTypes, ","))
}

// check as many permission prerequisites as possible before starting
client, err := awscloud.NewDefault(region)
if err != nil {
return false, err
}

logrus.Info("Checking AWS region access...")
regions, err := client.Regions()
if err != nil {
return false, fmt.Errorf("retrieving AWS regions for '%s' failed: %w", region, err)
}

if !slices.Contains(regions, region) {
return false, fmt.Errorf("given AWS region '%s' not found", region)
uploaderOpts := &awscloud.UploaderOptions{
TargetArch: targetArch,
}

logrus.Info("Checking AWS bucket...")
buckets, err := client.Buckets()
uploader, err := awscloud.NewUploader(region, bucketName, imageName, uploaderOpts)
if err != nil {
return false, fmt.Errorf("retrieving AWS list of buckets failed: %w", err)
}
if !slices.Contains(buckets, bucketName) {
return false, fmt.Errorf("bucket '%s' not found in the given AWS account", bucketName)
return nil, err
}

logrus.Info("Checking AWS bucket permissions...")
writePermission, err := client.CheckBucketPermission(bucketName, awscloud.S3PermissionWrite)
if err != nil {
return false, err
status := io.Discard
if logrus.GetLevel() >= logrus.InfoLevel {
status = os.Stderr
}
if !writePermission {
return false, fmt.Errorf("you don't have write permissions to bucket '%s' with the given AWS account", bucketName)
if err := uploader.Check(status); err != nil {
return nil, err
}
logrus.Info("Upload conditions met.")
return true, nil
return uploader, nil
}

func cmdBuild(cmd *cobra.Command, args []string) error {
Expand Down Expand Up @@ -430,7 +418,7 @@ func cmdBuild(cmd *cobra.Command, args []string) error {
return fmt.Errorf("cannot setup build dir: %w", err)
}

upload, err := handleAWSFlags(cmd)
uploader, err := handleAWSFlags(cmd)
if err != nil {
return fmt.Errorf("cannot handle AWS setup: %w", err)
}
Expand Down Expand Up @@ -493,7 +481,7 @@ func cmdBuild(cmd *cobra.Command, args []string) error {
}

pbar.SetMessagef("Build complete!")
if upload {
if uploader != nil {
// XXX: pass our own progress.ProgressBar here
// *for now* just stop our own progress and let the uploadAMI
// progress take over - but we really need to fix this in a
Expand All @@ -503,7 +491,7 @@ func cmdBuild(cmd *cobra.Command, args []string) error {
switch imgType {
case "ami":
diskpath := filepath.Join(outputDir, exports[idx], "disk.raw")
if err := uploadAMI(diskpath, targetArch, cmd.Flags()); err != nil {
if err := upload(uploader, diskpath, cmd.Flags()); err != nil {
return fmt.Errorf("cannot upload AMI: %w", err)
}
default:
Expand Down
15 changes: 11 additions & 4 deletions bib/cmd/upload/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"

"github.com/osbuild/bootc-image-builder/bib/internal/uploader"
"github.com/osbuild/images/pkg/cloud/awscloud"
)

Expand All @@ -26,16 +25,24 @@ func uploadAMI(cmd *cobra.Command, args []string) {

region, err := flags.GetString("region")
check(err)
client, err := awscloud.NewDefault(region)
check(err)
bucketName, err := flags.GetString("bucket")
check(err)
imageName, err := flags.GetString("ami-name")
check(err)
targetArch, err := flags.GetString("target-arch")
check(err)

check(uploader.UploadAndRegister(client, filename, bucketName, imageName, targetArch, nil))
opts := &awscloud.UploaderOptions{
TargetArch: targetArch,
}
uploader, err := awscloud.NewUploader(region, bucketName, imageName, opts)
check(err)

f, err := os.Open(filename)
check(err)
defer f.Close()

check(uploader.UploadAndRegister(f, os.Stderr))
}

func setupCLI() *cobra.Command {
Expand Down
74 changes: 39 additions & 35 deletions bib/go.mod
Original file line number Diff line number Diff line change
@@ -1,51 +1,54 @@
module github.com/osbuild/bootc-image-builder/bib

go 1.22.0
go 1.22.6

toolchain go1.23.5

require (
github.com/BurntSushi/toml v1.4.0
github.com/aws/aws-sdk-go v1.55.6
github.com/cheggaaa/pb/v3 v3.1.6
github.com/google/uuid v1.6.0
github.com/hashicorp/go-version v1.7.0
github.com/mattn/go-isatty v0.0.20
github.com/osbuild/images v0.112.0
github.com/osbuild/images v0.114.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.1
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.10.0
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
golang.org/x/sys v0.29.0
gopkg.in/yaml.v3 v3.0.1
)

require (
dario.cat/mergo v1.0.0 // indirect
dario.cat/mergo v1.0.1 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/Microsoft/hcsshim v0.12.5 // indirect
github.com/Microsoft/hcsshim v0.12.9 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go v1.55.6 // indirect
github.com/containerd/cgroups/v3 v3.0.3 // indirect
github.com/containerd/errdefs v0.1.0 // indirect
github.com/containerd/errdefs v0.3.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
github.com/containers/common v0.60.4 // indirect
github.com/containers/image/v5 v5.32.2 // indirect
github.com/containerd/typeurl/v2 v2.2.0 // indirect
github.com/containers/common v0.61.1 // indirect
github.com/containers/image/v5 v5.33.1 // indirect
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
github.com/containers/ocicrypt v1.2.0 // indirect
github.com/containers/storage v1.55.0 // indirect
github.com/containers/storage v1.56.1 // indirect
github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f // indirect
github.com/cyphar/filepath-securejoin v0.3.1 // indirect
github.com/cyphar/filepath-securejoin v0.3.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker v27.1.1+incompatible // indirect
github.com/docker/docker v27.3.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.8.2 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.0.2 // indirect
github.com/go-jose/go-jose/v4 v4.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/analysis v0.23.0 // indirect
Expand All @@ -62,66 +65,67 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-containerregistry v0.20.0 // indirect
github.com/google/go-containerregistry v0.20.2 // indirect
github.com/google/go-intervals v0.0.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/letsencrypt/boulder v0.0.0-20240418210053-89b07f4543e0 // indirect
github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/mattn/go-sqlite3 v1.14.24 // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/mistifyio/go-zfs/v3 v3.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/capability v0.3.0 // indirect
github.com/moby/sys/mountinfo v0.7.2 // indirect
github.com/moby/sys/user v0.2.0 // indirect
github.com/moby/sys/user v0.3.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/opencontainers/selinux v1.11.1 // indirect
github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/proglottis/gpgme v0.1.3 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/secure-systems-lab/go-securesystemslib v0.8.0 // indirect
github.com/sigstore/fulcio v1.4.5 // indirect
github.com/sigstore/fulcio v1.6.4 // indirect
github.com/sigstore/rekor v1.3.6 // indirect
github.com/sigstore/sigstore v1.8.4 // indirect
github.com/sigstore/sigstore v1.8.9 // indirect
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect
github.com/sylabs/sif/v2 v2.18.0 // indirect
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
github.com/sylabs/sif/v2 v2.19.1 // indirect
github.com/tchap/go-patricia/v2 v2.3.1 // indirect
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect
github.com/ulikunitz/xz v0.5.12 // indirect
github.com/vbatts/tar-split v0.11.5 // indirect
github.com/vbauerster/mpb/v8 v8.7.5 // indirect
github.com/vbatts/tar-split v0.11.6 // indirect
github.com/vbauerster/mpb/v8 v8.8.3 // indirect
go.mongodb.org/mongo-driver v1.14.0 // indirect
go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect
go.opentelemetry.io/otel/metric v1.29.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/net v0.33.0 // indirect
go.opentelemetry.io/otel v1.32.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect
go.opentelemetry.io/otel/trace v1.32.0 // indirect
golang.org/x/crypto v0.32.0 // indirect
golang.org/x/net v0.34.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/term v0.27.0 // indirect
golang.org/x/term v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
google.golang.org/grpc v1.67.3 // indirect
google.golang.org/protobuf v1.35.2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250124145028-65684f501c47 // indirect
google.golang.org/grpc v1.70.0 // indirect
google.golang.org/protobuf v1.36.4 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
)
Loading

0 comments on commit 3d5b28d

Please sign in to comment.